@dbos-inc/dbos-sdk 5.4.10-preview → 5.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/dist/schemas/system_db_schema.d.ts +7 -0
  2. package/dist/schemas/system_db_schema.d.ts.map +1 -1
  3. package/dist/schemas/system_db_schema.js.map +1 -1
  4. package/dist/src/cli/cli.js +56 -13
  5. package/dist/src/cli/cli.js.map +1 -1
  6. package/dist/src/client.d.ts +44 -10
  7. package/dist/src/client.d.ts.map +1 -1
  8. package/dist/src/client.js +140 -44
  9. package/dist/src/client.js.map +1 -1
  10. package/dist/src/config.d.ts +1 -0
  11. package/dist/src/config.d.ts.map +1 -1
  12. package/dist/src/config.js +4 -0
  13. package/dist/src/config.js.map +1 -1
  14. package/dist/src/context.d.ts +6 -0
  15. package/dist/src/context.d.ts.map +1 -1
  16. package/dist/src/context.js +15 -1
  17. package/dist/src/context.js.map +1 -1
  18. package/dist/src/datasource.d.ts +2 -0
  19. package/dist/src/datasource.d.ts.map +1 -1
  20. package/dist/src/datasource.js +7 -1
  21. package/dist/src/datasource.js.map +1 -1
  22. package/dist/src/dbos-executor.d.ts +32 -4
  23. package/dist/src/dbos-executor.d.ts.map +1 -1
  24. package/dist/src/dbos-executor.js +102 -33
  25. package/dist/src/dbos-executor.js.map +1 -1
  26. package/dist/src/dbos.d.ts +104 -15
  27. package/dist/src/dbos.d.ts.map +1 -1
  28. package/dist/src/dbos.js +275 -86
  29. package/dist/src/dbos.js.map +1 -1
  30. package/dist/src/debugpoint.d.ts +1 -0
  31. package/dist/src/debugpoint.d.ts.map +1 -1
  32. package/dist/src/debugpoint.js +4 -2
  33. package/dist/src/debugpoint.js.map +1 -1
  34. package/dist/src/enqueue_options.d.ts +2 -0
  35. package/dist/src/enqueue_options.d.ts.map +1 -1
  36. package/dist/src/enqueue_options.js +3 -0
  37. package/dist/src/enqueue_options.js.map +1 -1
  38. package/dist/src/enqueue_workflow.d.ts.map +1 -1
  39. package/dist/src/enqueue_workflow.js +11 -5
  40. package/dist/src/enqueue_workflow.js.map +1 -1
  41. package/dist/src/enterprise.d.ts +78 -3
  42. package/dist/src/enterprise.d.ts.map +1 -1
  43. package/dist/src/enterprise.js +104 -45
  44. package/dist/src/enterprise.js.map +1 -1
  45. package/dist/src/error.d.ts +42 -0
  46. package/dist/src/error.d.ts.map +1 -1
  47. package/dist/src/error.js +85 -1
  48. package/dist/src/error.js.map +1 -1
  49. package/dist/src/eventreceiver.d.ts +2 -0
  50. package/dist/src/eventreceiver.d.ts.map +1 -1
  51. package/dist/src/eventreceiver.js +7 -1
  52. package/dist/src/eventreceiver.js.map +1 -1
  53. package/dist/src/external_storage.d.ts +27 -0
  54. package/dist/src/external_storage.d.ts.map +1 -0
  55. package/dist/src/external_storage.js +28 -0
  56. package/dist/src/external_storage.js.map +1 -0
  57. package/dist/src/index.d.ts +4 -3
  58. package/dist/src/index.d.ts.map +1 -1
  59. package/dist/src/index.js +2 -1
  60. package/dist/src/index.js.map +1 -1
  61. package/dist/src/scheduler/scheduler.d.ts +83 -3
  62. package/dist/src/scheduler/scheduler.d.ts.map +1 -1
  63. package/dist/src/scheduler/scheduler.js +422 -171
  64. package/dist/src/scheduler/scheduler.js.map +1 -1
  65. package/dist/src/shard_keys.d.ts +13 -0
  66. package/dist/src/shard_keys.d.ts.map +1 -0
  67. package/dist/src/shard_keys.js +55 -0
  68. package/dist/src/shard_keys.js.map +1 -0
  69. package/dist/src/streams.d.ts +2 -0
  70. package/dist/src/streams.d.ts.map +1 -1
  71. package/dist/src/streams.js +5 -2
  72. package/dist/src/streams.js.map +1 -1
  73. package/dist/src/sysdb_migrations/internal/migrations.js +82 -0
  74. package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
  75. package/dist/src/system_database.d.ts +185 -10
  76. package/dist/src/system_database.d.ts.map +1 -1
  77. package/dist/src/system_database.js +1429 -626
  78. package/dist/src/system_database.js.map +1 -1
  79. package/dist/src/utils.d.ts +2 -0
  80. package/dist/src/utils.d.ts.map +1 -1
  81. package/dist/src/utils.js +9 -1
  82. package/dist/src/utils.js.map +1 -1
  83. package/dist/src/wfqueue.d.ts +1 -2
  84. package/dist/src/wfqueue.d.ts.map +1 -1
  85. package/dist/src/wfqueue.js +83 -63
  86. package/dist/src/wfqueue.js.map +1 -1
  87. package/dist/src/workflow.d.ts +15 -3
  88. package/dist/src/workflow.d.ts.map +1 -1
  89. package/dist/src/workflow.js +30 -12
  90. package/dist/src/workflow.js.map +1 -1
  91. package/dist/src/workflow_management.d.ts +17 -3
  92. package/dist/src/workflow_management.d.ts.map +1 -1
  93. package/dist/src/workflow_management.js +93 -20
  94. package/dist/src/workflow_management.js.map +1 -1
  95. package/dist/tsconfig.tsbuildinfo +1 -1
  96. package/package.json +2 -2
@@ -1,15 +1,38 @@
1
1
  "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
2
18
  var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
19
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
20
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
21
  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
22
  return c > 3 && r && Object.defineProperty(target, key, r), r;
7
23
  };
24
+ var __importStar = (this && this.__importStar) || function (mod) {
25
+ if (mod && mod.__esModule) return mod;
26
+ var result = {};
27
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
28
+ __setModuleDefault(result, mod);
29
+ return result;
30
+ };
8
31
  var __metadata = (this && this.__metadata) || function (k, v) {
9
32
  if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
33
  };
11
34
  Object.defineProperty(exports, "__esModule", { value: true });
12
- exports.SystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
35
+ exports.SystemDatabase = exports.createSystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.qualify = exports.holdsShardedSchema = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.compareCodeUnits = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
13
36
  const dbos_executor_1 = require("./dbos-executor");
14
37
  const pg_1 = require("pg");
15
38
  const error_1 = require("./error");
@@ -18,6 +41,8 @@ const utils_1 = require("./utils");
18
41
  const context_1 = require("./context");
19
42
  const async_hooks_1 = require("async_hooks");
20
43
  const crypto_1 = require("crypto");
44
+ const shard_keys_1 = require("./shard_keys");
45
+ const enterprise = __importStar(require("./enterprise"));
21
46
  const utils_2 = require("./utils");
22
47
  const database_utils_1 = require("./database_utils");
23
48
  const migration_runner_1 = require("./sysdb_migrations/migration_runner");
@@ -70,6 +95,13 @@ function validateObservabilityQueryTimeoutMs(value) {
70
95
  exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutMs;
71
96
  // Bounds how long a frozen or unreachable client can hold system database locks.
72
97
  exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = 60_000;
98
+ // How long a background job's lock may sit idle before Postgres ends its holder's session.
99
+ exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = 5000;
100
+ /** Orders strings by code unit, which matches Postgres's "C" collation outside surrogate pairs. */
101
+ function compareCodeUnits(a, b) {
102
+ return a < b ? -1 : a > b ? 1 : 0;
103
+ }
104
+ exports.compareCodeUnits = compareCodeUnits;
73
105
  /** The timeout is rendered into a `SET idle_in_transaction_session_timeout`, which takes only an in-range integer. */
74
106
  function validateIdleTransactionTimeoutMs(value) {
75
107
  if (value !== undefined && (!Number.isFinite(value) || value > MAX_STATEMENT_TIMEOUT_MS)) {
@@ -157,6 +189,7 @@ async function grantDbosSchemaPermissions(databaseUrl, roleName, logger, schemaN
157
189
  await client.end();
158
190
  }
159
191
  }
192
+ exports.grantDbosSchemaPermissions = grantDbosSchemaPermissions;
160
193
  /**
161
194
  * Check out a connection carrying our own error handler, so a socket death while we hold it is
162
195
  * logged instead of thrown at an emitter with no listener. `release()` takes the handler back off,
@@ -219,6 +252,7 @@ async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = '
219
252
  }
220
253
  const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
221
254
  try {
255
+ await refuseShardedSchema(client, schemaName);
222
256
  const isCockroach = await isCockroachDB(client);
223
257
  await (0, migration_runner_1.runSysMigrationsPg)(client, (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }), schemaName, {
224
258
  onWarn: (e) => logger.info(e),
@@ -242,6 +276,7 @@ exports.migrateSystemDatabase = migrateSystemDatabase;
242
276
  async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
243
277
  const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
244
278
  try {
279
+ await refuseShardedSchema(client, schemaName);
245
280
  const isCockroach = await isCockroachDB(client);
246
281
  const requiredVersion = (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }).length;
247
282
  const currentVersion = await (0, migration_runner_1.getCurrentSysDBVersion)(client, schemaName);
@@ -259,6 +294,41 @@ async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = '
259
294
  }
260
295
  }
261
296
  exports.verifySystemDatabase = verifySystemDatabase;
297
+ /** Throw if this schema is sharded: an unsharded handle's statements would span every tenant's rows. */
298
+ async function refuseShardedSchema(client, schemaName) {
299
+ const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
300
+ WHERE table_schema = $1 AND table_name = 'workflow_status' AND column_name = 'shard_key'`, [schemaName]);
301
+ if (rows.length > 0) {
302
+ throw new error_1.DBOSError(`The DBOS system schema ${schemaName} is sharded. Enable sharding to use it: ` +
303
+ 'set sharding in your DBOS configuration, or pass sharding: true to DBOSClient.');
304
+ }
305
+ }
306
+ /** Whether this database holds a sharded DBOS system schema (this one, if named); false if it can't be reached. */
307
+ async function holdsShardedSchema(databaseUrl, schemaName) {
308
+ let client;
309
+ try {
310
+ // Bounded, so an unreachable host fails as fast as the caller's own connection would.
311
+ client = new pg_1.Client({ ...(0, utils_2.getClientConfig)(databaseUrl), connectionTimeoutMillis: 10_000 });
312
+ }
313
+ catch {
314
+ // A malformed URL, for the caller's own connection to report.
315
+ return false;
316
+ }
317
+ client.on('error', () => { });
318
+ try {
319
+ await client.connect();
320
+ const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
321
+ WHERE table_name = 'workflow_status' AND column_name = 'shard_key'${schemaName !== undefined ? ' AND table_schema = $1' : ''} LIMIT 1`, schemaName !== undefined ? [schemaName] : []);
322
+ return rows.length > 0;
323
+ }
324
+ catch {
325
+ return false;
326
+ }
327
+ finally {
328
+ await client.end().catch(() => { });
329
+ }
330
+ }
331
+ exports.holdsShardedSchema = holdsShardedSchema;
262
332
  class NotificationMap {
263
333
  map = new Map();
264
334
  curCK = 0;
@@ -300,6 +370,23 @@ function mapVersionInfo(row) {
300
370
  };
301
371
  }
302
372
  const SCHEDULE_COLUMNS = 'schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name';
373
+ /** A schedule's own columns, as a new row stores them. */
374
+ function scheduleRow(schedule, owner) {
375
+ return {
376
+ schedule_id: schedule.scheduleId,
377
+ schedule_name: schedule.scheduleName,
378
+ workflow_name: schedule.workflowName,
379
+ workflow_class_name: schedule.workflowClassName,
380
+ schedule: schedule.schedule,
381
+ status: schedule.status,
382
+ context: schedule.context,
383
+ last_fired_at: schedule.lastFiredAt,
384
+ automatic_backfill: schedule.automaticBackfill,
385
+ cron_timezone: schedule.cronTimezone,
386
+ queue_name: schedule.queueName,
387
+ application_name: owner ?? null,
388
+ };
389
+ }
303
390
  function mapWorkflowSchedule(row) {
304
391
  return {
305
392
  scheduleId: row.schedule_id,
@@ -314,6 +401,7 @@ function mapWorkflowSchedule(row) {
314
401
  cronTimezone: row.cron_timezone ?? null,
315
402
  queueName: row.queue_name ?? null,
316
403
  applicationName: row.application_name ?? undefined,
404
+ shardKey: row.workflow_shard_key ?? undefined,
317
405
  };
318
406
  }
319
407
  function mapWorkflowStatus(row) {
@@ -356,8 +444,16 @@ function mapWorkflowStatus(row) {
356
444
  debounceDeadlineEpochMS: row.debounce_deadline_epoch_ms ? Number(row.debounce_deadline_epoch_ms) : undefined,
357
445
  isDebounced: row.is_debounced ?? false,
358
446
  applicationName: row.application_name ?? undefined,
447
+ // Read back only from a sharded system database, whose rows carry them.
448
+ shardKey: row.shard_key ?? undefined,
449
+ parentShardKey: row.parent_shard_key ?? undefined,
359
450
  };
360
451
  }
452
+ /** `column`, qualified by `alias` if one is given. */
453
+ function qualify(alias, column) {
454
+ return alias ? `${alias}.${column}` : column;
455
+ }
456
+ exports.qualify = qualify;
361
457
  // SQLSTATE classes/codes that are generally safe to retry
362
458
  // https://www.postgresql.org/docs/current/errcodes-appendix.html
363
459
  const RETRY_SQLSTATE_PREFIXES = new Set([
@@ -527,6 +623,34 @@ function dbRetry(options = {}) {
527
623
  return descriptor;
528
624
  };
529
625
  }
626
+ /** Build the system database these options describe. */
627
+ function createSystemDatabase(options) {
628
+ const sysdb = createBaseSystemDatabase(options);
629
+ if (options.suspendMode) {
630
+ const { Suspend } = enterprise.load(enterprise.SUSPEND.description);
631
+ sysdb.suspend = new Suspend(sysdb);
632
+ }
633
+ return sysdb;
634
+ }
635
+ exports.createSystemDatabase = createSystemDatabase;
636
+ function createBaseSystemDatabase(options) {
637
+ if (options.sharding) {
638
+ const { ShardedSystemDatabase } = enterprise.load(enterprise.SHARDING.description);
639
+ return new ShardedSystemDatabase({
640
+ systemDatabaseUrl: options.systemDatabaseUrl,
641
+ logger: options.logger,
642
+ serializer: options.serializer,
643
+ sysDbPoolSize: options.sysDbPoolSize,
644
+ systemDatabasePool: options.systemDatabasePool,
645
+ schemaName: options.schemaName,
646
+ pollingConcurrency: options.pollingConcurrency,
647
+ appName: options.appName,
648
+ observabilityQueryTimeoutMs: options.observabilityQueryTimeoutMs,
649
+ idleTransactionTimeoutMs: options.idleTransactionTimeoutMs,
650
+ });
651
+ }
652
+ return new SystemDatabase(options.systemDatabaseUrl, options.logger, options.serializer, options.sysDbPoolSize, options.systemDatabasePool, options.schemaName, options.useListenNotify, options.pollingConcurrency, options.notificationCoalesceMs, options.appName, options.observabilityQueryTimeoutMs, options.idleTransactionTimeoutMs);
653
+ }
530
654
  /**
531
655
  * General notes:
532
656
  * The responsibilities of the `SystemDatabase` are to store data for workflows, and
@@ -570,8 +694,11 @@ class SystemDatabase {
570
694
  dbPollingIntervalStreamMs = 1000;
571
695
  dbPollingIntervalCancelMs = 1000;
572
696
  shouldUseDBNotifications = true;
697
+ // How long a background job's lock may sit idle before Postgres ends its holder's session.
698
+ backgroundJobLockIdleTimeoutMs = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS;
573
699
  notificationsMap = new NotificationMap();
574
700
  workflowEventsMap = new NotificationMap();
701
+ suspend = undefined;
575
702
  streamsMap = new NotificationMap();
576
703
  customPool = false;
577
704
  // Interval for coalescing LISTEN/NOTIFY notifications pushed off the write path (Postgres + L/N only).
@@ -595,6 +722,7 @@ class SystemDatabase {
595
722
  pollLimiter;
596
723
  /** Executions running in this process, by workflow ID; a re-dispatch runs alongside its stale execution, which parks at its next write. */
597
724
  runningWorkflowMap = new Map();
725
+ // Keyed by waiter key, as one ID can live under several shard keys.
598
726
  #cancelWatchers = new Map();
599
727
  #cancelPollerLoop = undefined;
600
728
  #cancelPollerWake = null;
@@ -727,6 +855,214 @@ class SystemDatabase {
727
855
  observabilityFilter(column, value, params) {
728
856
  return this.appNameFilter(column, value ?? this.appName, params);
729
857
  }
858
+ // ==================== Shard Keys ====================
859
+ // Statements locate rows through these, so a sharded subclass adds its key; each appends to `params` and returns SQL.
860
+ /** Whether this handle routes every statement by shard key. */
861
+ get sharded() {
862
+ return false;
863
+ }
864
+ #rejectShardKey(workflowID, shardKey) {
865
+ if ((shardKey !== undefined && shardKey !== null) || (0, shard_keys_1.boundShardKey)(workflowID) !== undefined) {
866
+ throw new error_1.DBOSError('A shard key was given, but this system database is not sharded');
867
+ }
868
+ }
869
+ /** Predicate selecting one workflow's rows: those whose `column` is `workflowID`. */
870
+ wf(alias, workflowID, params, column = 'workflow_uuid', shardKey) {
871
+ this.#rejectShardKey(workflowID, shardKey);
872
+ params.push(workflowID);
873
+ return `${qualify(alias, column)} = $${params.length}`;
874
+ }
875
+ /** Predicate selecting several workflows' rows. */
876
+ wfIn(alias, workflowIDs, params) {
877
+ for (const workflowID of workflowIDs) {
878
+ this.#rejectShardKey(workflowID, undefined);
879
+ }
880
+ params.push(workflowIDs);
881
+ return `${qualify(alias, 'workflow_uuid')} = ANY($${params.length}::text[])`;
882
+ }
883
+ /** Condition joining two tables' rows of the same workflow. */
884
+ wfJoin(left, right) {
885
+ return `${left}.workflow_uuid = ${right}.workflow_uuid`;
886
+ }
887
+ /** Extra column values for a new row of this workflow. */
888
+ wfValues(workflowID, shardKey) {
889
+ this.#rejectShardKey(workflowID, shardKey);
890
+ return {};
891
+ }
892
+ /** Extra column values storing the shard key of a new workflow's parent. */
893
+ parentValues(_parentWorkflowID) {
894
+ return {};
895
+ }
896
+ /** Extra column values storing the shard key of the workflow a step refers to. */
897
+ childValues(_childWorkflowID) {
898
+ return {};
899
+ }
900
+ /** Extra (column, expression) pairs for an INSERT ... SELECT of this workflow's rows. */
901
+ wfColumns(_workflowID, _params) {
902
+ return [];
903
+ }
904
+ /** Conflict target for a unique key of a workflow table. */
905
+ keyConflict(...columns) {
906
+ return columns.join(', ');
907
+ }
908
+ /** Insert rows into `table` in one statement, returning the RETURNING rows; a sharded subclass inserts row by row. */
909
+ async insertRows(client, table, columns, rows, suffix = '') {
910
+ if (rows.length === 0)
911
+ return [];
912
+ const params = [];
913
+ const tuples = rows.map((row) => `(${row.map((value) => `$${params.push(value)}`).join(', ')})`);
914
+ const { rows: returned } = await client.query(`INSERT INTO "${this.schemaName}".${table} (${columns.join(', ')}) VALUES ${tuples.join(', ')} ${suffix}`, params);
915
+ return returned;
916
+ }
917
+ /** Predicate a queue claim must also meet, evaluated once before it scans; sharded, it caps claimers per key. */
918
+ claimGate(_queueName, _partitionKey, _params) {
919
+ return 'TRUE';
920
+ }
921
+ /** Statement whose `acquired` says whether a claim took its budget scope's transaction lock; no row means nothing to claim. */
922
+ claimLockQuery(queueName, partitionKey, params) {
923
+ const name = JSON.stringify(['dbos.queue_claim', this.schemaName, queueName, partitionKey ?? null]);
924
+ params.push(advisoryLockKey(name).toString());
925
+ return `SELECT pg_try_advisory_xact_lock($${params.length}) AS acquired`;
926
+ }
927
+ /** Statement whose `held` lists which of `partitionKeys` a sweep's transaction locked; no row means nothing to claim. */
928
+ partitionLockQuery(queueName, partitionKeys, params) {
929
+ const name = JSON.stringify(['dbos.partition_claim', this.schemaName, queueName]);
930
+ // Two-int keys: the queue, then the partition, hashed so every worker computes the same lock.
931
+ const lockIDs = partitionKeys.map((pk) => (0, crypto_1.createHash)('sha256').update(pk).digest().readInt32BE(0));
932
+ params.push(partitionKeys, lockIDs, Number(BigInt.asIntN(32, advisoryLockKey(name))));
933
+ const n = params.length;
934
+ return `SELECT coalesce(array_agg(p.pk), '{}') AS held
935
+ FROM unnest($${n - 2}::text[], $${n - 1}::int4[]) AS p(pk, lock_id)
936
+ WHERE pg_try_advisory_xact_lock($${n}::int4, p.lock_id)`;
937
+ }
938
+ /** Predicate confining a statement that names no single workflow to the current shard. */
939
+ scoped(_alias, _params) {
940
+ return 'TRUE';
941
+ }
942
+ /** Extra (column, expression) pairs for an INSERT ... SELECT confined to the current shard. */
943
+ scopedColumns(_params) {
944
+ return [];
945
+ }
946
+ /** Predicate selecting a global table's rows (queues, schedules, versions). */
947
+ global(_alias, _params) {
948
+ return 'TRUE';
949
+ }
950
+ /** Extra column values for a new row of a global table. */
951
+ globalValues() {
952
+ return {};
953
+ }
954
+ /** Predicate selecting rows whose `column` holds one of these shard keys. */
955
+ keyIn(_alias, _shardKeys, _params, _column = 'shard_key') {
956
+ throw new error_1.DBOSError('Filtering by shard key requires a sharded system database');
957
+ }
958
+ /** A shard key column to read back with a row, if this handle has one. */
959
+ keyColumn(_alias, _column = 'shard_key') {
960
+ return undefined;
961
+ }
962
+ /** Extra column values storing the shard key a schedule's workflows run under. */
963
+ scheduleValues(schedule) {
964
+ if (schedule.shardKey !== undefined && schedule.shardKey !== null) {
965
+ throw new error_1.DBOSError('Scheduling under a shard key requires a sharded system database');
966
+ }
967
+ return {};
968
+ }
969
+ /** The column holding the shard key a schedule's workflows run under, if any. */
970
+ scheduleKeyColumn() {
971
+ return undefined;
972
+ }
973
+ /** This workflow's shard key, resolved now for a statement or waiter that needs it later. */
974
+ keyHint(workflowID) {
975
+ this.#rejectShardKey(workflowID, undefined);
976
+ return undefined;
977
+ }
978
+ /** The notification-map key for a workflow's topic or key, distinct per shard key as IDs repeat across keys. */
979
+ waiter(workflowID, name) {
980
+ const key = this.keyHint(workflowID);
981
+ const payload = `${workflowID}::${name}`;
982
+ return key === undefined ? payload : `${[...key].length}:${key}:${payload}`;
983
+ }
984
+ /** Bind a workflow to `key`, else to the running workflow's own key even where a same-ID target is bound. */
985
+ ownKey(workflowID, key, callback) {
986
+ if (key === undefined) {
987
+ const ctx = (0, context_1.getCurrentContextStore)();
988
+ if (ctx !== undefined && ctx.workflowId === workflowID) {
989
+ key = ctx.shardKey;
990
+ }
991
+ }
992
+ return (0, shard_keys_1.bindShardKeys)([[workflowID, key]], callback);
993
+ }
994
+ /** Run `callback` under this application's lock for a background job, passing undefined if refused, else a call renewing it in place. */
995
+ async backgroundJobLock(jobName, callback) {
996
+ // CockroachDB grants every advisory lock, so each process runs every job.
997
+ if (await this.#isCockroach()) {
998
+ return await callback(() => Promise.resolve());
999
+ }
1000
+ // Leading 8 bytes of SHA-256, big-endian signed, as for the retention lock.
1001
+ const key = advisoryLockKey(`dbos.background_job.${this.schemaName}.${this.appName ?? ''}.${jobName}`);
1002
+ const timeout = `'${Math.trunc(this.backgroundJobLockIdleTimeoutMs)}'`;
1003
+ return await this.lockedBackgroundJob(jobName, `SELECT pg_try_advisory_xact_lock(${key}) FROM set_config('idle_in_transaction_session_timeout', ${timeout}, true)`,
1004
+ // Snapshot-free, so a concurrent index build never waits on it; fails if the session ended.
1005
+ async (client) => {
1006
+ await client.query(`SET LOCAL idle_in_transaction_session_timeout = ${timeout}`);
1007
+ }, callback);
1008
+ }
1009
+ /** Take a transaction-scoped lock with literal `lockQuery` (its first column says if taken), as a bound statement's portal keeps its snapshot (an xmin) while the session idles. */
1010
+ async lockedBackgroundJob(jobName, lockQuery, renew, callback) {
1011
+ const client = await this.connect();
1012
+ try {
1013
+ await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1014
+ // The local idle timeout ends a frozen holder's session, releasing its lock.
1015
+ const { rows } = await client.query({ text: lockQuery, rowMode: 'array' });
1016
+ return await callback(rows[0]?.[0] === true ? () => renew(client) : undefined);
1017
+ }
1018
+ finally {
1019
+ try {
1020
+ await client.query('ROLLBACK');
1021
+ client.release();
1022
+ }
1023
+ catch (e) {
1024
+ // A session the idle timeout ended has released the lock already.
1025
+ this.logger.debug(`Could not release the ${jobName} lock: ${String(e)}`);
1026
+ client.release(e instanceof Error ? e : true);
1027
+ }
1028
+ }
1029
+ }
1030
+ #cockroach = undefined;
1031
+ /** Whether this handle's database is CockroachDB, read on first use. */
1032
+ #isCockroach() {
1033
+ this.#cockroach ??= (async () => {
1034
+ const client = await this.connect();
1035
+ try {
1036
+ return await isCockroachDB(client);
1037
+ }
1038
+ finally {
1039
+ client.release();
1040
+ }
1041
+ })();
1042
+ // A failed read is retried on next use, rather than cached.
1043
+ this.#cockroach.catch(() => {
1044
+ this.#cockroach = undefined;
1045
+ });
1046
+ return this.#cockroach;
1047
+ }
1048
+ /** Scopes a queue's workers dequeue from in turn; one unkeyed scope unless sharded. */
1049
+ dequeueScopes(_queueName) {
1050
+ return Promise.resolve([undefined]);
1051
+ }
1052
+ /** Whether a dequeue failed because another worker holds the rows it needs. */
1053
+ isLockConflict(e) {
1054
+ return isPgDatabaseError(e) && e.code === '55P03';
1055
+ }
1056
+ /** Whether a dequeue failed on a serialization conflict with a concurrent one. */
1057
+ isSerializationConflict(e) {
1058
+ return isPgDatabaseError(e) && e.code === '40001';
1059
+ }
1060
+ /** Whether `appVersion` is this application's latest registered version, or none is registered. */
1061
+ async isLatestVersion(appVersion, client) {
1062
+ // Own plus unclaimed: a named peer's deploy must not demote this one.
1063
+ const latest = await this.#latestApplicationVersionName(client ?? this.pool);
1064
+ return latest === undefined || latest === appVersion;
1065
+ }
730
1066
  /**
731
1067
  * The version name a dequeue treats as latest: this application's own plus unclaimed
732
1068
  * ones, so a named peer's deploy does not demote this one.
@@ -737,15 +1073,17 @@ class SystemDatabase {
737
1073
  async #latestApplicationVersion(client, applicationName) {
738
1074
  const columns = 'version_id, version_name, version_timestamp, created_at, application_name';
739
1075
  const table = `"${this.schemaName}".application_versions`;
1076
+ const params = applicationName ? [applicationName] : [];
1077
+ const scope = this.global(undefined, params);
740
1078
  // Probe each eligible owner separately; an OR filter can scan an unbounded history of newer peer versions.
741
1079
  // Keep the constant owner in the ordering so the IS NULL probe uses the composite index too.
742
1080
  const query = applicationName
743
- ? `(SELECT ${columns} FROM ${table} WHERE application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
1081
+ ? `(SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
744
1082
  UNION ALL
745
- (SELECT ${columns} FROM ${table} WHERE application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
1083
+ (SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
746
1084
  ORDER BY version_timestamp DESC LIMIT 1`
747
- : `SELECT ${columns} FROM ${table} ORDER BY version_timestamp DESC LIMIT 1`;
748
- const { rows } = await client.query(query, applicationName ? [applicationName] : []);
1085
+ : `SELECT ${columns} FROM ${table} WHERE ${scope} ORDER BY version_timestamp DESC LIMIT 1`;
1086
+ const { rows } = await client.query(query, params);
749
1087
  return rows[0];
750
1088
  }
751
1089
  /**
@@ -753,7 +1091,8 @@ class SystemDatabase {
753
1091
  * leaves the owner intact; a named one collides only with a different name.
754
1092
  */
755
1093
  async #resolveRowOwner(client, table, keyColumn, name, owner, kind) {
756
- const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${keyColumn} = $1`, [name]);
1094
+ const params = [name];
1095
+ const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${this.global(undefined, params)} AND ${keyColumn} = $1`, params);
757
1096
  const current = rows[0]?.application_name ?? null;
758
1097
  if (current === null)
759
1098
  return owner;
@@ -770,8 +1109,12 @@ class SystemDatabase {
770
1109
  }
771
1110
  /** Migrates the system database, or, when `runMigrations` is false, verifies it is already migrated. */
772
1111
  async init(runMigrations = true) {
773
- const migrateOrVerify = runMigrations ? ensureSystemDatabase : verifySystemDatabase;
774
- await migrateOrVerify(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
1112
+ if (runMigrations) {
1113
+ await this.runMigrations();
1114
+ }
1115
+ else {
1116
+ await this.verifyMigrations();
1117
+ }
775
1118
  if (this.shouldUseDBNotifications) {
776
1119
  await this.#listenForNotifications();
777
1120
  // Push coalesced stream and event notifications off the write path.
@@ -779,6 +1122,32 @@ class SystemDatabase {
779
1122
  this.#notifierLoop = this.#runNotifier();
780
1123
  }
781
1124
  }
1125
+ /** Create or migrate the system database. */
1126
+ async runMigrations() {
1127
+ await ensureSystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
1128
+ }
1129
+ /** Throw if this unsharded handle's schema is sharded; an unreachable database is left for its first statement. */
1130
+ async checkShardedSchema() {
1131
+ if (this.sharded)
1132
+ return;
1133
+ let client;
1134
+ try {
1135
+ client = await this.connect();
1136
+ }
1137
+ catch {
1138
+ return;
1139
+ }
1140
+ try {
1141
+ await refuseShardedSchema(client, this.schemaName);
1142
+ }
1143
+ finally {
1144
+ client.release();
1145
+ }
1146
+ }
1147
+ /** Check the system database is migrated, creating and changing nothing. */
1148
+ async verifyMigrations() {
1149
+ await verifySystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
1150
+ }
782
1151
  async destroy() {
783
1152
  // Set synchronously, before any await, so no reconnect is scheduled or published after this point.
784
1153
  this.#notificationsStopped = true;
@@ -824,7 +1193,7 @@ class SystemDatabase {
824
1193
  /** Runs on `client` if given, joining its transaction; otherwise in its own retried transaction. */
825
1194
  async initWorkflowStatus(initStatus, creatorXid, client, reusePolicy = 'return-existing') {
826
1195
  if (client !== undefined) {
827
- return await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
1196
+ return await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
828
1197
  }
829
1198
  return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
830
1199
  }
@@ -833,7 +1202,7 @@ class SystemDatabase {
833
1202
  let shouldCommit = false;
834
1203
  try {
835
1204
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
836
- const result = await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
1205
+ const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
837
1206
  // If there is an existing DB record and we aren't here to recover it, leave it be.
838
1207
  shouldCommit = result.shouldExecuteOnThisExecutor;
839
1208
  return result;
@@ -861,10 +1230,12 @@ class SystemDatabase {
861
1230
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
862
1231
  const parentOwnerXid = (0, context_1.currentOwnerXid)(parentWorkflowID);
863
1232
  if (parentOwnerXid !== undefined) {
864
- await this.#checkOwner(client, parentWorkflowID, parentOwnerXid);
1233
+ await this.checkOwner(client, parentWorkflowID, parentOwnerXid);
865
1234
  }
866
- const result = await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
867
- await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs, { childWorkflowID: initStatus.workflowUUID, ownerChecked: true });
1235
+ const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
1236
+ await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs,
1237
+ // Unkeyed, as only a sharded database records the child's key, through its own override.
1238
+ { childWorkflowID: initStatus.workflowUUID, ownerChecked: true, childValues: {} });
868
1239
  shouldCommit = true;
869
1240
  return result;
870
1241
  }
@@ -883,7 +1254,7 @@ class SystemDatabase {
883
1254
  }
884
1255
  }
885
1256
  }
886
- async #initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy) {
1257
+ async initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy) {
887
1258
  const resRow = await this.insertWorkflowStatus(client, initStatus, creatorXid);
888
1259
  if (reusePolicy === 'reject' && (creatorXid === null || resRow.creator_xid !== creatorXid)) {
889
1260
  throw new error_1.DBOSWorkflowIDInUseError(initStatus.workflowUUID, resRow.status, resRow.name);
@@ -915,6 +1286,7 @@ class SystemDatabase {
915
1286
  async deadLetterWorkflows(workflowIDs, minRecoveryAttempts) {
916
1287
  if (workflowIDs.length === 0)
917
1288
  return;
1289
+ const params = [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflow_1.StatusString.PENDING, minRecoveryAttempts];
918
1290
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
919
1291
  SET status = $1,
920
1292
  owner_xid = NULL,
@@ -923,17 +1295,18 @@ class SystemDatabase {
923
1295
  queue_name = NULL,
924
1296
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
925
1297
  completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
926
- WHERE workflow_uuid = ANY($2::text[]) AND status = $3 AND recovery_attempts >= $4`, [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflowIDs, workflow_1.StatusString.PENDING, minRecoveryAttempts]);
1298
+ WHERE ${this.wfIn(undefined, workflowIDs, params)} AND status = $2 AND recovery_attempts >= $3`, params);
927
1299
  }
928
1300
  /** Highest created_at among still-active rows per partition key, used to seed the in-memory cursor. */
929
1301
  async #maxPartitionKeyCreatedAt(keys) {
930
1302
  const maxima = new Map();
931
1303
  if (keys.length === 0)
932
1304
  return maxima;
1305
+ const params = [keys, [workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING]];
933
1306
  const { rows } = await this.pool.query(`SELECT queue_partition_key, MAX(created_at) AS max_created_at
934
1307
  FROM "${this.schemaName}".workflow_status
935
- WHERE queue_partition_key = ANY($1) AND status = ANY($2)
936
- GROUP BY queue_partition_key`, [keys, [workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING]]);
1308
+ WHERE ${this.scoped(undefined, params)} AND queue_partition_key = ANY($1) AND status = ANY($2)
1309
+ GROUP BY queue_partition_key`, params);
937
1310
  for (const row of rows) {
938
1311
  if (row.max_created_at !== null) {
939
1312
  maxima.set(row.queue_partition_key, Number(row.max_created_at));
@@ -945,7 +1318,7 @@ class SystemDatabase {
945
1318
  * Stamp created_at monotonic within each partition key so per-key order holds across batches.
946
1319
  * Unordered rows (no partition key) get wall-clock time and never touch the cursors.
947
1320
  */
948
- async #assignBatchCreatedAt(statuses) {
1321
+ async assignBatchCreatedAt(statuses) {
949
1322
  const nowMS = Date.now();
950
1323
  const batchKeys = new Set();
951
1324
  for (const status of statuses) {
@@ -990,9 +1363,8 @@ class SystemDatabase {
990
1363
  * shutdown. Callers retry this themselves.
991
1364
  */
992
1365
  async enqueueWorkflows(statuses) {
993
- const inserted = new Set();
994
1366
  if (statuses.length === 0)
995
- return inserted;
1367
+ return new Set();
996
1368
  for (const status of statuses) {
997
1369
  if (status.status !== workflow_1.StatusString.ENQUEUED) {
998
1370
  throw new error_1.DBOSError(`enqueueWorkflows only accepts ${workflow_1.StatusString.ENQUEUED} workflows, but ${status.workflowUUID} is ${status.status}`);
@@ -1001,7 +1373,63 @@ class SystemDatabase {
1001
1373
  throw new error_1.DBOSError(`enqueueWorkflows does not support deduplication IDs, but ${status.workflowUUID} has one`);
1002
1374
  }
1003
1375
  }
1004
- const createdAts = await this.#assignBatchCreatedAt(statuses);
1376
+ // Stamped before the transaction, as seeding a key's cursor reads on another connection.
1377
+ const createdAts = await this.assignBatchCreatedAt(statuses);
1378
+ const client = await this.connect();
1379
+ try {
1380
+ return await this.inTransaction(client, () => this.insertWorkflowBatch(client, statuses, createdAts));
1381
+ }
1382
+ finally {
1383
+ client.release();
1384
+ }
1385
+ }
1386
+ /** Batch-enqueue scheduled runs, skipping existing IDs, stamping last_fired_at from `firedAt` (ID to tick) for those inserted, in one transaction. */
1387
+ async enqueueScheduledWorkflows(statuses, firedAt) {
1388
+ if (statuses.length === 0)
1389
+ return new Set();
1390
+ // Stamped before the transaction, as seeding a key's cursor reads on another connection.
1391
+ const createdAts = await this.assignBatchCreatedAt(statuses);
1392
+ const client = await this.connect();
1393
+ try {
1394
+ return await this.inTransaction(client, async () => {
1395
+ const inserted = await this.insertWorkflowBatch(client, statuses, createdAts);
1396
+ await this.stampLastFiredAt(client, statuses, firedAt, inserted);
1397
+ return inserted;
1398
+ });
1399
+ }
1400
+ finally {
1401
+ client.release();
1402
+ }
1403
+ }
1404
+ /** Set each schedule's last_fired_at to its last inserted tick, `statuses` being in tick order. */
1405
+ async stampLastFiredAt(client, statuses, firedAt, inserted) {
1406
+ const latest = new Map();
1407
+ for (const status of statuses) {
1408
+ const fired = firedAt.get(status.workflowUUID);
1409
+ if (inserted.has(status.workflowUUID) && fired !== undefined && status.scheduleName !== undefined) {
1410
+ latest.set(status.scheduleName, fired);
1411
+ }
1412
+ }
1413
+ if (latest.size === 0)
1414
+ return;
1415
+ // Sorted, like applySchedules, so concurrent writers lock schedule rows in one order.
1416
+ const names = Array.from(latest.keys()).sort(compareCodeUnits);
1417
+ const lockParams = [names];
1418
+ await client.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules
1419
+ WHERE ${this.global(undefined, lockParams)} AND schedule_name = ANY($1::text[])
1420
+ ORDER BY schedule_name COLLATE "C" FOR UPDATE`, lockParams);
1421
+ const params = [names];
1422
+ const cases = names.map((name) => `WHEN schedule_name = $${params.push(name)} THEN $${params.push(latest.get(name))}`);
1423
+ await client.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = CASE ${cases.join(' ')} END
1424
+ WHERE ${this.global(undefined, params)} AND schedule_name = ANY($1::text[])`, params);
1425
+ }
1426
+ /** Insert these ENQUEUED workflows' rows on `client` with `stamped` from assignBatchCreatedAt, skipping existing IDs. Returns the IDs inserted. */
1427
+ async insertWorkflowBatch(client, batch, stamped) {
1428
+ // Inserted in one global order, so batches overlapping across processes wait on each other instead of deadlocking.
1429
+ const order = batch.map((_, i) => i).sort((a, b) => compareCodeUnits(batch[a].workflowUUID, batch[b].workflowUUID));
1430
+ const statuses = order.map((i) => batch[i]);
1431
+ const createdAts = order.map((i) => stamped[i]);
1432
+ const inserted = new Set();
1005
1433
  const columns = [
1006
1434
  'workflow_uuid',
1007
1435
  'status',
@@ -1032,75 +1460,72 @@ class SystemDatabase {
1032
1460
  'schedule_name',
1033
1461
  'application_name',
1034
1462
  ];
1035
- const client = await this.connect();
1036
- try {
1037
- await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1038
- // Chunk to stay well under the bind-parameter limit.
1039
- const chunkSize = 500;
1040
- for (let start = 0; start < statuses.length; start += chunkSize) {
1041
- const chunk = statuses.slice(start, start + chunkSize);
1042
- const tuples = [];
1043
- const params = [];
1044
- let paramIdx = 1;
1045
- for (let i = 0; i < chunk.length; i++) {
1046
- const status = chunk[i];
1047
- const createdAt = createdAts[start + i];
1048
- tuples.push(`(${columns.map(() => `$${paramIdx++}`).join(', ')})`);
1049
- params.push(status.workflowUUID, status.status, status.workflowName,
1463
+ // Chunk to stay well under the bind-parameter limit.
1464
+ const chunkSize = 500;
1465
+ for (let start = 0; start < statuses.length; start += chunkSize) {
1466
+ const chunk = statuses.slice(start, start + chunkSize);
1467
+ const keyed = chunk.map((status) => ({
1468
+ ...this.wfValues(status.workflowUUID, status.shardKey),
1469
+ ...this.parentValues(status.parentWorkflowID),
1470
+ }));
1471
+ const rows = chunk.map((status, i) => {
1472
+ const createdAt = createdAts[start + i];
1473
+ return [
1474
+ ...Object.values(keyed[i]),
1475
+ status.workflowUUID,
1476
+ status.status,
1477
+ status.workflowName,
1050
1478
  // For cross-language compatibility, these MUST be NULL in the database when not set
1051
- status.workflowClassName === '' ? null : status.workflowClassName, status.workflowConfigName === '' ? null : status.workflowConfigName, status.queueName ?? null, status.authenticatedUser, status.assumedRole, JSON.stringify(status.authenticatedRoles), status.executorId, status.applicationVersion ?? null, status.applicationID, createdAt, 0, createdAt, status.timeoutMS ?? null, status.deadlineEpochMS ?? null,
1479
+ status.workflowClassName === '' ? null : status.workflowClassName,
1480
+ status.workflowConfigName === '' ? null : status.workflowConfigName,
1481
+ status.queueName ?? null,
1482
+ status.authenticatedUser,
1483
+ status.assumedRole,
1484
+ JSON.stringify(status.authenticatedRoles),
1485
+ status.executorId,
1486
+ status.applicationVersion ?? null,
1487
+ status.applicationID,
1488
+ createdAt,
1489
+ 0,
1490
+ createdAt,
1491
+ status.timeoutMS ?? null,
1492
+ status.deadlineEpochMS ?? null,
1052
1493
  // Legacy column: the payload lives in workflow_input.
1053
- null, null, status.priority, status.queuePartitionKey ?? null, status.parentWorkflowID ?? null, status.serialization, null, status.delayUntilEpochMS ?? null, status.attributes ? JSON.stringify(status.attributes) : null, status.scheduleName ?? null, status.applicationName ?? null);
1054
- }
1055
- const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
1056
- VALUES ${tuples.join(', ')}
1057
- ON CONFLICT (workflow_uuid) DO NOTHING
1058
- RETURNING workflow_uuid`, params);
1059
- for (const row of rows) {
1060
- inserted.add(row.workflow_uuid);
1061
- }
1062
- }
1063
- for (let start = 0; start < statuses.length; start += chunkSize) {
1064
- const chunk = statuses.slice(start, start + chunkSize);
1065
- const tuples = [];
1066
- const params = [];
1067
- let paramIdx = 1;
1068
- for (let i = 0; i < chunk.length; i++) {
1069
- tuples.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
1070
- params.push(chunk[i].workflowUUID, chunk[i].input, createdAts[start + i]);
1071
- }
1072
- await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
1073
- VALUES ${tuples.join(', ')}
1074
- ON CONFLICT (workflow_uuid) DO NOTHING`, params);
1494
+ null,
1495
+ null,
1496
+ status.priority,
1497
+ status.queuePartitionKey ?? null,
1498
+ status.parentWorkflowID ?? null,
1499
+ status.serialization,
1500
+ null,
1501
+ status.delayUntilEpochMS ?? null,
1502
+ status.attributes ? JSON.stringify(status.attributes) : null,
1503
+ status.scheduleName ?? null,
1504
+ status.applicationName ?? null,
1505
+ ];
1506
+ });
1507
+ const returned = await this.insertRows(client, 'workflow_status', [...Object.keys(keyed[0]), ...columns], rows, `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING RETURNING workflow_uuid`);
1508
+ for (const row of returned) {
1509
+ inserted.add(row.workflow_uuid);
1075
1510
  }
1076
- await client.query('COMMIT');
1077
1511
  }
1078
- catch (e) {
1079
- await client.query('ROLLBACK');
1080
- throw e;
1081
- }
1082
- finally {
1083
- client.release();
1512
+ for (let start = 0; start < statuses.length; start += chunkSize) {
1513
+ const chunk = statuses.slice(start, start + chunkSize);
1514
+ const keyed = chunk.map((status) => this.wfValues(status.workflowUUID, status.shardKey));
1515
+ await this.insertRows(client, 'workflow_input', [...Object.keys(keyed[0]), 'workflow_uuid', 'inputs', 'retention_timestamp'], chunk.map((status, i) => [
1516
+ ...Object.values(keyed[i]),
1517
+ status.workflowUUID,
1518
+ status.input,
1519
+ createdAts[start + i],
1520
+ ]), `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`);
1084
1521
  }
1085
1522
  return inserted;
1086
1523
  }
1087
1524
  async recordWorkflowOutput(workflowID, status, ownerXid) {
1088
- const client = await this.connect();
1089
- try {
1090
- return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
1091
- }
1092
- finally {
1093
- client.release();
1094
- }
1525
+ return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
1095
1526
  }
1096
1527
  async recordWorkflowError(workflowID, status, ownerXid) {
1097
- const client = await this.connect();
1098
- try {
1099
- return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
1100
- }
1101
- finally {
1102
- client.release();
1103
- }
1528
+ return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
1104
1529
  }
1105
1530
  // Record a workflow's terminal outcome (SUCCESS or ERROR), reporting whether
1106
1531
  // the write landed. The write applies only to a PENDING row still owned by
@@ -1111,8 +1536,49 @@ class SystemDatabase {
1111
1536
  // terminal, handed to another execution (e.g. by a concurrent resume or
1112
1537
  // recovery), or deleted; the caller resolves which by awaiting the recorded
1113
1538
  // outcome.
1114
- async #recordWorkflowOutcome(client, workflowID, status, outcome, ownerXid) {
1539
+ async #recordWorkflowOutcome(workflowID, status, outcome, ownerXid) {
1115
1540
  ownerXid ??= (0, context_1.currentOwnerXid)(workflowID);
1541
+ if (!(await this.outcomeInOneStatement())) {
1542
+ return await this.#recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid);
1543
+ }
1544
+ // One autocommit round trip; the upsert reads the UPDATE's rows, so the output lands only with the status.
1545
+ const params = [status, workflow_1.StatusString.PENDING, outcome.output ?? null, outcome.error ?? null];
1546
+ const target = this.wf(undefined, workflowID, params);
1547
+ const owned = ownerXid !== undefined ? `AND owner_xid = $${params.push(ownerXid)}` : '';
1548
+ const keys = this.wfValues(workflowID);
1549
+ const keyColumns = Object.keys(keys);
1550
+ const keyExprs = Object.values(keys).map((value) => `$${params.push(value)}::text`);
1551
+ const workflowParam = `$${params.push(workflowID)}::text`;
1552
+ const { rows } = await this.pool.query(`WITH updated AS (
1553
+ UPDATE "${this.schemaName}".workflow_status
1554
+ SET status = $1,
1555
+ -- As the workflow is complete, remove its deduplication ID; a finished workflow has no owner.
1556
+ deduplication_id = NULL,
1557
+ owner_xid = NULL,
1558
+ updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
1559
+ completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1560
+ WHERE ${target} AND status = $2 ${owned}
1561
+ RETURNING 1
1562
+ ), stored AS (
1563
+ INSERT INTO "${this.schemaName}".workflow_output (${[...keyColumns, 'workflow_uuid', 'output', 'error'].join(', ')})
1564
+ SELECT ${[...keyExprs, workflowParam, '$3::text', '$4::text'].join(', ')}
1565
+ WHERE EXISTS (SELECT 1 FROM updated)
1566
+ -- EXCLUDED is the row this INSERT proposed, so these are the values above.
1567
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error
1568
+ -- CockroachDB refuses a data-modifying CTE without RETURNING.
1569
+ RETURNING 1
1570
+ )
1571
+ -- Selecting from the CTEs rather than inserting at the top level lets a router see the shard key.
1572
+ SELECT count(*) AS count FROM updated`, params);
1573
+ return Number(rows[0]?.count ?? 0) > 0;
1574
+ }
1575
+ /** Whether a workflow's outcome is written in one statement, rather than a transaction of two. */
1576
+ outcomeInOneStatement() {
1577
+ return Promise.resolve(true);
1578
+ }
1579
+ /** The outcome write as a transaction of two statements, for a backend that cannot run it as one. */
1580
+ async #recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid) {
1581
+ const client = await this.connect();
1116
1582
  let committed = false;
1117
1583
  try {
1118
1584
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
@@ -1126,19 +1592,31 @@ class SystemDatabase {
1126
1592
  // The outcome was not ours to write, so leave no orphan payload.
1127
1593
  return false;
1128
1594
  }
1129
- await client.query(`INSERT INTO "${this.schemaName}".workflow_output (workflow_uuid, output, error)
1130
- VALUES ($1, $2, $3)
1131
- ON CONFLICT (workflow_uuid) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error`, [workflowID, outcome.output ?? null, outcome.error ?? null]);
1595
+ const values = {
1596
+ ...this.wfValues(workflowID),
1597
+ workflow_uuid: workflowID,
1598
+ output: outcome.output ?? null,
1599
+ error: outcome.error ?? null,
1600
+ };
1601
+ const columns = Object.keys(values);
1602
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_output (${columns.join(', ')})
1603
+ VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
1604
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error`, Object.values(values));
1132
1605
  committed = true;
1133
1606
  return true;
1134
1607
  }
1135
1608
  finally {
1136
- if (committed) {
1137
- await client.query('COMMIT');
1609
+ try {
1610
+ if (committed) {
1611
+ await client.query('COMMIT');
1612
+ }
1613
+ else {
1614
+ // Swallowed: a rollback failure must not mask why we are rolling back.
1615
+ await client.query('ROLLBACK').catch(() => undefined);
1616
+ }
1138
1617
  }
1139
- else {
1140
- // Swallowed: a rollback failure must not mask why we are rolling back.
1141
- await client.query('ROLLBACK').catch(() => undefined);
1618
+ finally {
1619
+ client.release();
1142
1620
  }
1143
1621
  }
1144
1622
  }
@@ -1147,6 +1625,7 @@ class SystemDatabase {
1147
1625
  const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
1148
1626
  // executor_id defaults to "local", so it collides across applications.
1149
1627
  const scope = this.appNameFilter('application_name', this.appName, params);
1628
+ const keyColumn = this.keyColumn(undefined);
1150
1629
  const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1151
1630
  SET started_at_epoch_ms = NULL,
1152
1631
  status = $1,
@@ -1157,27 +1636,31 @@ class SystemDatabase {
1157
1636
  AND executor_id = $4
1158
1637
  AND application_version = $5
1159
1638
  AND ${scope}
1160
- RETURNING workflow_uuid`, params);
1161
- return result.rows.map((row) => row.workflow_uuid);
1639
+ RETURNING workflow_uuid${keyColumn !== undefined ? `, ${keyColumn} AS shard_key` : ''}`, params);
1640
+ return result.rows.map((row) => ({ workflowID: row.workflow_uuid, shardKey: row.shard_key ?? undefined }));
1162
1641
  }
1163
1642
  async getWorkflowStatus(workflowID, callerID, callerFN) {
1643
+ // Resolved before binding the caller's own key, which a same-ID target would share.
1644
+ const shardKey = this.keyHint(workflowID);
1164
1645
  // Inside a workflow this runs on the transaction's client: taking a second pool connection while
1165
1646
  // holding the first deadlocks the pool once every connection is held by a waiting caller.
1166
1647
  const funcGetStatus = async (client) => {
1167
- const statuses = await this.#listWorkflows({ workflowIDs: [workflowID] }, client);
1648
+ const statuses = await this.#listWorkflows({ workflowIDs: [workflowID], shardKey }, {}, client);
1168
1649
  const status = statuses.find((s) => s.workflowUUID === workflowID);
1169
1650
  return status ? JSON.stringify(status) : null;
1170
1651
  };
1171
1652
  if (callerID && callerFN !== undefined) {
1172
- const client = await this.connect();
1173
- try {
1174
- // Check if the operation has been done before for OAOO (only do this inside a workflow).
1175
- const json = await this.#inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, () => funcGetStatus(client)));
1176
- return parseStatus(json);
1177
- }
1178
- finally {
1179
- client.release();
1180
- }
1653
+ const json = await this.ownKey(callerID, undefined, async () => {
1654
+ const client = await this.connect();
1655
+ try {
1656
+ // Check if the operation has been done before for OAOO (only do this inside a workflow).
1657
+ return await this.inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, () => funcGetStatus(client)));
1658
+ }
1659
+ finally {
1660
+ client.release();
1661
+ }
1662
+ });
1663
+ return parseStatus(json);
1181
1664
  }
1182
1665
  else {
1183
1666
  const json = await funcGetStatus();
@@ -1191,7 +1674,7 @@ class SystemDatabase {
1191
1674
  statusFetchChunkSize = 500;
1192
1675
  // Retried per chunk so a reconnect refetches one chunk, not every chunk before it.
1193
1676
  async fetchWorkflowStatusChunk(workflowIDs) {
1194
- return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false });
1677
+ return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false }, { keyedIDs: true });
1195
1678
  }
1196
1679
  /** Fetch many statuses in as few round trips as possible. IDs with no row are omitted. */
1197
1680
  async getWorkflowStatuses(workflowIDs) {
@@ -1222,23 +1705,26 @@ class SystemDatabase {
1222
1705
  }
1223
1706
  }
1224
1707
  // ==================== Step Results ====================
1225
- async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
1226
- const client = await this.connect();
1227
- try {
1228
- return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
1229
- }
1230
- finally {
1231
- client.release();
1232
- }
1708
+ async getOperationResultAndThrowIfCancelled(workflowID, functionID, ownerKey) {
1709
+ return await this.ownKey(workflowID, ownerKey, async () => {
1710
+ const client = await this.connect();
1711
+ try {
1712
+ return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
1713
+ }
1714
+ finally {
1715
+ client.release();
1716
+ }
1717
+ });
1233
1718
  }
1234
1719
  async getAllOperationResults(workflowID, limit, offset, loadOutput = true) {
1235
1720
  // Skip the output/error payload columns when the caller does not need them,
1236
1721
  // mirroring how listWorkflows skips its payload joins when loadOutput is false.
1722
+ const childKey = this.keyColumn(undefined, 'child_shard_key');
1237
1723
  const columns = loadOutput
1238
1724
  ? '*'
1239
- : 'workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp';
1240
- let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE workflow_uuid=$1 ORDER BY function_id`;
1241
- const params = [workflowID];
1725
+ : `workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? `, ${childKey}` : ''}`;
1726
+ const params = [];
1727
+ let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE ${this.wf(undefined, workflowID, params)} ORDER BY function_id`;
1242
1728
  if (limit !== undefined) {
1243
1729
  params.push(limit);
1244
1730
  query += ` LIMIT $${params.length}`;
@@ -1251,14 +1737,18 @@ class SystemDatabase {
1251
1737
  return rows;
1252
1738
  }
1253
1739
  async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
1254
- const client = await this.connect();
1255
- try {
1256
- await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options));
1257
- }
1258
- finally {
1259
- client.release();
1260
- await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
1261
- }
1740
+ // Resolved before binding the caller's own key, which a same-ID child would share.
1741
+ const childValues = this.childValues(options.childWorkflowID);
1742
+ await this.ownKey(workflowID, options.ownerKey, async () => {
1743
+ const client = await this.connect();
1744
+ try {
1745
+ await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, { ...options, childValues }));
1746
+ }
1747
+ finally {
1748
+ client.release();
1749
+ await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
1750
+ }
1751
+ });
1262
1752
  }
1263
1753
  /** Checkpoint an operation's failure so a replay re-throws it instead of re-running. */
1264
1754
  async recordOperationError(workflowID, functionID, functionName, startTimeEpochMs, error, childWorkflowID) {
@@ -1332,9 +1822,10 @@ class SystemDatabase {
1332
1822
  throw new TypeError('functionID must be defined');
1333
1823
  patchName = `DBOS.patch-${patchName}`;
1334
1824
  const readCheckpointName = async (queryable) => {
1825
+ const params = [functionID];
1335
1826
  const { rows } = await queryable.query(`SELECT function_name
1336
1827
  FROM "${this.schemaName}".operation_outputs
1337
- WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
1828
+ WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
1338
1829
  return rows.length === 0 ? undefined : rows[0].function_name;
1339
1830
  };
1340
1831
  if (deprecated) {
@@ -1346,13 +1837,20 @@ class SystemDatabase {
1346
1837
  return { isPatched: true, hasEntry: checkpointName === patchName };
1347
1838
  }
1348
1839
  // Nondeprecated - skip matching entry, unpatched if nonmatching entry,
1349
- // If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
1840
+ // If the history ends before this point, we insert an entry that indicates it is patched, as an owner-checked checkpoint.
1841
+ // If the history has a checkpoint past this point but none here, the execution reached this point on pre-patch code (e.g., recv).
1842
+ // It is unpatched, and nothing is consumed.
1350
1843
  const client = await this.connect();
1351
1844
  try {
1352
- return await this.#inTransaction(client, async () => {
1353
- const checkpointName = await readCheckpointName(client);
1354
- if (checkpointName !== undefined) {
1355
- const matches = checkpointName === patchName;
1845
+ return await this.inTransaction(client, async () => {
1846
+ const params = [functionID];
1847
+ const { rows } = await client.query(`SELECT function_id, function_name
1848
+ FROM "${this.schemaName}".operation_outputs
1849
+ WHERE ${this.wf(undefined, workflowID, params)} AND function_id>=$1
1850
+ ORDER BY function_id
1851
+ LIMIT 1`, params);
1852
+ if (rows.length !== 0) {
1853
+ const matches = rows[0].function_id === functionID && rows[0].function_name === patchName;
1356
1854
  return { isPatched: matches, hasEntry: matches };
1357
1855
  }
1358
1856
  const dn = Date.now();
@@ -1375,7 +1873,7 @@ class SystemDatabase {
1375
1873
  let frontier = workflowIDs;
1376
1874
  while (frontier.length > 0) {
1377
1875
  await this.#cancelWorkflows(frontier);
1378
- const children = await this.#getDirectChildren(frontier);
1876
+ const children = await this.getDirectChildren(frontier);
1379
1877
  frontier = children.filter((id) => !visited.has(id));
1380
1878
  for (const id of frontier) {
1381
1879
  visited.add(id);
@@ -1383,41 +1881,45 @@ class SystemDatabase {
1383
1881
  }
1384
1882
  }
1385
1883
  async #cancelWorkflows(workflowIDs) {
1884
+ const params = [workflow_1.StatusString.CANCELLED, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR];
1386
1885
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1387
1886
  SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
1887
+ suspend_key = NULL,
1388
1888
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
1389
1889
  completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1390
- WHERE workflow_uuid = ANY($2)
1391
- AND status NOT IN ($3, $4)`, [workflow_1.StatusString.CANCELLED, workflowIDs, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR]);
1890
+ WHERE ${this.wfIn(undefined, workflowIDs, params)}
1891
+ AND status NOT IN ($2, $3)`, params);
1392
1892
  }
1393
1893
  async checkIfCanceled(workflowID) {
1394
- await this.#checkIfCanceled(this.pool, workflowID);
1894
+ // The caller's own workflow, even where a same-ID workflow is bound as a target.
1895
+ await this.ownKey(workflowID, undefined, () => this.#checkIfCanceled(this.pool, workflowID));
1395
1896
  }
1396
1897
  async resumeWorkflows(workflowIDs, queueName) {
1397
1898
  const client = await this.connect();
1398
1899
  try {
1399
1900
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1400
1901
  // Check existence separately: a zero-row update also means "already complete", a legal no-op.
1401
- const { rows } = await client.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [workflowIDs]);
1902
+ const existingParams = [];
1903
+ const { rows } = await client.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${this.wfIn(undefined, workflowIDs, existingParams)}`, existingParams);
1402
1904
  const existing = new Set(rows.map((r) => r.workflow_uuid));
1403
1905
  const missing = workflowIDs.filter((id) => !existing.has(id));
1404
1906
  if (missing.length > 0) {
1405
1907
  throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${missing.join(', ')} does not exist`);
1406
1908
  }
1909
+ const updateParams = [
1910
+ workflow_1.StatusString.ENQUEUED,
1911
+ queueName ?? utils_1.INTERNAL_QUEUE_NAME,
1912
+ workflow_1.StatusString.SUCCESS,
1913
+ workflow_1.StatusString.ERROR,
1914
+ ];
1407
1915
  await client.query(`UPDATE "${this.schemaName}".workflow_status
1408
1916
  SET status = $1, owner_xid = NULL, queue_name = $2, recovery_attempts = 0,
1409
1917
  workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
1410
- started_at_epoch_ms = NULL,
1918
+ started_at_epoch_ms = NULL, suspend_key = NULL,
1411
1919
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
1412
1920
  completed_at = NULL
1413
- WHERE workflow_uuid = ANY($3)
1414
- AND status NOT IN ($4, $5)`, [
1415
- workflow_1.StatusString.ENQUEUED,
1416
- queueName ?? utils_1.INTERNAL_QUEUE_NAME,
1417
- workflowIDs,
1418
- workflow_1.StatusString.SUCCESS,
1419
- workflow_1.StatusString.ERROR,
1420
- ]);
1921
+ WHERE ${this.wfIn(undefined, workflowIDs, updateParams)}
1922
+ AND status NOT IN ($3, $4)`, updateParams);
1421
1923
  await client.query('COMMIT');
1422
1924
  }
1423
1925
  catch (e) {
@@ -1430,16 +1932,18 @@ class SystemDatabase {
1430
1932
  }
1431
1933
  }
1432
1934
  async setWorkflowPriority(workflowID, priority) {
1935
+ const params = [priority, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED];
1433
1936
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1434
1937
  SET priority = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1435
- WHERE workflow_uuid = $2
1436
- AND status IN ($3, $4)`, [priority, workflowID, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED]);
1938
+ WHERE ${this.wf(undefined, workflowID, params)}
1939
+ AND status IN ($2, $3)`, params);
1437
1940
  }
1438
1941
  async setWorkflowDelay(workflowID, delayUntilEpochMS) {
1942
+ const params = [delayUntilEpochMS, workflow_1.StatusString.DELAYED];
1439
1943
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1440
1944
  SET delay_until_epoch_ms = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1441
- WHERE workflow_uuid = $2
1442
- AND status = $3`, [delayUntilEpochMS, workflowID, workflow_1.StatusString.DELAYED]);
1945
+ WHERE ${this.wf(undefined, workflowID, params)}
1946
+ AND status = $2`, params);
1443
1947
  }
1444
1948
  /**
1445
1949
  * Extend an existing debounced DELAYED workflow's delay and update its inputs, atomically.
@@ -1505,7 +2009,7 @@ class SystemDatabase {
1505
2009
  if (updated.rows.length > 0) {
1506
2010
  await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
1507
2011
  VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
1508
- ON CONFLICT (workflow_uuid) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
2012
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
1509
2013
  return {
1510
2014
  bouncedWorkflowID: updated.rows[0].workflow_uuid,
1511
2015
  holderWorkflowID: null,
@@ -1515,10 +2019,11 @@ class SystemDatabase {
1515
2019
  holderApplicationName: null,
1516
2020
  };
1517
2021
  }
1518
- // Unscoped, so a holder that blocked the update above is reportable.
2022
+ // No application filter, so a holder that blocked the update above is reportable.
2023
+ const holderParams = [params.queueName, params.deduplicationID];
1519
2024
  const holder = await client.query(`SELECT workflow_uuid, is_debounced, name, class_name, application_name
1520
2025
  FROM "${this.schemaName}".workflow_status
1521
- WHERE queue_name = $1 AND deduplication_id = $2`, [params.queueName, params.deduplicationID]);
2026
+ WHERE ${this.scoped(undefined, holderParams)} AND queue_name = $1 AND deduplication_id = $2`, holderParams);
1522
2027
  if (holder.rows.length === 0) {
1523
2028
  return {
1524
2029
  bouncedWorkflowID: null,
@@ -1539,7 +2044,7 @@ class SystemDatabase {
1539
2044
  };
1540
2045
  }
1541
2046
  // Get the immediate (one-level) child workflow IDs for a set of workflows.
1542
- async #getDirectChildren(workflowIDs) {
2047
+ async getDirectChildren(workflowIDs) {
1543
2048
  if (workflowIDs.length === 0) {
1544
2049
  return [];
1545
2050
  }
@@ -1553,7 +2058,7 @@ class SystemDatabase {
1553
2058
  const descendants = new Set();
1554
2059
  let frontier = [workflowID];
1555
2060
  while (frontier.length > 0) {
1556
- const children = await this.#getDirectChildren(frontier);
2061
+ const children = await this.getDirectChildren(frontier);
1557
2062
  frontier = children.filter((id) => !descendants.has(id));
1558
2063
  for (const id of frontier) {
1559
2064
  descendants.add(id);
@@ -1561,6 +2066,7 @@ class SystemDatabase {
1561
2066
  }
1562
2067
  return [...descendants];
1563
2068
  }
2069
+ /** Delete workflows and all associated data, and with deleteChildren every workflow they started, recursively. */
1564
2070
  async deleteWorkflows(workflowIDs, deleteChildren = false) {
1565
2071
  const allIds = [...workflowIDs];
1566
2072
  if (deleteChildren) {
@@ -1568,15 +2074,18 @@ class SystemDatabase {
1568
2074
  allIds.push(...(await this.getWorkflowChildren(wfid)));
1569
2075
  }
1570
2076
  }
2077
+ await this.deleteWorkflowsInternal(allIds);
2078
+ }
2079
+ async deleteWorkflowsInternal(allIds) {
1571
2080
  const client = await this.connect();
1572
2081
  try {
1573
2082
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1574
2083
  // The payload tables carry no foreign key, so the status delete does not cascade
1575
2084
  // into them; without this they survive as orphans.
1576
- for (const table of ['workflow_input', 'workflow_output', 'operation_outputs']) {
1577
- await client.query(`DELETE FROM "${this.schemaName}".${table} WHERE workflow_uuid = ANY($1)`, [allIds]);
2085
+ for (const table of ['workflow_input', 'workflow_output', 'operation_outputs', 'workflow_status']) {
2086
+ const params = [];
2087
+ await client.query(`DELETE FROM "${this.schemaName}".${table} WHERE ${this.wfIn(undefined, allIds, params)}`, params);
1578
2088
  }
1579
- await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [allIds]);
1580
2089
  await client.query('COMMIT');
1581
2090
  }
1582
2091
  catch (e) {
@@ -1617,67 +2126,73 @@ class SystemDatabase {
1617
2126
  const client = await this.connect();
1618
2127
  try {
1619
2128
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1620
- const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE workflow_uuid = $1`, [workflowID]);
2129
+ const statusParams = [];
2130
+ const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE ${this.wf(undefined, workflowID, statusParams)}`, statusParams);
1621
2131
  if (statusRows.length === 0) {
1622
2132
  throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
1623
2133
  }
1624
2134
  const status = statusRows[0].status;
1625
- if (status === workflow_1.StatusString.PENDING || status === workflow_1.StatusString.ENQUEUED || status === workflow_1.StatusString.DELAYED) {
2135
+ if ((0, workflow_1.isWorkflowActive)(status)) {
1626
2136
  throw new error_1.DBOSError(`Cannot rewind ${workflowID} (${status}): only a workflow in a terminal state can be rewound, so cancel it first`);
1627
2137
  }
1628
2138
  // Roll workflow_events back to the last value published before the cut, using
1629
2139
  // workflow_events_history as an undo log, then rewind that history too.
1630
2140
  // `publishedPastCut` is the correlated EXISTS both halves key off.
1631
- const publishedPastCut = `EXISTS (
2141
+ const publishedPastCut = (key, params) => `EXISTS (
1632
2142
  SELECT 1 FROM "${schema}".workflow_events_history discarded
1633
- WHERE discarded.workflow_uuid = $1 AND discarded.key = %KEY% AND discarded.function_id >= $2
2143
+ WHERE ${this.wf('discarded', workflowID, params)} AND discarded.key = ${key} AND discarded.function_id >= $1
1634
2144
  )`;
1635
2145
  // First unpublish everything the discarded run touched.
2146
+ const unpublishParams = [startStep];
1636
2147
  await client.query(`DELETE FROM "${schema}".workflow_events
1637
- WHERE workflow_uuid = $1 AND ${publishedPastCut.replace('%KEY%', '"' + schema + '".workflow_events.key')}`, [workflowID, startStep]);
2148
+ WHERE ${this.wf(undefined, workflowID, unpublishParams)}
2149
+ AND ${publishedPastCut('"' + schema + '".workflow_events.key', unpublishParams)}`, unpublishParams);
1638
2150
  // Then restore those keys, if any, to the value published below the cut.
1639
- await client.query(`INSERT INTO "${schema}".workflow_events (workflow_uuid, key, value, serialization)
1640
- SELECT surviving.workflow_uuid, surviving.key, surviving.value, surviving.serialization
2151
+ const restoreParams = [startStep];
2152
+ const keyColumns = this.wfColumns(workflowID, restoreParams);
2153
+ await client.query(`INSERT INTO "${schema}".workflow_events (${[...keyColumns.map(([c]) => c), 'workflow_uuid', 'key', 'value', 'serialization'].join(', ')})
2154
+ SELECT ${[...keyColumns.map(([, e]) => e), 'surviving.workflow_uuid', 'surviving.key', 'surviving.value', 'surviving.serialization'].join(', ')}
1641
2155
  FROM (
1642
2156
  SELECT weh.workflow_uuid, weh.key, weh.value, weh.serialization,
1643
2157
  ROW_NUMBER() OVER (PARTITION BY weh.key ORDER BY weh.function_id DESC) AS rn
1644
2158
  FROM "${schema}".workflow_events_history weh
1645
- WHERE weh.workflow_uuid = $1 AND weh.function_id < $2
1646
- AND ${publishedPastCut.replace('%KEY%', 'weh.key')}
2159
+ WHERE ${this.wf('weh', workflowID, restoreParams)} AND weh.function_id < $1
2160
+ AND ${publishedPastCut('weh.key', restoreParams)}
1647
2161
  ) surviving
1648
- WHERE surviving.rn = 1`, [workflowID, startStep]);
2162
+ WHERE surviving.rn = 1`, restoreParams);
1649
2163
  // Clear the streams' close sentinels so the replay can append again.
2164
+ const sentinelParams = [startStep, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name()];
1650
2165
  await client.query(`DELETE FROM "${schema}".streams
1651
- WHERE workflow_uuid = $1 AND function_id >= $2 AND value = $3 AND serialization = $4`, [workflowID, startStep, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name()]);
2166
+ WHERE ${this.wf(undefined, workflowID, sentinelParams)} AND function_id >= $1 AND value = $2 AND serialization = $3`, sentinelParams);
1652
2167
  // Discard the steps and the events history past the cut.
1653
2168
  for (const table of ['operation_outputs', 'workflow_events_history']) {
1654
- await client.query(`DELETE FROM "${schema}".${table} WHERE workflow_uuid = $1 AND function_id >= $2`, [
1655
- workflowID,
1656
- startStep,
1657
- ]);
2169
+ const params = [startStep];
2170
+ await client.query(`DELETE FROM "${schema}".${table} WHERE ${this.wf(undefined, workflowID, params)} AND function_id >= $1`, params);
1658
2171
  }
1659
2172
  // Delete messages consumed or received after the rewind point
1660
- await client.query(`DELETE FROM "${schema}".notifications WHERE destination_uuid = $1 AND (consumed_by_function_id >= $2 OR consumed = false)`, [workflowID, startStep]);
1661
- await client.query(`DELETE FROM "${schema}".workflow_output WHERE workflow_uuid = $1`, [workflowID]);
2173
+ const messageParams = [startStep];
2174
+ await client.query(`DELETE FROM "${schema}".notifications
2175
+ WHERE ${this.wf(undefined, workflowID, messageParams, 'destination_uuid')}
2176
+ AND (consumed_by_function_id >= $1 OR consumed = false)`, messageParams);
2177
+ const outputParams = [];
2178
+ await client.query(`DELETE FROM "${schema}".workflow_output WHERE ${this.wf(undefined, workflowID, outputParams)}`, outputParams);
1662
2179
  // Re-enqueue the workflow. Re-asserting the status we read keeps a workflow that
1663
2180
  // moved on underneath us from being resurrected.
1664
- const setVersion = options.applicationVersion !== undefined ? ', application_version = $6' : '';
1665
2181
  const params = [
1666
2182
  workflow_1.StatusString.ENQUEUED,
1667
2183
  options.queueName ?? utils_1.INTERNAL_QUEUE_NAME,
1668
2184
  options.queuePartitionKey ?? null,
1669
- workflowID,
1670
2185
  status,
1671
2186
  ];
1672
- if (options.applicationVersion !== undefined) {
1673
- params.push(options.applicationVersion);
1674
- }
2187
+ const setVersion = options.applicationVersion !== undefined
2188
+ ? `, application_version = $${params.push(options.applicationVersion)}`
2189
+ : '';
1675
2190
  const { rowCount } = await client.query(`UPDATE "${schema}".workflow_status
1676
2191
  SET status = $1, owner_xid = NULL, queue_name = $2, queue_partition_key = $3, recovery_attempts = 0,
1677
2192
  workflow_deadline_epoch_ms = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
1678
- completed_at = NULL, output = NULL, error = NULL,
2193
+ suspend_key = NULL, completed_at = NULL, output = NULL, error = NULL,
1679
2194
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint${setVersion}
1680
- WHERE workflow_uuid = $4 AND status = $5`, params);
2195
+ WHERE ${this.wf(undefined, workflowID, params)} AND status = $4`, params);
1681
2196
  if (rowCount !== 1) {
1682
2197
  throw new error_1.DBOSError(`Workflow ${workflowID} changed status while being rewound; retry the rewind`);
1683
2198
  }
@@ -1707,13 +2222,14 @@ class SystemDatabase {
1707
2222
  try {
1708
2223
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1709
2224
  // Fetch the status of all original workflows inside the transaction.
2225
+ const statusParams = [];
1710
2226
  const { rows: statusRows } = await client.query(`SELECT ws.workflow_uuid, ws.name, ws.class_name, ws.config_name, ws.application_id,
1711
2227
  ws.authenticated_user, ws.authenticated_roles, ws.assumed_role,
1712
2228
  COALESCE(wi.inputs, ws.inputs) AS inputs, ws.serialization,
1713
2229
  ws.application_version, ws.attributes, ws.application_name
1714
2230
  FROM "${this.schemaName}".workflow_status ws
1715
- LEFT JOIN "${this.schemaName}".workflow_input wi ON wi.workflow_uuid = ws.workflow_uuid
1716
- WHERE ws.workflow_uuid = ANY($1)`, [originalWorkflowIDs]);
2231
+ LEFT JOIN "${this.schemaName}".workflow_input wi ON ${this.wfJoin('ws', 'wi')}
2232
+ WHERE ${this.wfIn('ws', originalWorkflowIDs, statusParams)}`, statusParams);
1717
2233
  const statusByID = new Map(statusRows.map((r) => [r.workflow_uuid, r]));
1718
2234
  for (const wid of originalWorkflowIDs) {
1719
2235
  if (!statusByID.has(wid)) {
@@ -1749,35 +2265,42 @@ class SystemDatabase {
1749
2265
  forkedWorkflowIDs[i],
1750
2266
  statusByID.get(origID).application_name ?? this.appName ?? null,
1751
2267
  ]));
1752
- const valuesPlaceholders = [];
1753
- const params = [];
1754
- let paramIdx = 1;
1755
- for (let i = 0; i < originalWorkflowIDs.length; i++) {
1756
- const origID = originalWorkflowIDs[i];
2268
+ const forkKeys = forkedWorkflowIDs.map((forkID) => this.wfValues(forkID));
2269
+ await this.insertRows(client, 'workflow_status', [...Object.keys(forkKeys[0]), ...insertCols], originalWorkflowIDs.map((origID, i) => {
1757
2270
  const forkID = forkedWorkflowIDs[i];
1758
2271
  const ws = statusByID.get(origID);
1759
- const placeholders = insertCols.map(() => `$${paramIdx++}`).join(', ');
1760
- valuesPlaceholders.push(`(${placeholders})`);
1761
- params.push(forkID, workflow_1.StatusString.ENQUEUED, ws.name, ws.class_name ?? null, ws.config_name ?? null, queueName, ws.authenticated_user, ws.assumed_role, ws.authenticated_roles, options.applicationVersion ?? ws.application_version ?? null, ws.application_id,
1762
- // Legacy column: the payload lives in workflow_input.
1763
- null, options.queuePartitionKey ?? null, origID, ws.serialization, ws.attributes ? JSON.stringify(ws.attributes) : null, forkOwners.get(forkID) ?? null);
1764
- if (options.timeoutMS !== undefined) {
1765
- params.push(options.timeoutMS);
1766
- }
1767
- }
1768
- await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${insertCols.join(', ')})
1769
- VALUES ${valuesPlaceholders.join(', ')}`, params);
1770
- const inputPlaceholders = [];
1771
- const inputParams = [];
1772
- let inputIdx = 1;
1773
- for (let i = 0; i < originalWorkflowIDs.length; i++) {
1774
- inputPlaceholders.push(`($${inputIdx++}, $${inputIdx++})`);
1775
- inputParams.push(forkedWorkflowIDs[i], statusByID.get(originalWorkflowIDs[i]).inputs);
1776
- }
1777
- await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs)
1778
- VALUES ${inputPlaceholders.join(', ')}`, inputParams);
2272
+ return [
2273
+ ...Object.values(forkKeys[i]),
2274
+ forkID,
2275
+ workflow_1.StatusString.ENQUEUED,
2276
+ ws.name,
2277
+ ws.class_name ?? null,
2278
+ ws.config_name ?? null,
2279
+ queueName,
2280
+ ws.authenticated_user,
2281
+ ws.assumed_role,
2282
+ ws.authenticated_roles,
2283
+ options.applicationVersion ?? ws.application_version ?? null,
2284
+ ws.application_id,
2285
+ // Legacy column: the payload lives in workflow_input.
2286
+ null,
2287
+ options.queuePartitionKey ?? null,
2288
+ origID,
2289
+ ws.serialization,
2290
+ ws.attributes ? JSON.stringify(ws.attributes) : null,
2291
+ forkOwners.get(forkID) ?? null,
2292
+ ...(options.timeoutMS !== undefined ? [options.timeoutMS] : []),
2293
+ ];
2294
+ }));
2295
+ await this.insertRows(client, 'workflow_input', [...Object.keys(forkKeys[0]), 'workflow_uuid', 'inputs'], originalWorkflowIDs.map((origID, i) => [
2296
+ ...Object.values(forkKeys[i]),
2297
+ forkedWorkflowIDs[i],
2298
+ statusByID.get(origID).inputs,
2299
+ ]));
1779
2300
  // Mark all original workflows as having been forked from.
1780
- await client.query(`UPDATE "${this.schemaName}".workflow_status SET was_forked_from = TRUE WHERE workflow_uuid = ANY($1)`, [originalWorkflowIDs]);
2301
+ const forkedFromParams = [];
2302
+ await client.query(`UPDATE "${this.schemaName}".workflow_status SET was_forked_from = TRUE
2303
+ WHERE ${this.wfIn(undefined, originalWorkflowIDs, forkedFromParams)}`, forkedFromParams);
1781
2304
  // For workflows with start_step > 0, copy checkpoints/events/streams.
1782
2305
  // Build a mapping CTE so each copy is a single SQL statement.
1783
2306
  const forkMappings = originalWorkflowIDs
@@ -1794,55 +2317,85 @@ class SystemDatabase {
1794
2317
  mIdx += 4;
1795
2318
  }
1796
2319
  const mappingCTE = `WITH mapping(orig_id, fork_id, start_step, owner) AS (VALUES ${mappingValues.join(', ')})`;
2320
+ // The forks live under their originals' shard key, read from the scope; a copy's params extend the mapping's.
2321
+ const copyParams = () => [...mappingParams];
2322
+ const keyed = (params) => {
2323
+ const columns = this.scopedColumns(params);
2324
+ return { names: columns.map(([c]) => `${c}, `).join(''), exprs: columns.map(([, e]) => `${e}, `).join('') };
2325
+ };
2326
+ // A replacement child is a fork, so it shares the replaced child's key.
2327
+ const childKey = this.keyColumn('oo', 'child_shard_key');
2328
+ if (options.replacementChildren && childKey !== undefined) {
2329
+ // A replacement names a child by ID, which can't pick one of several keys.
2330
+ const ambiguityParams = copyParams();
2331
+ const replaced = `$${ambiguityParams.push(Object.keys(options.replacementChildren))}::text[]`;
2332
+ const { rows: ambiguous } = await client.query(`${mappingCTE}
2333
+ SELECT oo.child_workflow_id
2334
+ FROM mapping m
2335
+ JOIN "${this.schemaName}".operation_outputs oo
2336
+ ON ${this.scoped('oo', ambiguityParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step
2337
+ WHERE oo.child_workflow_id = ANY(${replaced})
2338
+ GROUP BY oo.child_workflow_id
2339
+ HAVING COUNT(DISTINCT ${childKey}) > 1
2340
+ LIMIT 1`, ambiguityParams);
2341
+ if (ambiguous.length > 0) {
2342
+ throw new error_1.DBOSError(`Can't replace child workflow ${ambiguous[0].child_workflow_id}: the forked steps refer to children with that ID under more than one shard key`);
2343
+ }
2344
+ }
1797
2345
  // Build the child_workflow_id expression, applying replacements if provided.
2346
+ const ooParams = copyParams();
1798
2347
  let childWfExpr = 'oo.child_workflow_id';
1799
- const ooParams = [...mappingParams];
1800
2348
  if (options.replacementChildren && Object.keys(options.replacementChildren).length > 0) {
1801
2349
  const whenClauses = [];
1802
2350
  for (const [oldId, newId] of Object.entries(options.replacementChildren)) {
1803
- whenClauses.push(`WHEN oo.child_workflow_id = $${mIdx} THEN $${mIdx + 1}::text`);
1804
- ooParams.push(oldId, newId);
1805
- mIdx += 2;
2351
+ whenClauses.push(`WHEN oo.child_workflow_id = $${ooParams.push(oldId)} THEN $${ooParams.push(newId)}::text`);
1806
2352
  }
1807
2353
  childWfExpr = `CASE ${whenClauses.join(' ')} ELSE oo.child_workflow_id END`;
1808
2354
  }
2355
+ const ooKeys = keyed(ooParams);
1809
2356
  // Copy operation outputs
1810
2357
  await client.query(`${mappingCTE}
1811
2358
  INSERT INTO "${this.schemaName}".operation_outputs
1812
- (workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp)
1813
- SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint
2359
+ (${ooKeys.names}workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? ', child_shard_key' : ''})
2360
+ SELECT ${ooKeys.exprs}m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint${childKey !== undefined ? `, ${childKey}` : ''}
1814
2361
  FROM mapping m
1815
2362
  JOIN "${this.schemaName}".operation_outputs oo
1816
- ON oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
2363
+ ON ${this.scoped('oo', ooParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
1817
2364
  // Copy streams
2365
+ const streamParams = copyParams();
2366
+ const streamKeys = keyed(streamParams);
1818
2367
  await client.query(`${mappingCTE}
1819
2368
  INSERT INTO "${this.schemaName}".streams
1820
- (workflow_uuid, key, value, serialization, "offset", function_id)
1821
- SELECT m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
2369
+ (${streamKeys.names}workflow_uuid, key, value, serialization, "offset", function_id)
2370
+ SELECT ${streamKeys.exprs}m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
1822
2371
  FROM mapping m
1823
2372
  JOIN "${this.schemaName}".streams s
1824
- ON s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`, mappingParams);
2373
+ ON ${this.scoped('s', streamParams)} AND s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`, streamParams);
1825
2374
  // Copy events history
2375
+ const historyParams = copyParams();
2376
+ const historyKeys = keyed(historyParams);
1826
2377
  await client.query(`${mappingCTE}
1827
2378
  INSERT INTO "${this.schemaName}".workflow_events_history
1828
- (workflow_uuid, function_id, key, value, serialization)
1829
- SELECT m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
2379
+ (${historyKeys.names}workflow_uuid, function_id, key, value, serialization)
2380
+ SELECT ${historyKeys.exprs}m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
1830
2381
  FROM mapping m
1831
2382
  JOIN "${this.schemaName}".workflow_events_history weh
1832
- ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`, mappingParams);
2383
+ ON ${this.scoped('weh', historyParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`, historyParams);
1833
2384
  // Copy only the latest version of each event using a window function
2385
+ const eventParams = copyParams();
2386
+ const eventKeys = keyed(eventParams);
1834
2387
  await client.query(`${mappingCTE}
1835
2388
  INSERT INTO "${this.schemaName}".workflow_events
1836
- (workflow_uuid, key, value, serialization)
1837
- SELECT ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
2389
+ (${eventKeys.names}workflow_uuid, key, value, serialization)
2390
+ SELECT ${eventKeys.exprs}ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
1838
2391
  FROM (
1839
2392
  SELECT m.fork_id AS workflow_uuid, weh.key, weh.value, weh.serialization,
1840
- ROW_NUMBER() OVER (PARTITION BY weh.workflow_uuid, weh.key ORDER BY weh.function_id DESC) AS rn
2393
+ ROW_NUMBER() OVER (PARTITION BY m.fork_id, weh.key ORDER BY weh.function_id DESC) AS rn
1841
2394
  FROM mapping m
1842
2395
  JOIN "${this.schemaName}".workflow_events_history weh
1843
- ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
2396
+ ON ${this.scoped('weh', eventParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
1844
2397
  ) ranked
1845
- WHERE ranked.rn = 1`, mappingParams);
2398
+ WHERE ranked.rn = 1`, eventParams);
1846
2399
  }
1847
2400
  await client.query('COMMIT');
1848
2401
  return forkedWorkflowIDs;
@@ -1965,27 +2518,36 @@ class SystemDatabase {
1965
2518
  }
1966
2519
  /** Cancellation check for polling waits: goes through the limiter so readers cannot starve the pool. */
1967
2520
  async checkIfCanceledLimited(workflowID) {
1968
- await this.#pollWithLimiter(() => this.#checkIfCanceled(this.pool, workflowID));
2521
+ // The caller's own workflow, even where a same-ID workflow is bound as a target.
2522
+ const params = [];
2523
+ const target = this.ownKey(workflowID, undefined, () => this.wf(undefined, workflowID, params));
2524
+ const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${target}`, params));
2525
+ if (rows[0]?.status === workflow_1.StatusString.CANCELLED) {
2526
+ throw new error_1.DBOSWorkflowCancelledError(workflowID);
2527
+ }
1969
2528
  }
1970
2529
  watchForCancellation(workflowID) {
1971
2530
  const controller = new AbortController();
1972
2531
  let state = 'idle';
2532
+ // The running workflow's own key, resolved now for the poller that reads it later outside this context.
2533
+ const shardKey = this.ownKey(workflowID, undefined, () => this.keyHint(workflowID));
2534
+ const waiterKey = this.ownKey(workflowID, undefined, () => this.waiter(workflowID, ''));
1973
2535
  const start = () => {
1974
2536
  state = 'watching';
1975
2537
  if (this.#destroyed)
1976
2538
  return;
1977
- let watchers = this.#cancelWatchers.get(workflowID);
2539
+ let watchers = this.#cancelWatchers.get(waiterKey);
1978
2540
  if (watchers === undefined) {
1979
- watchers = new Set();
1980
- this.#cancelWatchers.set(workflowID, watchers);
2541
+ watchers = { workflowID, shardKey, controllers: new Set() };
2542
+ this.#cancelWatchers.set(waiterKey, watchers);
1981
2543
  }
1982
- watchers.add(controller);
1983
- this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => this.#runCancelPoller());
2544
+ watchers.controllers.add(controller);
2545
+ this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => (0, shard_keys_1.withClearedShardKeyState)(() => this.#runCancelPoller()));
1984
2546
  };
1985
2547
  const stop = () => {
1986
- const watchers = this.#cancelWatchers.get(workflowID);
1987
- if (watchers?.delete(controller) && watchers.size === 0) {
1988
- this.#cancelWatchers.delete(workflowID);
2548
+ const watchers = this.#cancelWatchers.get(waiterKey);
2549
+ if (watchers?.controllers.delete(controller) && watchers.controllers.size === 0) {
2550
+ this.#cancelWatchers.delete(waiterKey);
1989
2551
  if (this.#cancelWatchers.size === 0)
1990
2552
  this.#cancelPollerWake?.();
1991
2553
  }
@@ -2024,15 +2586,36 @@ class SystemDatabase {
2024
2586
  this.#cancelPollerLoop = undefined;
2025
2587
  }
2026
2588
  async #abortCancelledWatchers() {
2027
- const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1) AND status = $2`, [Array.from(this.#cancelWatchers.keys()), workflow_1.StatusString.CANCELLED]));
2028
- for (const { workflow_uuid: workflowID } of rows) {
2029
- const watchers = this.#cancelWatchers.get(workflowID);
2030
- if (watchers === undefined)
2589
+ // One read per shard key, each bound to its own workflows.
2590
+ const byKey = new Map();
2591
+ for (const { workflowID, shardKey } of this.#cancelWatchers.values()) {
2592
+ const ids = byKey.get(shardKey) ?? [];
2593
+ ids.push(workflowID);
2594
+ byKey.set(shardKey, ids);
2595
+ }
2596
+ for (const [shardKey, workflowIDs] of byKey) {
2597
+ // Each key's read on its own, so one key's failure doesn't hide another key's cancellations.
2598
+ let rows;
2599
+ try {
2600
+ ({ rows } = await (0, shard_keys_1.bindShardKeys)(workflowIDs.map((id) => [id, shardKey]), () => {
2601
+ const params = [workflow_1.StatusString.CANCELLED];
2602
+ const ids = this.wfIn(undefined, workflowIDs, params);
2603
+ return this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${ids} AND status = $1`, params));
2604
+ }));
2605
+ }
2606
+ catch (e) {
2607
+ this.logger.warn(`Error polling status for cancellable steps: ${String(e)}`);
2031
2608
  continue;
2032
- this.#cancelWatchers.delete(workflowID);
2033
- const reason = new error_1.DBOSWorkflowCancelledError(workflowID);
2034
- for (const controller of watchers)
2035
- controller.abort(reason);
2609
+ }
2610
+ const cancelled = new Set(rows.map((row) => row.workflow_uuid));
2611
+ for (const [waiterKey, watchers] of this.#cancelWatchers) {
2612
+ if (watchers.shardKey !== shardKey || !cancelled.has(watchers.workflowID))
2613
+ continue;
2614
+ this.#cancelWatchers.delete(waiterKey);
2615
+ const reason = new error_1.DBOSWorkflowCancelledError(watchers.workflowID);
2616
+ for (const controller of watchers.controllers)
2617
+ controller.abort(reason);
2618
+ }
2036
2619
  }
2037
2620
  }
2038
2621
  // A missing row normally means the workflow has not been inserted yet, so
@@ -2044,21 +2627,23 @@ class SystemDatabase {
2044
2627
  const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
2045
2628
  let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
2046
2629
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
2047
- // Record the durable timeout deadline once before polling. #durableSleep persists it on the
2630
+ // Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
2048
2631
  // first call and reads back the same value on recovery, so it never changes across iterations.
2049
2632
  if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
2050
- finishTime = await this.#durableSleep(callerID, timerFuncID, timeoutms);
2633
+ finishTime = await this.durableSleepInternal(callerID, timerFuncID, timeoutms);
2051
2634
  }
2052
2635
  while (true) {
2053
2636
  if (callerID)
2054
2637
  await this.checkIfCanceledLimited(callerID);
2055
2638
  let rows;
2056
2639
  try {
2640
+ const params = [];
2641
+ const target = this.wf('ws', workflowID, params);
2057
2642
  ({ rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT ws.status, COALESCE(wo.output, ws.output) AS output,
2058
2643
  COALESCE(wo.error, ws.error) AS error, ws.serialization
2059
2644
  FROM "${this.schemaName}".workflow_status ws
2060
- LEFT JOIN "${this.schemaName}".workflow_output wo ON wo.workflow_uuid = ws.workflow_uuid
2061
- WHERE ws.workflow_uuid=$1`, [workflowID])));
2645
+ LEFT JOIN "${this.schemaName}".workflow_output wo ON ${this.wfJoin('ws', 'wo')}
2646
+ WHERE ${target}`, params)));
2062
2647
  }
2063
2648
  catch (e) {
2064
2649
  const err = e;
@@ -2095,21 +2680,27 @@ class SystemDatabase {
2095
2680
  }
2096
2681
  }
2097
2682
  async awaitFirstWorkflowId(workflowIds, callerID, pollingIntervalMs) {
2098
- const placeholders = workflowIds.map((_, i) => `$${i + 1}`).join(', ');
2099
2683
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
2100
2684
  while (true) {
2101
2685
  if (callerID)
2102
2686
  await this.checkIfCanceledLimited(callerID);
2103
- const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2104
- WHERE workflow_uuid IN (${placeholders})
2105
- AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
2106
- LIMIT 1`, workflowIds));
2107
- if (rows.length > 0) {
2108
- return rows[0].workflow_uuid;
2687
+ const first = await this.checkFirstWorkflowId(workflowIds);
2688
+ if (first !== undefined) {
2689
+ return first;
2109
2690
  }
2110
2691
  await (0, utils_1.sleepms)(pollIntervalMs);
2111
2692
  }
2112
2693
  }
2694
+ /** The first of these workflows found complete, if any. */
2695
+ async checkFirstWorkflowId(workflowIds) {
2696
+ const params = [];
2697
+ const ids = this.wfIn(undefined, workflowIds, params);
2698
+ const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2699
+ WHERE ${ids}
2700
+ AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')
2701
+ LIMIT 1`, params));
2702
+ return rows[0]?.workflow_uuid;
2703
+ }
2113
2704
  async awaitWorkflowIds(workflowIds, callerID, pollingIntervalMs) {
2114
2705
  const remainingWorkflowIds = new Set(workflowIds);
2115
2706
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
@@ -2117,11 +2708,8 @@ class SystemDatabase {
2117
2708
  const currentWorkflowIds = [...remainingWorkflowIds];
2118
2709
  if (callerID)
2119
2710
  await this.checkIfCanceledLimited(callerID);
2120
- const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2121
- WHERE workflow_uuid = ANY($1::text[])
2122
- AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, [currentWorkflowIds]));
2123
- for (const row of rows) {
2124
- remainingWorkflowIds.delete(row.workflow_uuid);
2711
+ for (const workflowID of await this.checkCompletedWorkflowIds(currentWorkflowIds)) {
2712
+ remainingWorkflowIds.delete(workflowID);
2125
2713
  }
2126
2714
  if (remainingWorkflowIds.size === 0) {
2127
2715
  return;
@@ -2129,9 +2717,18 @@ class SystemDatabase {
2129
2717
  await (0, utils_1.sleepms)(pollIntervalMs);
2130
2718
  }
2131
2719
  }
2720
+ /** Those of these workflows found complete. */
2721
+ async checkCompletedWorkflowIds(workflowIds) {
2722
+ const params = [];
2723
+ const ids = this.wfIn(undefined, workflowIds, params);
2724
+ const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2725
+ WHERE ${ids}
2726
+ AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')`, params));
2727
+ return rows.map((row) => row.workflow_uuid);
2728
+ }
2132
2729
  // ==================== Sleep ====================
2133
2730
  async durableSleepms(workflowID, functionID, durationMS) {
2134
- const endTime = await this.#durableSleep(workflowID, functionID, durationMS, true);
2731
+ const endTime = await this.durableSleepInternal(workflowID, functionID, durationMS, true);
2135
2732
  while (Date.now() < endTime) {
2136
2733
  await (0, utils_1.sleepms)(Math.min(endTime - Date.now(), utils_1.sleepConfig.maxTimeoutMS));
2137
2734
  }
@@ -2139,31 +2736,28 @@ class SystemDatabase {
2139
2736
  }
2140
2737
  // ==================== Messaging ====================
2141
2738
  nullTopic = '__null__topic__';
2142
- async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey) {
2143
- topic = topic ?? this.nullTopic;
2144
- const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
2739
+ async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey,
2740
+ // The destination's shard key; required on a sharded system database.
2741
+ shardKey) {
2145
2742
  const startTime = Date.now();
2146
2743
  let missing;
2147
2744
  const client = await this.connect();
2148
2745
  try {
2149
2746
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
2150
2747
  await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
2151
- await client.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
2152
- VALUES ($1, $2, $3, $4, $5)
2153
- ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
2748
+ await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2154
2749
  return undefined;
2155
2750
  });
2156
2751
  await client.query('COMMIT');
2157
2752
  }
2158
2753
  catch (error) {
2159
2754
  await client.query('ROLLBACK');
2160
- const err = error;
2161
- if (err.code === '23503') {
2755
+ if (error instanceof error_1.DBOSNonExistentWorkflowError) {
2162
2756
  // Foreign key constraint violation (only expected for the INSERT query)
2163
- missing = new error_1.DBOSNonExistentWorkflowError(`Sent to non-existent destination workflow UUID: ${destinationID}`);
2757
+ missing = error;
2164
2758
  }
2165
2759
  else {
2166
- throw err;
2760
+ throw error;
2167
2761
  }
2168
2762
  }
2169
2763
  finally {
@@ -2176,22 +2770,22 @@ class SystemDatabase {
2176
2770
  }
2177
2771
  }
2178
2772
  /** Runs on `client` if given, joining its transaction; otherwise on the pool with retries. */
2179
- async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client) {
2773
+ async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client, shardKey) {
2180
2774
  if (client !== undefined) {
2181
- return await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
2775
+ return await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2182
2776
  }
2183
- return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey);
2777
+ return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey);
2184
2778
  }
2185
2779
  /** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
2186
- async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
2780
+ async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
2187
2781
  const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
2188
2782
  const client = await this.connect();
2189
2783
  try {
2190
- await this.#inTransaction(client, async () => {
2191
- await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
2784
+ await this.inTransaction(client, async () => {
2785
+ await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2192
2786
  // After the insert, in the order the recorded send locks, so they cannot deadlock.
2193
2787
  if (ownerXid !== undefined) {
2194
- await this.#checkOwner(client, workflowID, ownerXid);
2788
+ await this.checkOwner(client, workflowID, ownerXid);
2195
2789
  }
2196
2790
  });
2197
2791
  }
@@ -2199,17 +2793,33 @@ class SystemDatabase {
2199
2793
  client.release();
2200
2794
  }
2201
2795
  }
2202
- async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey) {
2203
- return await this.#sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey);
2796
+ async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey) {
2797
+ return await this.sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2204
2798
  }
2205
- async #sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey) {
2799
+ async sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
2800
+ if (db === this.pool) {
2801
+ // All sends need a transaction now because they wake up suspended workflows unconditionally.
2802
+ const client = await this.connect();
2803
+ try {
2804
+ return await this.inTransaction(client, () => this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey));
2805
+ }
2806
+ finally {
2807
+ client.release();
2808
+ }
2809
+ }
2206
2810
  topic = topic ?? this.nullTopic;
2207
- // Same per-destination scoping as send() above.
2811
+ // An idempotency key is scoped per destination: suffix it with the recipient's workflow ID.
2208
2812
  const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
2813
+ const values = {
2814
+ ...this.wfValues(destinationID, shardKey),
2815
+ destination_uuid: destinationID,
2816
+ topic,
2817
+ message,
2818
+ serialization,
2819
+ message_uuid: messageUUID,
2820
+ };
2209
2821
  try {
2210
- await db.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
2211
- VALUES ($1, $2, $3, $4, $5)
2212
- ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
2822
+ await this.insertRows(db, 'notifications', Object.keys(values), [Object.values(values)], `ON CONFLICT (${this.keyConflict('message_uuid')}) DO NOTHING`);
2213
2823
  }
2214
2824
  catch (error) {
2215
2825
  const err = error;
@@ -2218,6 +2828,23 @@ class SystemDatabase {
2218
2828
  }
2219
2829
  throw err;
2220
2830
  }
2831
+ await this.wakeReceivers(db, [destinationID], [topic]);
2832
+ }
2833
+ /**
2834
+ * Make SUSPENDED receivers waiting on one of `topics` due, for the background pass to re-enqueue.
2835
+ * Runs on every send, whether or not this process uses suspend mode, so any sender wakes a workflow
2836
+ * another process parked. A primary-key probe per destination; it matches nothing in the common case.
2837
+ *
2838
+ * The match is destinations x topics, so a batch sending A on topic X and B on topic Y also wakes A
2839
+ * if it waits on Y. The spurious wake-up is harmless: the replayed recv finds no message and parks again.
2840
+ */
2841
+ async wakeReceivers(db, destinations, topics) {
2842
+ const params = [topics];
2843
+ await db.query(
2844
+ // Status is tested in SET, not WHERE, so the planner cannot scan idx_workflow_status_suspended.
2845
+ `UPDATE "${this.schemaName}".workflow_status
2846
+ SET delay_until_epoch_ms = CASE WHEN status = '${workflow_1.StatusString.SUSPENDED}' THEN 0 ELSE delay_until_epoch_ms END
2847
+ WHERE ${this.wfIn(undefined, destinations, params)} AND suspend_key = ANY($1::text[])`, params);
2221
2848
  }
2222
2849
  async recv(workflowID, functionID, timeoutFunctionID, topic, timeoutSeconds = dbos_executor_1.DBOSExecutor.defaultNotificationTimeoutSec, pollingIntervalMs) {
2223
2850
  topic = topic ?? this.nullTopic;
@@ -2233,10 +2860,10 @@ class SystemDatabase {
2233
2860
  const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
2234
2861
  let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
2235
2862
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
2236
- // Record the durable timeout deadline once before polling. #durableSleep persists it on the
2863
+ // Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
2237
2864
  // first call and reads back the same value on recovery, so it never changes across iterations.
2238
2865
  if (timeoutms) {
2239
- finishTime = await this.#durableSleep(workflowID, timeoutFunctionID, timeoutms);
2866
+ finishTime = await this.durableSleepInternal(workflowID, timeoutFunctionID, timeoutms);
2240
2867
  }
2241
2868
  while (true) {
2242
2869
  // register the key with the global notifications listener.
@@ -2244,12 +2871,14 @@ class SystemDatabase {
2244
2871
  const messagePromise = new Promise((resolve) => {
2245
2872
  resolveNotification = resolve;
2246
2873
  });
2247
- const payload = `${workflowID}::${topic}`;
2874
+ const payload = this.waiter(workflowID, topic);
2248
2875
  const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
2249
2876
  try {
2250
2877
  await this.checkIfCanceledLimited(workflowID);
2251
2878
  // Check if the key is already in the DB, then wait for the notification if it isn't.
2252
- const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE destination_uuid=$1 AND topic=$2 AND consumed = false;`, [workflowID, topic]))).rows;
2879
+ const params = [topic];
2880
+ const target = this.wf(undefined, workflowID, params, 'destination_uuid');
2881
+ const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE ${target} AND topic=$1 AND consumed = false;`, params))).rows;
2253
2882
  if (initRecvRows.length !== 0)
2254
2883
  break;
2255
2884
  const ct = Date.now();
@@ -2269,28 +2898,46 @@ class SystemDatabase {
2269
2898
  this.notificationsMap.deregisterCallback(cbr);
2270
2899
  }
2271
2900
  }
2272
- await this.checkIfCanceled(workflowID);
2901
+ return await this.recvConsume(workflowID, functionID, topic, startTime);
2902
+ }
2903
+ /**
2904
+ * Consume and return the oldest message on `topic`, or null if there is none, recording it as the recv at `functionID`. Idempotent on `functionID`.
2905
+ */
2906
+ async recvConsume(workflowID, functionID, topic, startTime) {
2907
+ topic = topic ?? this.nullTopic;
2908
+ const res = await this.getOperationResultAndThrowIfCancelled(workflowID, functionID);
2909
+ if (res) {
2910
+ if (res.functionName !== exports.DBOS_FUNCNAME_RECV) {
2911
+ throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_RECV, res.functionName);
2912
+ }
2913
+ const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
2914
+ if (ownerXid !== undefined && (await this.getWorkflowOwner(workflowID)) !== ownerXid) {
2915
+ throw new error_1.DBOSWorkflowConflictError(workflowID);
2916
+ }
2917
+ return { serializedValue: res.output, serialization: res.serialization ?? null };
2918
+ }
2273
2919
  // Transactionally consume and return the message if it's in the DB, otherwise return null.
2274
2920
  let message = null;
2275
2921
  let serialization = null;
2276
2922
  const client = await this.connect();
2277
2923
  try {
2278
2924
  await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
2925
+ const consumeParams = [topic, functionID];
2279
2926
  const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
2280
- SET consumed = true, consumed_by_function_id = $3
2281
- WHERE destination_uuid = $1
2282
- AND topic = $2
2927
+ SET consumed = true, consumed_by_function_id = $2
2928
+ WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
2929
+ AND topic = $1
2283
2930
  AND consumed = false
2284
2931
  AND message_uuid = (
2285
2932
  SELECT message_uuid
2286
2933
  FROM "${this.schemaName}".notifications
2287
- WHERE destination_uuid = $1
2288
- AND topic = $2
2934
+ WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
2935
+ AND topic = $1
2289
2936
  AND consumed = false
2290
2937
  ORDER BY created_at_epoch_ms ASC
2291
2938
  LIMIT 1
2292
2939
  )
2293
- RETURNING notifications.message, notifications.serialization;`, [workflowID, topic, functionID])).rows;
2940
+ RETURNING notifications.message, notifications.serialization;`, consumeParams)).rows;
2294
2941
  if (finalRecvRows.length > 0) {
2295
2942
  message = finalRecvRows[0].message;
2296
2943
  serialization = finalRecvRows[0].serialization;
@@ -2319,16 +2966,19 @@ class SystemDatabase {
2319
2966
  // Only a real write (not a replay) should wake readers.
2320
2967
  let didWrite = false;
2321
2968
  await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SETEVENT, workflowID, functionID, async () => {
2322
- await client.query(`INSERT INTO "${this.schemaName}".workflow_events (workflow_uuid, key, value, serialization)
2323
- VALUES ($1, $2, $3, $4)
2324
- ON CONFLICT (workflow_uuid, key)
2969
+ const keys = this.wfValues(workflowID);
2970
+ const keyColumns = Object.keys(keys).map((c) => `${c}, `);
2971
+ const keyPlaceholders = (from) => Object.keys(keys).map((_, i) => `$${from + i}, `);
2972
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_events (${keyColumns.join('')}workflow_uuid, key, value, serialization)
2973
+ VALUES (${keyPlaceholders(5).join('')}$1, $2, $3, $4)
2974
+ ON CONFLICT (${this.keyConflict('workflow_uuid', 'key')})
2325
2975
  DO UPDATE SET value = $3, serialization = $4
2326
- RETURNING workflow_uuid;`, [workflowID, key, message, serialization]);
2976
+ RETURNING workflow_uuid;`, [workflowID, key, message, serialization, ...Object.values(keys)]);
2327
2977
  // Also write to the immutable history table for fork support
2328
- await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (workflow_uuid, function_id, key, value, serialization)
2329
- VALUES ($1, $2, $3, $4, $5)
2330
- ON CONFLICT (workflow_uuid, function_id, key)
2331
- DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization]);
2978
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (${keyColumns.join('')}workflow_uuid, function_id, key, value, serialization)
2979
+ VALUES (${keyPlaceholders(6).join('')}$1, $2, $3, $4, $5)
2980
+ ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id', 'key')})
2981
+ DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization, ...Object.values(keys)]);
2332
2982
  didWrite = true;
2333
2983
  return undefined;
2334
2984
  });
@@ -2362,15 +3012,15 @@ class SystemDatabase {
2362
3012
  // Get the return the value. if it's in the DB, otherwise return null.
2363
3013
  let value = null;
2364
3014
  let valueSer = null;
2365
- const payloadKey = `${workflowID}::${key}`;
3015
+ const payloadKey = this.waiter(workflowID, key);
2366
3016
  const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
2367
3017
  let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
2368
3018
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
2369
3019
  // If we have a callerWorkflow, we want a durable sleep, otherwise, not. Record the durable
2370
- // timeout deadline once before polling. #durableSleep persists it on the first call and reads
3020
+ // timeout deadline once before polling. durableSleepInternal persists it on the first call and reads
2371
3021
  // back the same value on recovery, so it never changes across iterations.
2372
3022
  if (callerWorkflow && timeoutms) {
2373
- finishTime = await this.#durableSleep(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms);
3023
+ finishTime = await this.durableSleepInternal(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms);
2374
3024
  }
2375
3025
  // Register the key with the global notifications listener first... we do not want to look in the DB first
2376
3026
  // or that would cause a timing hole.
@@ -2384,9 +3034,11 @@ class SystemDatabase {
2384
3034
  if (callerWorkflow?.workflowID)
2385
3035
  await this.checkIfCanceledLimited(callerWorkflow?.workflowID);
2386
3036
  // Check if the key is already in the DB, then wait for the notification if it isn't.
3037
+ const params = [key];
3038
+ const target = this.wf(undefined, workflowID, params);
2387
3039
  const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT key, value, serialization
2388
3040
  FROM "${this.schemaName}".workflow_events
2389
- WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key]))).rows;
3041
+ WHERE ${target} AND key=$1;`, params))).rows;
2390
3042
  if (initRecvRows.length > 0) {
2391
3043
  value = initRecvRows[0].value;
2392
3044
  valueSer = initRecvRows[0].serialization;
@@ -2425,15 +3077,12 @@ class SystemDatabase {
2425
3077
  try {
2426
3078
  while (true) {
2427
3079
  try {
2428
- await this.#inTransaction(client, async () => {
3080
+ await this.inTransaction(client, async () => {
2429
3081
  // Derives the first unused offset inside the insert; two writers can still pick the same one.
2430
- await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
2431
- SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
2432
- FROM "${this.schemaName}".streams s
2433
- WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
3082
+ await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
2434
3083
  // After the insert, in the order the workflow-level writes lock, so they cannot deadlock.
2435
3084
  if (ownerXid !== undefined) {
2436
- await this.#checkOwner(client, workflowID, ownerXid);
3085
+ await this.checkOwner(client, workflowID, ownerXid);
2437
3086
  }
2438
3087
  });
2439
3088
  }
@@ -2467,10 +3116,7 @@ class SystemDatabase {
2467
3116
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
2468
3117
  await this.#runAndRecordResult(client, functionName, workflowID, functionID, async () => {
2469
3118
  // Derives the first unused offset inside the insert; two writers can still pick the same one.
2470
- await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
2471
- SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
2472
- FROM "${this.schemaName}".streams s
2473
- WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
3119
+ await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
2474
3120
  didWrite = true;
2475
3121
  return undefined;
2476
3122
  });
@@ -2502,6 +3148,15 @@ class SystemDatabase {
2502
3148
  client.release();
2503
3149
  }
2504
3150
  }
3151
+ /** Append a value to a stream, at the first unused offset, derived inside the insert. */
3152
+ async #insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization) {
3153
+ const params = [key, serializedValue, functionID, serialization, workflowID];
3154
+ const keyColumns = this.wfColumns(workflowID, params);
3155
+ await client.query(`INSERT INTO "${this.schemaName}".streams (${keyColumns.map(([c]) => `${c}, `).join('')}workflow_uuid, key, value, "offset", function_id, serialization)
3156
+ SELECT ${keyColumns.map(([, e]) => `${e}, `).join('')}$5::text, $1::text, $2::text, COALESCE(MAX(s."offset"), -1) + 1, $3::int, $4::text
3157
+ FROM "${this.schemaName}".streams s
3158
+ WHERE ${this.wf('s', workflowID, params)} AND s.key = $1`, params);
3159
+ }
2505
3160
  async closeStreamFromWorkflow(workflowID, functionID, key) {
2506
3161
  await this.writeStreamFromWorkflow(workflowID, functionID, key, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name(), exports.DBOS_FUNCNAME_CLOSESTREAM);
2507
3162
  }
@@ -2511,13 +3166,15 @@ class SystemDatabase {
2511
3166
  // Read the value at `offset` and the workflow's status in one query: status null = no such workflow, value undefined = nothing at that offset.
2512
3167
  async readStreamValue(workflowID, key, offset) {
2513
3168
  // LEFT JOIN so a workflow with nothing at offset still reports its status (single PK lookup); under the poll limiter, inside @dbRetry so the permit frees across backoff.
3169
+ const params = [key, offset];
3170
+ const target = this.wf('ws', workflowID, params);
2514
3171
  const result = await this.#pollWithLimiter(() => this.pool.query(
2515
3172
  // "offset" is a reserved word, so alias it (stream_offset) to read it back plainly.
2516
3173
  `SELECT ws.status AS status, s.value AS value, s.serialization AS serialization, s."offset" AS stream_offset
2517
3174
  FROM "${this.schemaName}".workflow_status ws
2518
3175
  LEFT OUTER JOIN "${this.schemaName}".streams s
2519
- ON s.workflow_uuid = ws.workflow_uuid AND s.key = $2 AND s."offset" = $3
2520
- WHERE ws.workflow_uuid = $1`, [workflowID, key, offset]));
3176
+ ON ${this.wfJoin('s', 'ws')} AND s.key = $1 AND s."offset" = $2
3177
+ WHERE ${target}`, params));
2521
3178
  if (result.rows.length === 0) {
2522
3179
  return { status: null, value: undefined };
2523
3180
  }
@@ -2624,15 +3281,16 @@ class SystemDatabase {
2624
3281
  async cancelTimedOutWorkflows(limit) {
2625
3282
  const params = [workflow_1.StatusString.CANCELLED, Date.now(), limit];
2626
3283
  const scope = this.appNameFilter('application_name', this.appName, params);
2627
- // Literal statuses let the planner match idx_workflow_status_deadline under any plan mode.
3284
+ // Literal statuses let the planner match idx_workflow_status_deadline_v2 under any plan mode.
2628
3285
  // SKIP LOCKED leaves a row a dequeue or a peer's sweep holds for the next sweep.
2629
3286
  const { rows } = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
2630
3287
  SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
3288
+ suspend_key = NULL,
2631
3289
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
2632
3290
  completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
2633
3291
  WHERE workflow_uuid IN (
2634
3292
  SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2635
- WHERE status IN ('${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.DELAYED}')
3293
+ WHERE status IN ('${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')
2636
3294
  AND workflow_deadline_epoch_ms IS NOT NULL
2637
3295
  AND workflow_deadline_epoch_ms <= $2
2638
3296
  AND ${scope}
@@ -2654,7 +3312,7 @@ class SystemDatabase {
2654
3312
  // Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v3, so cost scales with the number of partitions rather than the backlog depth.
2655
3313
  const params = [queueName, workflow_1.StatusString.ENQUEUED];
2656
3314
  // Only partitions this application can actually dequeue from.
2657
- const scope = this.appNameFilter('application_name', this.appName, params);
3315
+ const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
2658
3316
  const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
2659
3317
  (SELECT MIN(queue_partition_key) AS pk
2660
3318
  FROM "${this.schemaName}".workflow_status
@@ -2671,7 +3329,14 @@ class SystemDatabase {
2671
3329
  }
2672
3330
  async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0,
2673
3331
  // Written to every claimed row: the dispatches of this claim own their executions.
2674
- ownerXid = null) {
3332
+ ownerXid = null, isLatestVersion) {
3333
+ const { claimedIDs } = await this.claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount, partitionLocalRunningCount, ownerXid, isLatestVersion);
3334
+ return claimedIDs;
3335
+ }
3336
+ /** Claim this queue's next workflows, plus the limit that left no room to claim any (undefined if every limit had room). */
3337
+ async claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0, ownerXid = null, isLatestVersion) {
3338
+ // Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
3339
+ const startTimeMs = Date.now();
2675
3340
  const claimedIDs = [];
2676
3341
  const partitionParams = queuePartitionKey !== undefined ? [queuePartitionKey] : [];
2677
3342
  // Shares a concurrency or rate limit budget with other executors.
@@ -2679,13 +3344,49 @@ class SystemDatabase {
2679
3344
  queue.partitionConcurrency !== undefined ||
2680
3345
  queue.rateLimit !== undefined ||
2681
3346
  queue.partitionRateLimit !== undefined;
2682
- // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
3347
+ // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it: lock the whole queue.
2683
3348
  const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
3349
+ // Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
3350
+ let maxTasks = Infinity;
3351
+ // The first limit found with no room, which the caller reads as why nothing was claimed.
3352
+ let stopReason = undefined;
3353
+ const limit = (room, reason) => {
3354
+ maxTasks = Math.min(maxTasks, room);
3355
+ if (maxTasks <= 0)
3356
+ stopReason ??= reason;
3357
+ };
3358
+ // This worker's own limits need no database, so a full worker returns before taking the claim lock a peer could use.
3359
+ if (queue.workerConcurrency !== undefined) {
3360
+ // Use the in-memory registry for this worker's running count — avoids a DB round trip.
3361
+ limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
3362
+ }
3363
+ if (queue.partitionWorkerConcurrency !== undefined) {
3364
+ limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
3365
+ }
3366
+ if (maxTasks <= 0) {
3367
+ return { claimedIDs, stopReason };
3368
+ }
3369
+ // A sharded database keeps the version row under another key, which the claim's transaction cannot span.
3370
+ if (isLatestVersion === undefined && this.sharded) {
3371
+ isLatestVersion = await this.isLatestVersion(appVersion);
3372
+ }
3373
+ // Row locks only collide when claimers see the same queue head, so serialize claims on the budget's scope; CockroachDB has no advisory locks before 26.3.
3374
+ const useClaimLock = hasSharedBudget && (this.sharded || !(await this.#isCockroach()));
2684
3375
  const client = await this.connect();
2685
3376
  try {
2686
- // Default to READ COMMITTED except with a budget shared across executors
2687
- if (hasSharedBudget) {
2688
- await client.query(`BEGIN ISOLATION LEVEL ${hasWriteSkew ? 'SERIALIZABLE' : 'REPEATABLE READ'}`);
3377
+ if (useClaimLock) {
3378
+ await client.query('BEGIN');
3379
+ const lockParams = [];
3380
+ const { rows } = await client.query(this.claimLockQuery(queue.name, hasWriteSkew ? undefined : queuePartitionKey, lockParams), lockParams);
3381
+ if (rows[0]?.acquired !== true) {
3382
+ await client.query('COMMIT');
3383
+ // Another worker is claiming from this scope, so it still has work; the next poll retries.
3384
+ return rows.length > 0 ? { claimedIDs, stopReason: 'claim_lock' } : { claimedIDs };
3385
+ }
3386
+ }
3387
+ else if (hasSharedBudget) {
3388
+ // CockroachDB, without advisory locks before 26.3: SERIALIZABLE aborts claimers that spend the same budget.
3389
+ await client.query('BEGIN ISOLATION LEVEL SERIALIZABLE');
2689
3390
  }
2690
3391
  else {
2691
3392
  await client.query('BEGIN');
@@ -2694,7 +3395,7 @@ class SystemDatabase {
2694
3395
  const rateLimitRemaining = async (rateLimit, partitionScoped) => {
2695
3396
  const params = [queue.name, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED, rateLimit.periodSec * 1000];
2696
3397
  // Count only what this application would dequeue, matching the select below.
2697
- const scope = this.appNameFilter('application_name', this.appName, params);
3398
+ const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
2698
3399
  const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
2699
3400
  const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
2700
3401
  WHERE queue_name = $1
@@ -2713,35 +3414,26 @@ class SystemDatabase {
2713
3414
  */
2714
3415
  const pendingCount = async (partitionScoped) => {
2715
3416
  const params = [queue.name, workflow_1.StatusString.PENDING];
2716
- const scope = this.appNameFilter('application_name', this.appName, params);
3417
+ const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
2717
3418
  const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
2718
3419
  const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
2719
3420
  WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
2720
3421
  return Number(rows[0]?.count ?? 0);
2721
3422
  };
2722
- // Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
2723
- let maxTasks = Infinity;
2724
- if (queue.workerConcurrency !== undefined) {
2725
- // Use the in-memory registry for this worker's running count — avoids a DB round trip.
2726
- maxTasks = Math.min(maxTasks, Math.max(0, queue.workerConcurrency - localRunningCount));
2727
- }
2728
- if (queue.partitionWorkerConcurrency !== undefined) {
2729
- maxTasks = Math.min(maxTasks, Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount));
2730
- }
2731
- if (maxTasks <= 0) {
2732
- await client.query('COMMIT');
2733
- return claimedIDs;
2734
- }
2735
3423
  if (queue.rateLimit !== undefined) {
2736
3424
  // Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
2737
- maxTasks = Math.min(maxTasks, await rateLimitRemaining(queue.rateLimit, false));
3425
+ limit(await rateLimitRemaining(queue.rateLimit, false), 'limiter');
3426
+ if (maxTasks <= 0) {
3427
+ await client.query('COMMIT');
3428
+ return { claimedIDs, stopReason };
3429
+ }
2738
3430
  }
2739
3431
  if (queue.partitionRateLimit !== undefined) {
2740
- maxTasks = Math.min(maxTasks, await rateLimitRemaining(queue.partitionRateLimit, true));
3432
+ limit(await rateLimitRemaining(queue.partitionRateLimit, true), 'partition_limiter');
2741
3433
  }
2742
3434
  if (maxTasks <= 0) {
2743
3435
  await client.query('COMMIT');
2744
- return claimedIDs;
3436
+ return { claimedIDs, stopReason };
2745
3437
  }
2746
3438
  if (queue.concurrency !== undefined) {
2747
3439
  // Global concurrency still requires a DB query since other workers may be running workflows too.
@@ -2749,23 +3441,26 @@ class SystemDatabase {
2749
3441
  if (totalRunningTasks > queue.concurrency) {
2750
3442
  this.logger.warn(`Total running tasks (${totalRunningTasks}) exceeds the global concurrency limit (${queue.concurrency})`);
2751
3443
  }
2752
- maxTasks = Math.min(maxTasks, Math.max(0, queue.concurrency - totalRunningTasks));
3444
+ limit(Math.max(0, queue.concurrency - totalRunningTasks), 'concurrency');
3445
+ if (maxTasks <= 0) {
3446
+ await client.query('COMMIT');
3447
+ return { claimedIDs, stopReason };
3448
+ }
2753
3449
  }
2754
3450
  if (queue.partitionConcurrency !== undefined) {
2755
3451
  const partitionRunningTasks = await pendingCount(true);
2756
3452
  if (partitionRunningTasks > queue.partitionConcurrency) {
2757
3453
  this.logger.warn(`Total running tasks (${partitionRunningTasks}) on partition ${queuePartitionKey} of queue ${queue.name} exceeds the partition concurrency limit (${queue.partitionConcurrency})`);
2758
3454
  }
2759
- maxTasks = Math.min(maxTasks, Math.max(0, queue.partitionConcurrency - partitionRunningTasks));
3455
+ limit(Math.max(0, queue.partitionConcurrency - partitionRunningTasks), 'partition_concurrency');
2760
3456
  }
2761
3457
  // Return immediately if there are no available tasks due to flow control limits
2762
3458
  if (maxTasks <= 0) {
2763
3459
  await client.query('COMMIT');
2764
- return claimedIDs;
3460
+ return { claimedIDs, stopReason };
2765
3461
  }
2766
3462
  // Retrieve the first max_tasks workflows in the queue.
2767
- const latestVersion = await this.#latestApplicationVersionName(client);
2768
- const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
3463
+ isLatestVersion ??= await this.isLatestVersion(appVersion, client);
2769
3464
  const versionClause = isLatestVersion
2770
3465
  ? '(application_version = $3 OR application_version IS NULL)'
2771
3466
  : 'application_version = $3';
@@ -2775,13 +3470,17 @@ class SystemDatabase {
2775
3470
  const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
2776
3471
  const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
2777
3472
  const selectScope = this.appNameFilter('application_name', this.appName, selectParams);
3473
+ const keyScope = this.scoped(undefined, selectParams);
3474
+ const claimGate = this.claimGate(queue.name, queuePartitionKey, selectParams);
2778
3475
  const selectQuery = `
2779
3476
  SELECT workflow_uuid
2780
3477
  FROM "${this.schemaName}".workflow_status
2781
- WHERE status = $1
3478
+ WHERE ${keyScope}
3479
+ AND status = $1
2782
3480
  AND queue_name = $2
2783
3481
  AND ${versionClause}
2784
3482
  AND ${selectScope}
3483
+ AND ${claimGate}
2785
3484
  ${queuePartitionKey !== undefined ? 'AND queue_partition_key = $4' : ''}
2786
3485
  ORDER BY priority ASC, created_at ASC
2787
3486
  ${limitClause}
@@ -2800,35 +3499,36 @@ class SystemDatabase {
2800
3499
  executorID,
2801
3500
  appVersion,
2802
3501
  queue.rateLimit !== undefined || queue.partitionRateLimit !== undefined,
2803
- workflowIDs,
2804
3502
  workflow_1.StatusString.ENQUEUED,
2805
3503
  // Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
2806
3504
  this.appName ?? null,
2807
3505
  ownerXid,
3506
+ startTimeMs,
2808
3507
  ];
3508
+ const claimedIDsFilter = this.wfIn(undefined, workflowIDs, updateParams);
2809
3509
  // Re-check ownership alongside status, as the partitioned claim guard does.
2810
3510
  const claimScope = this.appNameFilter('application_name', this.appName, updateParams);
2811
3511
  // RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
2812
3512
  const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
2813
3513
  SET status = $1,
2814
- owner_xid = $8,
3514
+ owner_xid = $7,
2815
3515
  executor_id = $2,
2816
3516
  application_version = $3,
2817
3517
  started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2818
3518
  rate_limited = $4,
2819
- application_name = COALESCE(application_name, $7),
3519
+ application_name = COALESCE(application_name, $6),
2820
3520
  recovery_attempts = recovery_attempts + 1,
2821
3521
  updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2822
3522
  workflow_deadline_epoch_ms = CASE
2823
3523
  WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
2824
3524
  CASE
2825
- WHEN workflow_timeout_ms > 9223372036854775807 - (EXTRACT(epoch FROM now()) * 1000)::bigint
3525
+ WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
2826
3526
  THEN 9223372036854775807
2827
- ELSE (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
3527
+ ELSE $8::bigint + workflow_timeout_ms
2828
3528
  END
2829
3529
  ELSE workflow_deadline_epoch_ms
2830
3530
  END
2831
- WHERE workflow_uuid = ANY($5::text[]) AND status = $6 AND ${claimScope}
3531
+ WHERE ${claimedIDsFilter} AND status = $5 AND ${claimScope}
2832
3532
  RETURNING workflow_uuid`, updateParams);
2833
3533
  const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
2834
3534
  for (const id of workflowIDs) {
@@ -2846,14 +3546,18 @@ class SystemDatabase {
2846
3546
  client.release();
2847
3547
  }
2848
3548
  // Return the IDs of all functions we marked started
2849
- return claimedIDs;
3549
+ return { claimedIDs };
2850
3550
  }
2851
3551
  /** Max heads admitted per sweep: bounds dispatch, not the partition walk; an unconstrained sweep takes the lowest keys, so higher keys can wait under sustained load. */
2852
3552
  partitionedDequeueSweepCap = 8192;
2853
- /** Dequeue each partition's head-of-line workflow in one transaction, at most {@link partitionedDequeueSweepCap} per sweep; only valid for partition-concurrency-1 queues with no queue-wide limit. */
3553
+ /** Max partitions a sweep claims per transaction, which bounds the locks it holds at once and for how long. */
3554
+ partitionedDequeueBatchSize = 64;
3555
+ /** Dequeue each partition's head-of-line workflow, at most {@link partitionedDequeueSweepCap} per sweep in transactions of {@link partitionedDequeueBatchSize}; only valid for partition-concurrency-1 queues with no queue-wide limit. */
2854
3556
  async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity,
2855
3557
  // Written to every claimed row: the dispatches of this claim own their executions.
2856
- ownerXid = null) {
3558
+ ownerXid = null, isLatestVersion) {
3559
+ // Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
3560
+ const startTimeMs = Date.now();
2857
3561
  if (queue.partitionConcurrency !== 1 ||
2858
3562
  queue.concurrency !== undefined ||
2859
3563
  queue.rateLimit !== undefined ||
@@ -2861,14 +3565,35 @@ class SystemDatabase {
2861
3565
  throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
2862
3566
  }
2863
3567
  // partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
3568
+ // A sharded database keeps the version row under another key, which the claim's transaction cannot span.
3569
+ if (isLatestVersion === undefined && this.sharded) {
3570
+ isLatestVersion = await this.isLatestVersion(appVersion);
3571
+ }
3572
+ // Sweeps whose heads differ lock disjoint rows, so a claim also takes its partition's lock; CockroachDB, without advisory locks before 26.3, relies on its SERIALIZABLE default.
3573
+ const usePartitionLocks = this.sharded || !(await this.#isCockroach());
2864
3574
  const client = await this.connect();
2865
3575
  try {
2866
- await client.query('BEGIN');
2867
- const latestVersion = await this.#latestApplicationVersionName(client);
2868
- const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
3576
+ isLatestVersion ??= await this.isLatestVersion(appVersion, client);
2869
3577
  const versionClause = (n) => isLatestVersion
2870
3578
  ? `(application_version = $${n} OR application_version IS NULL)`
2871
3579
  : `application_version = $${n}`;
3580
+ // The partition's head: $1 queue, $2 ENQUEUED, `version` the app version, `pk` the partition key expression.
3581
+ const headOf = (pk, version, scope) => `SELECT workflow_uuid
3582
+ FROM "${this.schemaName}".workflow_status
3583
+ WHERE queue_name = $1 AND status = $2
3584
+ AND queue_partition_key = ${pk}
3585
+ AND ${versionClause(version)}
3586
+ AND ${scope}
3587
+ -- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
3588
+ ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
3589
+ LIMIT 1`;
3590
+ // No application filter by design: a mutual-exclusion probe must block on any owner's row.
3591
+ const noPending = (pk, pending, scope) => `NOT EXISTS (
3592
+ SELECT 1
3593
+ FROM "${this.schemaName}".workflow_status
3594
+ WHERE ${scope} AND queue_name = $1 AND status = $${pending}
3595
+ AND queue_partition_key IS NOT NULL AND queue_partition_key = ${pk}
3596
+ )`;
2872
3597
  // This worker's own budget bounds the sweep alongside the cap.
2873
3598
  const sweepLimit = Math.min(this.partitionedDequeueSweepCap, maxTasks);
2874
3599
  // When the worker's budget is the binding constraint, probe partitions in random order to prevent starvation.
@@ -2880,8 +3605,10 @@ class SystemDatabase {
2880
3605
  workflow_1.StatusString.PENDING,
2881
3606
  sweepLimit,
2882
3607
  ];
2883
- const candidateScope = this.appNameFilter('application_name', this.appName, candidateParams);
3608
+ const candidateScope = `${this.scoped(undefined, candidateParams)} AND ${this.appNameFilter('application_name', this.appName, candidateParams)}`;
3609
+ const probeScope = this.scoped(undefined, candidateParams);
2884
3610
  // Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
3611
+ // The walk takes no lock: each batch below picks its heads again under its partitions' locks.
2885
3612
  const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
2886
3613
  (SELECT MIN(queue_partition_key) AS pk
2887
3614
  FROM "${this.schemaName}".workflow_status
@@ -2897,118 +3624,140 @@ class SystemDatabase {
2897
3624
  SELECT partitions.pk
2898
3625
  FROM partitions
2899
3626
  WHERE partitions.pk IS NOT NULL
2900
- -- Unscoped by design: a mutual-exclusion probe must block on any owner's row.
2901
- AND NOT EXISTS (
2902
- SELECT 1
2903
- FROM "${this.schemaName}".workflow_status
2904
- WHERE queue_name = $1 AND status = $4
2905
- AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
2906
- )
3627
+ AND ${noPending('partitions.pk', 4, probeScope)}
2907
3628
  ORDER BY ${sweepOrder}
2908
3629
  LIMIT $5
2909
3630
  )
2910
- SELECT head.workflow_uuid
3631
+ SELECT chosen.pk
2911
3632
  FROM chosen
2912
- -- LATERAL plans as a tight nested loop; a correlated scalar subquery runs as a slower per-row SubPlan.
2913
- JOIN LATERAL (
2914
- SELECT workflow_uuid
2915
- FROM "${this.schemaName}".workflow_status
2916
- WHERE queue_name = $1 AND status = $2
2917
- AND queue_partition_key = chosen.pk
2918
- AND ${versionClause(3)}
2919
- AND ${candidateScope}
2920
- -- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
2921
- ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
2922
- LIMIT 1
2923
- ) head ON TRUE
2924
- -- Which partitions were chosen is settled above; order the result so the claim, and the dispatch it feeds, are deterministic.
3633
+ -- Only partitions with a head this worker may run; LATERAL plans as a tight nested loop.
3634
+ JOIN LATERAL (${headOf('chosen.pk', 3, candidateScope)}) head ON TRUE
3635
+ -- Which partitions were chosen is settled above; order them so the claim, and the dispatch it feeds, are deterministic.
2925
3636
  ORDER BY chosen.pk ASC`, candidateParams);
2926
- const candidateIDs = candidateResult.rows.map((row) => row.workflow_uuid);
2927
- if (candidateIDs.length === 0) {
2928
- await client.query('COMMIT');
2929
- return [];
2930
- }
2931
- await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
3637
+ const partitionKeys = candidateResult.rows.map((row) => row.pk);
2932
3638
  // Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
2933
- const claimGuard = (ids, status, name, version, scope) => `workflow_uuid = ANY($${ids}::text[])
3639
+ const claimGuard = (ids, status, name, version, scope) => `${ids}
2934
3640
  AND status = $${status}
2935
3641
  AND queue_name = $${name}
2936
3642
  AND queue_partition_key IS NOT NULL
2937
3643
  AND ${versionClause(version)}
2938
3644
  AND ${scope}`;
2939
- const lockedParams = [candidateIDs, workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
2940
- const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
2941
- // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
2942
- const lockedResult = await client.query(`SELECT workflow_uuid
2943
- FROM "${this.schemaName}".workflow_status
2944
- WHERE ${claimGuard(1, 2, 3, 4, lockedScope)}
2945
- FOR UPDATE SKIP LOCKED`, lockedParams);
2946
- const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
2947
- // Preserve partition order for submission.
2948
- const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
2949
- if (claimIDs.length === 0) {
2950
- await client.query('COMMIT');
2951
- return [];
3645
+ /** Claim the heads of these partitions inside the open transaction, returning the IDs it marked PENDING. */
3646
+ const claimBatch = async (batch) => {
3647
+ let heldPks = batch;
3648
+ if (usePartitionLocks) {
3649
+ await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_BEFORE_LOCKS);
3650
+ const lockParams = [];
3651
+ const { rows } = await client.query(this.partitionLockQuery(queue.name, batch, lockParams), lockParams);
3652
+ // Partitions another sweep holds are left to it, or to the next poll.
3653
+ heldPks = rows[0]?.held ?? [];
3654
+ if (heldPks.length === 0)
3655
+ return [];
3656
+ }
3657
+ // Pick the heads again under the locks, so a previous holder's claims and any newly visible head are seen.
3658
+ const headParams = [queue.name, workflow_1.StatusString.ENQUEUED, appVersion, workflow_1.StatusString.PENDING];
3659
+ const headScope = `${this.scoped(undefined, headParams)} AND ${this.appNameFilter('application_name', this.appName, headParams)}`;
3660
+ const headProbeScope = this.scoped(undefined, headParams);
3661
+ headParams.push(heldPks);
3662
+ const headResult = await client.query(`SELECT head.workflow_uuid
3663
+ FROM unnest($${headParams.length}::text[]) AS chosen(pk)
3664
+ JOIN LATERAL (${headOf('chosen.pk', 3, headScope)}) head ON TRUE
3665
+ WHERE ${noPending('chosen.pk', 4, headProbeScope)}
3666
+ ORDER BY chosen.pk ASC`, headParams);
3667
+ const candidateIDs = headResult.rows.map((row) => row.workflow_uuid);
3668
+ if (candidateIDs.length === 0)
3669
+ return [];
3670
+ await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
3671
+ const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
3672
+ const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
3673
+ const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
3674
+ // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
3675
+ const lockedResult = await client.query(`SELECT workflow_uuid
3676
+ FROM "${this.schemaName}".workflow_status
3677
+ WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
3678
+ FOR UPDATE SKIP LOCKED`, lockedParams);
3679
+ const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
3680
+ // Preserve partition order for submission.
3681
+ const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
3682
+ if (claimIDs.length === 0)
3683
+ return [];
3684
+ const flipParams = [
3685
+ workflow_1.StatusString.PENDING,
3686
+ executorID,
3687
+ workflow_1.StatusString.ENQUEUED,
3688
+ queue.name,
3689
+ appVersion,
3690
+ // Claim the row, as the unpartitioned dequeue does.
3691
+ this.appName ?? null,
3692
+ ownerXid,
3693
+ startTimeMs,
3694
+ ];
3695
+ const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
3696
+ const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
3697
+ // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
3698
+ const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
3699
+ SET status = $1,
3700
+ owner_xid = $7,
3701
+ executor_id = $2,
3702
+ application_version = $5,
3703
+ started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3704
+ rate_limited = FALSE,
3705
+ application_name = COALESCE(application_name, $6),
3706
+ recovery_attempts = recovery_attempts + 1,
3707
+ updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3708
+ workflow_deadline_epoch_ms = CASE
3709
+ WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
3710
+ CASE
3711
+ WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
3712
+ THEN 9223372036854775807
3713
+ ELSE $8::bigint + workflow_timeout_ms
3714
+ END
3715
+ ELSE workflow_deadline_epoch_ms
3716
+ END
3717
+ WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
3718
+ RETURNING workflow_uuid`, flipParams);
3719
+ const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
3720
+ return claimIDs.filter((id) => flippedIDs.has(id));
3721
+ };
3722
+ // One short transaction per batch, so a sweep holds at most a batch's locks, and only while it claims that batch.
3723
+ const claimedIDs = [];
3724
+ for (let start = 0; start < partitionKeys.length; start += this.partitionedDequeueBatchSize) {
3725
+ const batch = partitionKeys.slice(start, start + this.partitionedDequeueBatchSize);
3726
+ try {
3727
+ await client.query('BEGIN');
3728
+ const batchIDs = await claimBatch(batch);
3729
+ await client.query('COMMIT');
3730
+ claimedIDs.push(...batchIDs);
3731
+ }
3732
+ catch (error) {
3733
+ // Swallowed: a rollback failure must not mask why we are rolling back.
3734
+ await client.query('ROLLBACK').catch(() => undefined);
3735
+ // Earlier batches committed their claims, which must still reach dispatch; the next poll retries the rest.
3736
+ if (claimedIDs.length === 0)
3737
+ throw error;
3738
+ this.logger.warn(`[${queue.name}] stopping a partitioned sweep after ${claimedIDs.length} claim(s): ${String(error)}`);
3739
+ break;
3740
+ }
2952
3741
  }
2953
- const flipParams = [
2954
- workflow_1.StatusString.PENDING,
2955
- executorID,
2956
- claimIDs,
2957
- workflow_1.StatusString.ENQUEUED,
2958
- queue.name,
2959
- appVersion,
2960
- // Claim the row, as the unpartitioned dequeue does.
2961
- this.appName ?? null,
2962
- ownerXid,
2963
- ];
2964
- const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
2965
- // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
2966
- const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
2967
- SET status = $1,
2968
- owner_xid = $8,
2969
- executor_id = $2,
2970
- application_version = $6,
2971
- started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2972
- rate_limited = FALSE,
2973
- application_name = COALESCE(application_name, $7),
2974
- recovery_attempts = recovery_attempts + 1,
2975
- updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2976
- workflow_deadline_epoch_ms = CASE
2977
- WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
2978
- CASE
2979
- WHEN workflow_timeout_ms > 9223372036854775807 - (EXTRACT(epoch FROM now()) * 1000)::bigint
2980
- THEN 9223372036854775807
2981
- ELSE (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
2982
- END
2983
- ELSE workflow_deadline_epoch_ms
2984
- END
2985
- WHERE ${claimGuard(3, 4, 5, 6, flipScope)}
2986
- RETURNING workflow_uuid`, flipParams);
2987
- await client.query('COMMIT');
2988
- const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
2989
- const claimedIDs = claimIDs.filter((id) => flippedIDs.has(id));
2990
3742
  if (claimedIDs.length > 0) {
2991
3743
  this.logger.debug(`[${queue.name}] dequeueing ${claimedIDs.length} task(s)`);
2992
3744
  }
2993
3745
  return claimedIDs;
2994
3746
  }
2995
- catch (error) {
2996
- await client.query('ROLLBACK');
2997
- throw error;
2998
- }
2999
3747
  finally {
3000
3748
  client.release();
3001
3749
  }
3002
3750
  }
3003
3751
  // ==================== Queries & Maintenance ====================
3004
- async listWorkflows(input) {
3005
- return await this.#listWorkflows(input);
3752
+ /** With `keyedIDs`, the listed workflow IDs are addressed by their shard keys, as a batch read of known workflows. */
3753
+ async listWorkflows(input, options = {}) {
3754
+ return await this.#listWorkflows(input, options);
3006
3755
  }
3007
3756
  /**
3008
3757
  * Unretried body of {@link listWorkflows}. An ID-keyed read runs on `client` when given, so a caller
3009
3758
  * holding a transaction never waits on a second pool connection; the caller owns the retry.
3010
3759
  */
3011
- async #listWorkflows(input, client) {
3760
+ async #listWorkflows(input, options = {}, client) {
3012
3761
  const schemaName = this.schemaName;
3013
3762
  const selectColumns = [
3014
3763
  'workflow_uuid',
@@ -3050,16 +3799,23 @@ class SystemDatabase {
3050
3799
  const payloadJoins = [];
3051
3800
  if (input.loadInput) {
3052
3801
  payloadColumns.push('COALESCE(wi.inputs, workflow_status.inputs) AS inputs');
3053
- payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON wi.workflow_uuid = workflow_status.workflow_uuid`);
3802
+ payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON ${this.wfJoin('wi', 'workflow_status')}`);
3054
3803
  }
3055
3804
  if (input.loadOutput) {
3056
3805
  payloadColumns.push('COALESCE(wo.output, workflow_status.output) AS output');
3057
3806
  payloadColumns.push('COALESCE(wo.error, workflow_status.error) AS error');
3058
- payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON wo.workflow_uuid = workflow_status.workflow_uuid`);
3807
+ payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON ${this.wfJoin('wo', 'workflow_status')}`);
3059
3808
  }
3060
3809
  if (input.loadInput || input.loadOutput) {
3061
3810
  selectColumns.push('serialization');
3062
3811
  }
3812
+ const keyColumns = [];
3813
+ const keyColumn = this.keyColumn('workflow_status');
3814
+ if (keyColumn !== undefined)
3815
+ keyColumns.push(`${keyColumn} AS shard_key`);
3816
+ const parentKeyColumn = this.keyColumn('workflow_status', 'parent_shard_key');
3817
+ if (parentKeyColumn !== undefined)
3818
+ keyColumns.push(`${parentKeyColumn} AS parent_shard_key`);
3063
3819
  input.sortDesc = input.sortDesc ?? false; // By default, sort in ascending order
3064
3820
  // Build WHERE clauses
3065
3821
  const whereClauses = [];
@@ -3085,9 +3841,9 @@ class SystemDatabase {
3085
3841
  // If queuesOnly, filter for queued workflows
3086
3842
  if (input.queuesOnly) {
3087
3843
  whereClauses.push(`queue_name IS NOT NULL`);
3088
- whereClauses.push(`status IN ($${paramCounter}, $${paramCounter + 1}, $${paramCounter + 2})`);
3089
- params.push(workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING, workflow_1.StatusString.DELAYED);
3090
- paramCounter += 3;
3844
+ whereClauses.push(`status IN ($${paramCounter}, $${paramCounter + 1}, $${paramCounter + 2}, $${paramCounter + 3})`);
3845
+ params.push(workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING, workflow_1.StatusString.DELAYED, workflow_1.StatusString.SUSPENDED);
3846
+ paramCounter += 4;
3091
3847
  }
3092
3848
  addFilter('name', input.workflowName);
3093
3849
  addFilter('queue_name', input.queueName);
@@ -3102,6 +3858,18 @@ class SystemDatabase {
3102
3858
  whereClauses.push(idKeyed
3103
3859
  ? this.appNameFilter('application_name', input.applicationName, params)
3104
3860
  : this.observabilityFilter('application_name', input.applicationName, params));
3861
+ // An empty key list matches nothing, rather than every key.
3862
+ if (input.shardKey !== undefined && input.shardKey !== null) {
3863
+ const keys = Array.isArray(input.shardKey) ? input.shardKey : [input.shardKey];
3864
+ whereClauses.push(this.keyIn('workflow_status', keys, params));
3865
+ }
3866
+ if (input.parentShardKey !== undefined && input.parentShardKey !== null) {
3867
+ const keys = Array.isArray(input.parentShardKey) ? input.parentShardKey : [input.parentShardKey];
3868
+ whereClauses.push(this.keyIn('workflow_status', keys, params, 'parent_shard_key'));
3869
+ }
3870
+ if (options.keyedIDs && input.workflowIDs) {
3871
+ whereClauses.push(this.wfIn('workflow_status', input.workflowIDs, params));
3872
+ }
3105
3873
  paramCounter = params.length + 1;
3106
3874
  if (input.workflow_id_prefix) {
3107
3875
  if (Array.isArray(input.workflow_id_prefix)) {
@@ -3116,7 +3884,7 @@ class SystemDatabase {
3116
3884
  paramCounter++;
3117
3885
  }
3118
3886
  }
3119
- if (input.workflowIDs) {
3887
+ if (input.workflowIDs && !options.keyedIDs) {
3120
3888
  const placeholders = input.workflowIDs.map((_, i) => `$${paramCounter + i}`).join(', ');
3121
3889
  whereClauses.push(`workflow_status.workflow_uuid IN (${placeholders})`);
3122
3890
  params.push(...input.workflowIDs);
@@ -3198,7 +3966,7 @@ class SystemDatabase {
3198
3966
  params.push(input.offset);
3199
3967
  paramCounter++;
3200
3968
  }
3201
- const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns].join(', ');
3969
+ const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns, ...keyColumns].join(', ');
3202
3970
  const query = `
3203
3971
  SELECT ${projection}
3204
3972
  FROM "${schemaName}".workflow_status
@@ -3218,23 +3986,16 @@ class SystemDatabase {
3218
3986
  async createSchedule(schedule, client) {
3219
3987
  const q = client ?? this.pool;
3220
3988
  const owner = await this.#resolveRowOwner(q, 'workflow_schedules', 'schedule_name', schedule.scheduleName, schedule.applicationName, 'Schedule');
3989
+ const values = {
3990
+ ...this.globalValues(),
3991
+ ...this.scheduleValues(schedule),
3992
+ ...scheduleRow(schedule, owner),
3993
+ };
3221
3994
  try {
3222
- await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules
3223
- (schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name)
3224
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, [
3225
- schedule.scheduleId,
3226
- schedule.scheduleName,
3227
- schedule.workflowName,
3228
- schedule.workflowClassName,
3229
- schedule.schedule,
3230
- schedule.status,
3231
- schedule.context,
3232
- schedule.lastFiredAt,
3233
- schedule.automaticBackfill,
3234
- schedule.cronTimezone,
3235
- schedule.queueName,
3236
- owner ?? null,
3237
- ]);
3995
+ await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
3996
+ VALUES (${Object.keys(values)
3997
+ .map((_, i) => `$${i + 1}`)
3998
+ .join(', ')})`, Object.values(values));
3238
3999
  }
3239
4000
  catch (e) {
3240
4001
  if (isPgDatabaseError(e) && e.code === '23505') {
@@ -3281,31 +4042,37 @@ class SystemDatabase {
3281
4042
  // Unset scopes to this application, as on every other observability query.
3282
4043
  conditions.push(this.observabilityFilter('application_name', filters?.applicationName, params));
3283
4044
  paramIdx = params.length + 1;
4045
+ conditions.push(this.global(undefined, params));
3284
4046
  const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
3285
- const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
4047
+ const result = await q.query(`SELECT ${this.#scheduleColumns()}
3286
4048
  FROM "${this.schemaName}".workflow_schedules${where}
3287
4049
  ORDER BY schedule_name`, params);
3288
4050
  return result.rows.map((row) => mapWorkflowSchedule(row));
3289
4051
  }
3290
4052
  async getSchedule(name, client) {
3291
4053
  const q = client ?? this.pool;
3292
- const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
4054
+ const params = [name];
4055
+ const result = await q.query(`SELECT ${this.#scheduleColumns()}
3293
4056
  FROM "${this.schemaName}".workflow_schedules
3294
- WHERE schedule_name = $1`, [name]);
4057
+ WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
3295
4058
  if (result.rows.length === 0)
3296
4059
  return null;
3297
4060
  return mapWorkflowSchedule(result.rows[0]);
3298
4061
  }
4062
+ /** The columns a schedule is read back with, its workflows' shard key among them on a sharded database. */
4063
+ #scheduleColumns() {
4064
+ const keyColumn = this.scheduleKeyColumn();
4065
+ return keyColumn !== undefined ? `${SCHEDULE_COLUMNS}, ${keyColumn}` : SCHEDULE_COLUMNS;
4066
+ }
3299
4067
  async deleteSchedule(name, client) {
3300
4068
  const q = client ?? this.pool;
3301
- await q.query(`DELETE FROM "${this.schemaName}".workflow_schedules WHERE schedule_name = $1`, [name]);
4069
+ const params = [name];
4070
+ await q.query(`DELETE FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
3302
4071
  }
3303
4072
  async setScheduleStatus(name, status, client) {
3304
4073
  const q = client ?? this.pool;
3305
- await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET status = $1 WHERE schedule_name = $2`, [
3306
- status,
3307
- name,
3308
- ]);
4074
+ const params = [status, name];
4075
+ await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET status = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
3309
4076
  }
3310
4077
  async updateSchedule(name, updates, client) {
3311
4078
  const q = client ?? this.pool;
@@ -3328,35 +4095,46 @@ class SystemDatabase {
3328
4095
  }
3329
4096
  if (setClauses.length === 0) {
3330
4097
  // Nothing to change, but still surface a missing schedule as an error.
3331
- const existing = await q.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules WHERE schedule_name = $1`, [
3332
- name,
3333
- ]);
4098
+ const existingParams = [name];
4099
+ const existing = await q.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, existingParams)} AND schedule_name = $1`, existingParams);
3334
4100
  if (existing.rows.length === 0) {
3335
4101
  throw new error_1.DBOSError(`Schedule '${name}' not found`);
3336
4102
  }
3337
4103
  return;
3338
4104
  }
3339
4105
  params.push(name);
3340
- const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx}`, params);
4106
+ const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx} AND ${this.global(undefined, params)}`, params);
3341
4107
  if (result.rowCount === 0) {
3342
4108
  throw new error_1.DBOSError(`Schedule '${name}' not found`);
3343
4109
  }
3344
4110
  }
3345
4111
  async updateLastFiredAt(name, lastFiredAt) {
3346
- await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE schedule_name = $2`, [lastFiredAt, name]);
4112
+ const params = [lastFiredAt, name];
4113
+ await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
3347
4114
  }
3348
4115
  async applySchedules(schedules) {
3349
4116
  const client = await this.connect();
3350
4117
  try {
3351
4118
  await client.query('BEGIN');
3352
- for (const sched of schedules) {
4119
+ // Sorted, so concurrent writers lock schedule rows in one order.
4120
+ const sorted = [...schedules].sort((a, b) => compareCodeUnits(a.scheduleName, b.scheduleName));
4121
+ for (const sched of sorted) {
3353
4122
  const owner = await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
4123
+ const scheduleKeys = this.scheduleValues(sched);
4124
+ const values = {
4125
+ ...this.globalValues(),
4126
+ ...scheduleKeys,
4127
+ ...scheduleRow(sched, owner),
4128
+ };
3354
4129
  // Upsert on schedule_name; on conflict, preserve schedule_id and runtime state (status, last_fired_at) and update only the declared definition fields, so an unchanged re-apply is a no-op.
3355
- await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules
3356
- (schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name)
3357
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
3358
- ON CONFLICT (schedule_name) DO UPDATE SET
3359
- workflow_name = EXCLUDED.workflow_name,
4130
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
4131
+ VALUES (${Object.keys(values)
4132
+ .map((_, i) => `$${i + 1}`)
4133
+ .join(', ')})
4134
+ ON CONFLICT (${this.keyConflict('schedule_name')}) DO UPDATE SET
4135
+ ${Object.keys(scheduleKeys)
4136
+ .map((c) => `${c} = EXCLUDED.${c}, `)
4137
+ .join('')}workflow_name = EXCLUDED.workflow_name,
3360
4138
  workflow_class_name = EXCLUDED.workflow_class_name,
3361
4139
  schedule = EXCLUDED.schedule,
3362
4140
  context = EXCLUDED.context,
@@ -3364,20 +4142,7 @@ class SystemDatabase {
3364
4142
  cron_timezone = EXCLUDED.cron_timezone,
3365
4143
  queue_name = EXCLUDED.queue_name,
3366
4144
  -- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it took.
3367
- application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, [
3368
- sched.scheduleId,
3369
- sched.scheduleName,
3370
- sched.workflowName,
3371
- sched.workflowClassName,
3372
- sched.schedule,
3373
- sched.status,
3374
- sched.context,
3375
- sched.lastFiredAt,
3376
- sched.automaticBackfill,
3377
- sched.cronTimezone,
3378
- sched.queueName,
3379
- owner ?? null,
3380
- ]);
4145
+ application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, Object.values(values));
3381
4146
  // Read back, since the guard above is silent about why it declined to claim.
3382
4147
  await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
3383
4148
  }
@@ -3402,14 +4167,23 @@ class SystemDatabase {
3402
4167
  try {
3403
4168
  await client.query('BEGIN');
3404
4169
  // Claim a pre-upgrade row in place, so the version is not recreated or retimed.
4170
+ const claimParams = [owner ?? null, versionName];
3405
4171
  const claimed = await client.query(`UPDATE "${this.schemaName}".application_versions
3406
4172
  SET application_name = $1
3407
- WHERE version_name = $2 AND application_name IS NULL`, [owner ?? null, versionName]);
4173
+ WHERE ${this.global(undefined, claimParams)} AND version_name = $2 AND application_name IS NULL`, claimParams);
3408
4174
  if ((claimed.rowCount ?? 0) === 0) {
4175
+ const values = {
4176
+ ...this.globalValues(),
4177
+ version_id: (0, crypto_1.randomUUID)(),
4178
+ version_name: versionName,
4179
+ application_name: owner ?? null,
4180
+ };
3409
4181
  // Targetless DO NOTHING: names no arbiter, so it survives version_name's uniqueness being dropped while still absorbing a concurrent registrar.
3410
- await client.query(`INSERT INTO "${this.schemaName}".application_versions (version_id, version_name, application_name)
3411
- VALUES ($1, $2, $3)
3412
- ON CONFLICT DO NOTHING`, [(0, crypto_1.randomUUID)(), versionName, owner ?? null]);
4182
+ await client.query(`INSERT INTO "${this.schemaName}".application_versions (${Object.keys(values).join(', ')})
4183
+ VALUES (${Object.keys(values)
4184
+ .map((_, i) => `$${i + 1}`)
4185
+ .join(', ')})
4186
+ ON CONFLICT DO NOTHING`, Object.values(values));
3413
4187
  }
3414
4188
  // Read back, since the writes above are silent about why they declined to claim.
3415
4189
  await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
@@ -3440,7 +4214,7 @@ class SystemDatabase {
3440
4214
  params.push(resolved);
3441
4215
  await client.query(`UPDATE "${this.schemaName}".application_versions
3442
4216
  SET version_timestamp = $1, application_name = ${resolved === undefined ? 'application_name' : '$3'}
3443
- WHERE version_name = $2 AND ${scope}`, params);
4217
+ WHERE ${this.global(undefined, params)} AND version_name = $2 AND ${scope}`, params);
3444
4218
  await client.query('COMMIT');
3445
4219
  }
3446
4220
  catch (e) {
@@ -3453,7 +4227,7 @@ class SystemDatabase {
3453
4227
  }
3454
4228
  async listApplicationVersions() {
3455
4229
  const params = [];
3456
- const scope = this.appNameFilter('application_name', this.appName, params);
4230
+ const scope = `${this.global(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
3457
4231
  const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
3458
4232
  FROM "${this.schemaName}".application_versions
3459
4233
  WHERE ${scope}
@@ -3473,9 +4247,10 @@ class SystemDatabase {
3473
4247
  }
3474
4248
  // ==================== Queues ====================
3475
4249
  async getQueue(name) {
4250
+ const params = [name];
3476
4251
  const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
3477
4252
  FROM "${this.schemaName}".queues
3478
- WHERE name = $1`, [name]);
4253
+ WHERE ${this.global(undefined, params)} AND name = $1`, params);
3479
4254
  return rows.length === 0 ? null : queueRecordFromRow(rows[0]);
3480
4255
  }
3481
4256
  /**
@@ -3484,14 +4259,15 @@ class SystemDatabase {
3484
4259
  */
3485
4260
  async listQueues(applicationName) {
3486
4261
  const params = [];
3487
- const scope = this.observabilityFilter('application_name', applicationName, params);
4262
+ const scope = `${this.global(undefined, params)} AND ${this.observabilityFilter('application_name', applicationName, params)}`;
3488
4263
  const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
3489
4264
  FROM "${this.schemaName}".queues
3490
4265
  WHERE ${scope}`, params);
3491
4266
  return rows.map(queueRecordFromRow);
3492
4267
  }
3493
4268
  async deleteQueue(name) {
3494
- await this.pool.query(`DELETE FROM "${this.schemaName}".queues WHERE name = $1`, [name]);
4269
+ const params = [name];
4270
+ await this.pool.query(`DELETE FROM "${this.schemaName}".queues WHERE ${this.global(undefined, params)} AND name = $1`, params);
3495
4271
  }
3496
4272
  async updateQueue(name, fields) {
3497
4273
  const setClauses = [];
@@ -3507,7 +4283,7 @@ class SystemDatabase {
3507
4283
  setClauses.push(`"updated_at" = $${idx++}`);
3508
4284
  params.push(Date.now());
3509
4285
  params.push(name);
3510
- await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx}`, params);
4286
+ await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx} AND ${this.global(undefined, params)}`, params);
3511
4287
  }
3512
4288
  /** Returns true iff this call inserted a new row (i.e. the queue did not
3513
4289
  * previously exist). False if the row already existed, regardless of
@@ -3515,7 +4291,7 @@ class SystemDatabase {
3515
4291
  async upsertQueue(record, updateExisting) {
3516
4292
  const now = Date.now();
3517
4293
  const onConflict = updateExisting
3518
- ? `ON CONFLICT (name) DO UPDATE SET
4294
+ ? `ON CONFLICT (${this.keyConflict('name')}) DO UPDATE SET
3519
4295
  concurrency = EXCLUDED.concurrency,
3520
4296
  worker_concurrency = EXCLUDED.worker_concurrency,
3521
4297
  rate_limit_max = EXCLUDED.rate_limit_max,
@@ -3530,21 +4306,27 @@ class SystemDatabase {
3530
4306
  updated_at = EXCLUDED.updated_at,
3531
4307
  -- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it just took.
3532
4308
  application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
3533
- : `ON CONFLICT (name) DO NOTHING`;
4309
+ : `ON CONFLICT (${this.keyConflict('name')}) DO NOTHING`;
3534
4310
  const owner = record.applicationName ?? this.appName;
3535
4311
  const client = await this.connect();
3536
4312
  try {
3537
4313
  await client.query('BEGIN');
3538
- const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE name = $1`, [record.name]);
4314
+ const existedParams = [record.name];
4315
+ const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE ${this.global(undefined, existedParams)} AND name = $1`, existedParams);
3539
4316
  // A name collision is a conflict in every mode: the name is the queue's address.
3540
4317
  const resolvedOwner = await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
4318
+ const keys = this.globalValues();
3541
4319
  await client.query(`INSERT INTO "${this.schemaName}".queues
3542
- (name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
4320
+ (${Object.keys(keys)
4321
+ .map((c) => `${c}, `)
4322
+ .join('')}name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
3543
4323
  priority_enabled, partition_queue, partition_concurrency, partition_worker_concurrency,
3544
4324
  partition_rate_limit_max, partition_rate_limit_period_sec,
3545
4325
  polling_interval_sec, updated_at, application_name)
3546
4326
  -- priority_enabled is vestigial: every queue dispatches in priority order.
3547
- VALUES ($1, $2, $3, $4, $5, TRUE, $6, $7, $8, $9, $10, $11, $12, $13)
4327
+ VALUES (${Object.keys(keys)
4328
+ .map((_, i) => `$${14 + i}, `)
4329
+ .join('')}$1, $2, $3, $4, $5, TRUE, $6, $7, $8, $9, $10, $11, $12, $13)
3548
4330
  ${onConflict}`, [
3549
4331
  record.name,
3550
4332
  record.concurrency,
@@ -3559,6 +4341,7 @@ class SystemDatabase {
3559
4341
  record.pollingIntervalSec,
3560
4342
  now,
3561
4343
  resolvedOwner ?? null,
4344
+ ...Object.values(keys),
3562
4345
  ]);
3563
4346
  // Read back, since the guard above is silent about why it declined to claim.
3564
4347
  await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
@@ -3668,6 +4451,9 @@ class SystemDatabase {
3668
4451
  params.push(statuses);
3669
4452
  where = `${where} AND status = ANY($${params.length})`;
3670
4453
  }
4454
+ else {
4455
+ where = `${where} AND ${this.global(undefined, params)}`;
4456
+ }
3671
4457
  const res = await client.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${where}`, params);
3672
4458
  return res.rowCount ?? 0;
3673
4459
  };
@@ -3675,7 +4461,12 @@ class SystemDatabase {
3675
4461
  queues = await move('queues');
3676
4462
  schedules = await move('workflow_schedules');
3677
4463
  versions = await move('application_versions');
3678
- inFlight = await move('workflow_status', [workflow_1.StatusString.PENDING, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED]);
4464
+ inFlight = await move('workflow_status', [
4465
+ workflow_1.StatusString.PENDING,
4466
+ workflow_1.StatusString.ENQUEUED,
4467
+ workflow_1.StatusString.DELAYED,
4468
+ workflow_1.StatusString.SUSPENDED,
4469
+ ]);
3679
4470
  await client.query('COMMIT');
3680
4471
  }
3681
4472
  catch (e) {
@@ -3692,84 +4483,64 @@ class SystemDatabase {
3692
4483
  }
3693
4484
  // ==================== Internal ====================
3694
4485
  async insertWorkflowStatus(client, initStatus, creatorXid) {
4486
+ const values = {
4487
+ ...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
4488
+ workflow_uuid: initStatus.workflowUUID,
4489
+ status: initStatus.status,
4490
+ name: initStatus.workflowName,
4491
+ // For cross-language compatibility, these variables MUST be NULL in the database when not set
4492
+ class_name: initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
4493
+ config_name: initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
4494
+ queue_name: initStatus.queueName ?? null,
4495
+ authenticated_user: initStatus.authenticatedUser,
4496
+ assumed_role: initStatus.assumedRole,
4497
+ authenticated_roles: JSON.stringify(initStatus.authenticatedRoles),
4498
+ executor_id: initStatus.executorId,
4499
+ application_version: initStatus.applicationVersion ?? null,
4500
+ application_id: initStatus.applicationID,
4501
+ recovery_attempts: initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
4502
+ workflow_timeout_ms: initStatus.timeoutMS ?? null,
4503
+ workflow_deadline_epoch_ms: initStatus.deadlineEpochMS ?? null,
4504
+ // Legacy column: the payload lives in workflow_input.
4505
+ inputs: null,
4506
+ deduplication_id: initStatus.deduplicationID ?? null,
4507
+ priority: initStatus.priority,
4508
+ queue_partition_key: initStatus.queuePartitionKey ?? null,
4509
+ forked_from: initStatus.forkedFrom ?? null,
4510
+ parent_workflow_id: initStatus.parentWorkflowID ?? null,
4511
+ ...this.parentValues(initStatus.parentWorkflowID),
4512
+ serialization: initStatus.serialization,
4513
+ creator_xid: creatorXid,
4514
+ delay_until_epoch_ms: initStatus.delayUntilEpochMS ?? null,
4515
+ attributes: initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
4516
+ schedule_name: initStatus.scheduleName ?? null,
4517
+ debounce_deadline_epoch_ms: initStatus.debounceDeadlineEpochMS ?? null,
4518
+ is_debounced: initStatus.isDebounced ?? false,
4519
+ application_name: initStatus.applicationName ?? null,
4520
+ // A direct start runs at once, so its creator token also owns the execution.
4521
+ owner_xid: initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
4522
+ };
4523
+ const columns = Object.keys(values);
4524
+ const inputValues = {
4525
+ ...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
4526
+ workflow_uuid: initStatus.workflowUUID,
4527
+ inputs: initStatus.input ?? null,
4528
+ };
4529
+ const inputColumns = Object.keys(inputValues);
3695
4530
  try {
3696
- const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
3697
- workflow_uuid,
3698
- status,
3699
- name,
3700
- class_name,
3701
- config_name,
3702
- queue_name,
3703
- authenticated_user,
3704
- assumed_role,
3705
- authenticated_roles,
3706
- executor_id,
3707
- application_version,
3708
- application_id,
3709
- recovery_attempts,
3710
- workflow_timeout_ms,
3711
- workflow_deadline_epoch_ms,
3712
- inputs,
3713
- deduplication_id,
3714
- priority,
3715
- queue_partition_key,
3716
- forked_from,
3717
- parent_workflow_id,
3718
- serialization,
3719
- creator_xid,
3720
- delay_until_epoch_ms,
3721
- attributes,
3722
- schedule_name,
3723
- debounce_deadline_epoch_ms,
3724
- is_debounced,
3725
- application_name,
3726
- owner_xid
3727
- ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)
3728
- ON CONFLICT (workflow_uuid)
4531
+ const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
4532
+ VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
4533
+ ON CONFLICT (${this.keyConflict('workflow_uuid')})
3729
4534
  -- A no-op update, so an existing row comes back unchanged for the caller to inspect.
3730
4535
  DO UPDATE SET creator_xid = workflow_status.creator_xid
3731
- RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`, [
3732
- initStatus.workflowUUID,
3733
- initStatus.status,
3734
- initStatus.workflowName,
3735
- // For cross-language compatibility, these variables MUST be NULL in the database when not set
3736
- initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
3737
- initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
3738
- initStatus.queueName ?? null,
3739
- initStatus.authenticatedUser,
3740
- initStatus.assumedRole,
3741
- JSON.stringify(initStatus.authenticatedRoles),
3742
- initStatus.executorId,
3743
- initStatus.applicationVersion ?? null,
3744
- initStatus.applicationID,
3745
- initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
3746
- initStatus.timeoutMS ?? null,
3747
- initStatus.deadlineEpochMS ?? null,
3748
- // Legacy column: the payload lives in workflow_input.
3749
- null,
3750
- initStatus.deduplicationID ?? null,
3751
- initStatus.priority,
3752
- initStatus.queuePartitionKey ?? null,
3753
- initStatus.forkedFrom ?? null,
3754
- initStatus.parentWorkflowID ?? null,
3755
- initStatus.serialization,
3756
- creatorXid,
3757
- initStatus.delayUntilEpochMS ?? null,
3758
- initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
3759
- initStatus.scheduleName ?? null,
3760
- initStatus.debounceDeadlineEpochMS ?? null,
3761
- initStatus.isDebounced ?? false,
3762
- initStatus.applicationName ?? null,
3763
- // A direct start runs at once, so its creator token also owns the execution.
3764
- initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
3765
- ]);
4536
+ RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`, Object.values(values));
3766
4537
  if (rows.length === 0) {
3767
4538
  throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
3768
4539
  }
3769
4540
  // Two statements, not a data-modifying CTE: at scale the CTE costs more than the round trip it saves.
3770
- await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs)
3771
- VALUES ($1, $2)
3772
- ON CONFLICT (workflow_uuid) DO NOTHING`, [initStatus.workflowUUID, initStatus.input ?? null]);
4541
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_input (${inputColumns.join(', ')})
4542
+ VALUES (${inputColumns.map((_, i) => `$${i + 1}`).join(', ')})
4543
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`, Object.values(inputValues));
3773
4544
  const ret = rows[0];
3774
4545
  ret.class_name = ret.class_name ?? '';
3775
4546
  ret.config_name = ret.config_name ?? '';
@@ -3785,16 +4556,17 @@ class SystemDatabase {
3785
4556
  }
3786
4557
  }
3787
4558
  async getWorkflowStatusValue(client, workflowID) {
3788
- const { rows } = await client.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE workflow_uuid=$1`, [workflowID]);
4559
+ const params = [];
4560
+ const { rows } = await client.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
3789
4561
  return rows.length === 0 ? undefined : rows[0].status;
3790
4562
  }
3791
4563
  async updateWorkflowStatus(client, workflowID, status, options = {}) {
3792
4564
  // Use SQL now() so updated_at and completed_at (when set together) are
3793
4565
  // computed in the same statement against the same clock.
3794
4566
  const nowMsExpr = `(EXTRACT(EPOCH FROM now()) * 1000)::bigint`;
3795
- let setClause = `SET status=$2, updated_at=${nowMsExpr}`;
3796
- let whereClause = `WHERE workflow_uuid=$1`;
3797
- const args = [workflowID, status];
4567
+ let setClause = `SET status=$1, updated_at=${nowMsExpr}`;
4568
+ const args = [status];
4569
+ let whereClause = `WHERE ${this.wf(undefined, workflowID, args)}`;
3798
4570
  const update = options.update ?? {};
3799
4571
  if (update.resetRecoveryAttempts) {
3800
4572
  setClause += `, recovery_attempts = 0`;
@@ -3853,27 +4625,31 @@ class SystemDatabase {
3853
4625
  async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
3854
4626
  const ownerXid = options.ownerChecked ? undefined : (0, context_1.currentOwnerXid)(workflowID);
3855
4627
  if (ownerXid !== undefined) {
3856
- await this.#checkOwner(client, workflowID, ownerXid);
3857
- }
4628
+ await this.checkOwner(client, workflowID, ownerXid);
4629
+ }
4630
+ const values = {
4631
+ ...this.wfValues(workflowID),
4632
+ workflow_uuid: workflowID,
4633
+ function_id: functionID,
4634
+ output: options.output ?? null,
4635
+ error: options.error ?? null,
4636
+ function_name: functionName,
4637
+ child_workflow_id: options.childWorkflowID ?? null,
4638
+ ...(options.childValues ?? this.childValues(options.childWorkflowID)),
4639
+ started_at_epoch_ms: startTimeEpochMs,
4640
+ completed_at_epoch_ms: endTimeEpochMs,
4641
+ serialization: options.serialization ?? null,
4642
+ // Mirrors the parent: only the running application records its steps.
4643
+ application_name: this.appName ?? null,
4644
+ };
4645
+ const columns = Object.keys(values);
3858
4646
  try {
3859
4647
  const out = await client.query(`INSERT INTO "${this.schemaName}".operation_outputs
3860
- (workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization, application_name, retention_timestamp)
3861
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
3862
- ON CONFLICT (workflow_uuid, function_id) DO UPDATE
4648
+ (${columns.join(', ')}, retention_timestamp)
4649
+ VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')}, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
4650
+ ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id')}) DO UPDATE
3863
4651
  SET completed_at_epoch_ms = operation_outputs.completed_at_epoch_ms
3864
- RETURNING completed_at_epoch_ms;`, [
3865
- workflowID,
3866
- functionID,
3867
- options.output ?? null,
3868
- options.error ?? null,
3869
- functionName,
3870
- options.childWorkflowID ?? null,
3871
- startTimeEpochMs,
3872
- endTimeEpochMs,
3873
- options.serialization ?? null,
3874
- // Mirrors the parent: only the running application records its steps.
3875
- this.appName ?? null,
3876
- ]);
4652
+ RETURNING completed_at_epoch_ms;`, Object.values(values));
3877
4653
  // A row already there with another completion time is this execution's own doing
3878
4654
  // (a retry carries the same time), so it is nondeterminism, not a duplicate.
3879
4655
  if (checkConflict &&
@@ -3890,7 +4666,7 @@ class SystemDatabase {
3890
4666
  }
3891
4667
  }
3892
4668
  /** Run `fn` in a READ COMMITTED transaction on `client`, so an ownership check holds its row lock until the write commits. */
3893
- async #inTransaction(client, fn) {
4669
+ async inTransaction(client, fn) {
3894
4670
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
3895
4671
  let result;
3896
4672
  try {
@@ -3905,22 +4681,26 @@ class SystemDatabase {
3905
4681
  return result;
3906
4682
  }
3907
4683
  /** Throw DBOSWorkflowConflictError unless ownerXid still owns the workflow; the row lock holds until commit, so no hand-off lands in between. */
3908
- async #checkOwner(client, workflowID, ownerXid) {
3909
- const { rows } = await client.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = $1 FOR NO KEY UPDATE`, [workflowID]);
4684
+ async checkOwner(client, workflowID, ownerXid) {
4685
+ const params = [];
4686
+ const { rows } = await client.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)} FOR NO KEY UPDATE`, params);
3910
4687
  if (rows[0]?.owner_xid !== ownerXid) {
3911
4688
  throw new error_1.DBOSWorkflowConflictError(workflowID);
3912
4689
  }
3913
4690
  }
3914
4691
  /** The workflow's current ownership token; null if unowned or missing. */
3915
4692
  async getWorkflowOwner(workflowID) {
3916
- const { rows } = await this.pool.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = $1`, [workflowID]);
4693
+ const params = [];
4694
+ const { rows } = await this.pool.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
3917
4695
  return rows[0]?.owner_xid ?? null;
3918
4696
  }
3919
4697
  async #getOperationResultAndThrowIfCancelled(client, workflowID, functionID) {
3920
4698
  await this.#checkIfCanceled(client, workflowID);
3921
- const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name, serialization
4699
+ const childKey = this.keyColumn(undefined, 'child_shard_key');
4700
+ const params = [functionID];
4701
+ const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name, serialization${childKey !== undefined ? `, ${childKey}` : ''}
3922
4702
  FROM "${this.schemaName}".operation_outputs
3923
- WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
4703
+ WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
3924
4704
  if (rows.length === 0) {
3925
4705
  return undefined;
3926
4706
  }
@@ -3929,6 +4709,7 @@ class SystemDatabase {
3929
4709
  output: rows[0].output,
3930
4710
  error: rows[0].error,
3931
4711
  childWorkflowID: rows[0].child_workflow_id,
4712
+ childShardKey: rows[0].child_shard_key ?? undefined,
3932
4713
  functionName: rows[0].function_name,
3933
4714
  // Return serialization so recv/getEvent replay deserializes with the stored format, not the default.
3934
4715
  serialization: rows[0].serialization,
@@ -3966,7 +4747,11 @@ class SystemDatabase {
3966
4747
  // been cancelled.
3967
4748
  // For an actual sleep, completed_at is the wake deadline so the step's duration reflects the
3968
4749
  // sleep; a timeout marker records zero duration since its deadline may never be reached.
3969
- async #durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline = false) {
4750
+ async durableSleepInternal(workflowID, functionID, durationMS, recordCompletionAtDeadline = false) {
4751
+ // The caller's own workflow, even where a same-ID workflow is bound as a target.
4752
+ return await this.ownKey(workflowID, undefined, () => this.#durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline));
4753
+ }
4754
+ async #durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline) {
3970
4755
  const startTimeMs = Date.now();
3971
4756
  // Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
3972
4757
  const endTimeMs = startTimeMs + Math.ceil(durationMS);
@@ -3979,7 +4764,7 @@ class SystemDatabase {
3979
4764
  }
3980
4765
  return JSON.parse(res.output);
3981
4766
  }
3982
- await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
4767
+ await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
3983
4768
  output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
3984
4769
  serialization: serialization_1.DBOSPortableJSON.name(),
3985
4770
  }));
@@ -4132,6 +4917,12 @@ __decorate([
4132
4917
  __metadata("design:paramtypes", [Array, Number]),
4133
4918
  __metadata("design:returntype", Promise)
4134
4919
  ], SystemDatabase.prototype, "deadLetterWorkflows", null);
4920
+ __decorate([
4921
+ dbRetry(),
4922
+ __metadata("design:type", Function),
4923
+ __metadata("design:paramtypes", [Array, Map]),
4924
+ __metadata("design:returntype", Promise)
4925
+ ], SystemDatabase.prototype, "enqueueScheduledWorkflows", null);
4135
4926
  __decorate([
4136
4927
  dbRetry(),
4137
4928
  __metadata("design:type", Function),
@@ -4159,7 +4950,7 @@ __decorate([
4159
4950
  __decorate([
4160
4951
  dbRetry(),
4161
4952
  __metadata("design:type", Function),
4162
- __metadata("design:paramtypes", [String, Number]),
4953
+ __metadata("design:paramtypes", [String, Number, String]),
4163
4954
  __metadata("design:returntype", Promise)
4164
4955
  ], SystemDatabase.prototype, "getOperationResultAndThrowIfCancelled", null);
4165
4956
  __decorate([
@@ -4219,9 +5010,9 @@ __decorate([
4219
5010
  __decorate([
4220
5011
  dbRetry(),
4221
5012
  __metadata("design:type", Function),
4222
- __metadata("design:paramtypes", [Array, Boolean]),
5013
+ __metadata("design:paramtypes", [Array]),
4223
5014
  __metadata("design:returntype", Promise)
4224
- ], SystemDatabase.prototype, "deleteWorkflows", null);
5015
+ ], SystemDatabase.prototype, "deleteWorkflowsInternal", null);
4225
5016
  __decorate([
4226
5017
  dbRetry()
4227
5018
  // A missing row normally means the workflow has not been inserted yet, so
@@ -4255,19 +5046,19 @@ __decorate([
4255
5046
  __decorate([
4256
5047
  dbRetry(),
4257
5048
  __metadata("design:type", Function),
4258
- __metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String]),
5049
+ __metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String, String]),
4259
5050
  __metadata("design:returntype", Promise)
4260
5051
  ], SystemDatabase.prototype, "send", null);
4261
5052
  __decorate([
4262
5053
  dbRetry(),
4263
5054
  __metadata("design:type", Function),
4264
- __metadata("design:paramtypes", [String, String, Object, Object, Object, String]),
5055
+ __metadata("design:paramtypes", [String, String, Object, Object, Object, String, String]),
4265
5056
  __metadata("design:returntype", Promise)
4266
5057
  ], SystemDatabase.prototype, "sendFromStep", null);
4267
5058
  __decorate([
4268
5059
  dbRetry(),
4269
5060
  __metadata("design:type", Function),
4270
- __metadata("design:paramtypes", [String, Object, Object, Object, String]),
5061
+ __metadata("design:paramtypes", [String, Object, Object, Object, String, String]),
4271
5062
  __metadata("design:returntype", Promise)
4272
5063
  ], SystemDatabase.prototype, "sendDirectStandalone", null);
4273
5064
  __decorate([
@@ -4276,6 +5067,12 @@ __decorate([
4276
5067
  __metadata("design:paramtypes", [String, Number, Number, String, Number, Number]),
4277
5068
  __metadata("design:returntype", Promise)
4278
5069
  ], SystemDatabase.prototype, "recv", null);
5070
+ __decorate([
5071
+ dbRetry(),
5072
+ __metadata("design:type", Function),
5073
+ __metadata("design:paramtypes", [String, Number, Object, Number]),
5074
+ __metadata("design:returntype", Promise)
5075
+ ], SystemDatabase.prototype, "recvConsume", null);
4279
5076
  __decorate([
4280
5077
  dbRetry(),
4281
5078
  __metadata("design:type", Function),
@@ -4321,7 +5118,7 @@ __decorate([
4321
5118
  __decorate([
4322
5119
  dbRetry(),
4323
5120
  __metadata("design:type", Function),
4324
- __metadata("design:paramtypes", [Object]),
5121
+ __metadata("design:paramtypes", [Object, Object]),
4325
5122
  __metadata("design:returntype", Promise)
4326
5123
  ], SystemDatabase.prototype, "listWorkflows", null);
4327
5124
  __decorate([
@@ -4342,4 +5139,10 @@ __decorate([
4342
5139
  __metadata("design:paramtypes", [String]),
4343
5140
  __metadata("design:returntype", Promise)
4344
5141
  ], SystemDatabase.prototype, "getWorkflowOwner", null);
5142
+ __decorate([
5143
+ dbRetry(),
5144
+ __metadata("design:type", Function),
5145
+ __metadata("design:paramtypes", [String, Number, Number, Boolean]),
5146
+ __metadata("design:returntype", Promise)
5147
+ ], SystemDatabase.prototype, "durableSleepInternal", null);
4345
5148
  //# sourceMappingURL=system_database.js.map