@dbos-inc/dbos-sdk 5.3.13-preview → 5.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/dist/src/auto_upgrade.d.ts +27 -0
  2. package/dist/src/auto_upgrade.d.ts.map +1 -0
  3. package/dist/src/auto_upgrade.js +84 -0
  4. package/dist/src/auto_upgrade.js.map +1 -0
  5. package/dist/src/cli/cli.js +56 -13
  6. package/dist/src/cli/cli.js.map +1 -1
  7. package/dist/src/client.d.ts +20 -10
  8. package/dist/src/client.d.ts.map +1 -1
  9. package/dist/src/client.js +76 -44
  10. package/dist/src/client.js.map +1 -1
  11. package/dist/src/config.d.ts +1 -0
  12. package/dist/src/config.d.ts.map +1 -1
  13. package/dist/src/config.js +12 -0
  14. package/dist/src/config.js.map +1 -1
  15. package/dist/src/context.d.ts +6 -0
  16. package/dist/src/context.d.ts.map +1 -1
  17. package/dist/src/context.js +15 -1
  18. package/dist/src/context.js.map +1 -1
  19. package/dist/src/datasource.d.ts +2 -0
  20. package/dist/src/datasource.d.ts.map +1 -1
  21. package/dist/src/datasource.js +7 -1
  22. package/dist/src/datasource.js.map +1 -1
  23. package/dist/src/dbos-executor.d.ts +39 -4
  24. package/dist/src/dbos-executor.d.ts.map +1 -1
  25. package/dist/src/dbos-executor.js +86 -55
  26. package/dist/src/dbos-executor.js.map +1 -1
  27. package/dist/src/dbos.d.ts +60 -11
  28. package/dist/src/dbos.d.ts.map +1 -1
  29. package/dist/src/dbos.js +230 -70
  30. package/dist/src/dbos.js.map +1 -1
  31. package/dist/src/enqueue_options.d.ts +2 -0
  32. package/dist/src/enqueue_options.d.ts.map +1 -1
  33. package/dist/src/enqueue_options.js +3 -0
  34. package/dist/src/enqueue_options.js.map +1 -1
  35. package/dist/src/enqueue_workflow.d.ts.map +1 -1
  36. package/dist/src/enqueue_workflow.js +11 -5
  37. package/dist/src/enqueue_workflow.js.map +1 -1
  38. package/dist/src/enterprise.d.ts +67 -2
  39. package/dist/src/enterprise.d.ts.map +1 -1
  40. package/dist/src/enterprise.js +185 -9
  41. package/dist/src/enterprise.js.map +1 -1
  42. package/dist/src/eventreceiver.d.ts +2 -0
  43. package/dist/src/eventreceiver.d.ts.map +1 -1
  44. package/dist/src/eventreceiver.js +7 -1
  45. package/dist/src/eventreceiver.js.map +1 -1
  46. package/dist/src/index.d.ts +2 -1
  47. package/dist/src/index.d.ts.map +1 -1
  48. package/dist/src/index.js.map +1 -1
  49. package/dist/src/scheduler/scheduler.d.ts +5 -0
  50. package/dist/src/scheduler/scheduler.d.ts.map +1 -1
  51. package/dist/src/scheduler/scheduler.js +14 -4
  52. package/dist/src/scheduler/scheduler.js.map +1 -1
  53. package/dist/src/shard_keys.d.ts +13 -0
  54. package/dist/src/shard_keys.d.ts.map +1 -0
  55. package/dist/src/shard_keys.js +55 -0
  56. package/dist/src/shard_keys.js.map +1 -0
  57. package/dist/src/streams.d.ts +2 -0
  58. package/dist/src/streams.d.ts.map +1 -1
  59. package/dist/src/streams.js +5 -2
  60. package/dist/src/streams.js.map +1 -1
  61. package/dist/src/sysdb_migrations/internal/migrations.js +16 -0
  62. package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
  63. package/dist/src/system_database.d.ts +143 -9
  64. package/dist/src/system_database.d.ts.map +1 -1
  65. package/dist/src/system_database.js +1134 -519
  66. package/dist/src/system_database.js.map +1 -1
  67. package/dist/src/wfqueue.d.ts +1 -2
  68. package/dist/src/wfqueue.d.ts.map +1 -1
  69. package/dist/src/wfqueue.js +83 -63
  70. package/dist/src/wfqueue.js.map +1 -1
  71. package/dist/src/workflow.d.ts +19 -2
  72. package/dist/src/workflow.d.ts.map +1 -1
  73. package/dist/src/workflow.js +14 -9
  74. package/dist/src/workflow.js.map +1 -1
  75. package/dist/src/workflow_management.d.ts +6 -3
  76. package/dist/src/workflow_management.d.ts.map +1 -1
  77. package/dist/src/workflow_management.js +46 -14
  78. package/dist/src/workflow_management.js.map +1 -1
  79. package/dist/tsconfig.tsbuildinfo +1 -1
  80. package/package.json +6 -5
@@ -1,15 +1,38 @@
1
1
  "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
2
18
  var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
19
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
20
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
21
  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
22
  return c > 3 && r && Object.defineProperty(target, key, r), r;
7
23
  };
24
+ var __importStar = (this && this.__importStar) || function (mod) {
25
+ if (mod && mod.__esModule) return mod;
26
+ var result = {};
27
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
28
+ __setModuleDefault(result, mod);
29
+ return result;
30
+ };
8
31
  var __metadata = (this && this.__metadata) || function (k, v) {
9
32
  if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
33
  };
11
34
  Object.defineProperty(exports, "__esModule", { value: true });
12
- exports.SystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
35
+ exports.SystemDatabase = exports.createSystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.qualify = exports.holdsShardedSchema = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
13
36
  const dbos_executor_1 = require("./dbos-executor");
14
37
  const pg_1 = require("pg");
15
38
  const error_1 = require("./error");
@@ -18,6 +41,8 @@ const utils_1 = require("./utils");
18
41
  const context_1 = require("./context");
19
42
  const async_hooks_1 = require("async_hooks");
20
43
  const crypto_1 = require("crypto");
44
+ const shard_keys_1 = require("./shard_keys");
45
+ const enterprise = __importStar(require("./enterprise"));
21
46
  const utils_2 = require("./utils");
22
47
  const database_utils_1 = require("./database_utils");
23
48
  const migration_runner_1 = require("./sysdb_migrations/migration_runner");
@@ -70,6 +95,8 @@ function validateObservabilityQueryTimeoutMs(value) {
70
95
  exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutMs;
71
96
  // Bounds how long a frozen or unreachable client can hold system database locks.
72
97
  exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = 60_000;
98
+ // How long a background job's lock may sit idle before Postgres ends its holder's session.
99
+ exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = 5000;
73
100
  /** The timeout is rendered into a `SET idle_in_transaction_session_timeout`, which takes only an in-range integer. */
74
101
  function validateIdleTransactionTimeoutMs(value) {
75
102
  if (value !== undefined && (!Number.isFinite(value) || value > MAX_STATEMENT_TIMEOUT_MS)) {
@@ -157,6 +184,7 @@ async function grantDbosSchemaPermissions(databaseUrl, roleName, logger, schemaN
157
184
  await client.end();
158
185
  }
159
186
  }
187
+ exports.grantDbosSchemaPermissions = grantDbosSchemaPermissions;
160
188
  /**
161
189
  * Check out a connection carrying our own error handler, so a socket death while we hold it is
162
190
  * logged instead of thrown at an emitter with no listener. `release()` takes the handler back off,
@@ -219,6 +247,7 @@ async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = '
219
247
  }
220
248
  const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
221
249
  try {
250
+ await refuseShardedSchema(client, schemaName);
222
251
  const isCockroach = await isCockroachDB(client);
223
252
  await (0, migration_runner_1.runSysMigrationsPg)(client, (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }), schemaName, {
224
253
  onWarn: (e) => logger.info(e),
@@ -242,6 +271,7 @@ exports.migrateSystemDatabase = migrateSystemDatabase;
242
271
  async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
243
272
  const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
244
273
  try {
274
+ await refuseShardedSchema(client, schemaName);
245
275
  const isCockroach = await isCockroachDB(client);
246
276
  const requiredVersion = (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }).length;
247
277
  const currentVersion = await (0, migration_runner_1.getCurrentSysDBVersion)(client, schemaName);
@@ -259,6 +289,41 @@ async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = '
259
289
  }
260
290
  }
261
291
  exports.verifySystemDatabase = verifySystemDatabase;
292
+ /** Throw if this schema is sharded: an unsharded handle's statements would span every tenant's rows. */
293
+ async function refuseShardedSchema(client, schemaName) {
294
+ const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
295
+ WHERE table_schema = $1 AND table_name = 'workflow_status' AND column_name = 'shard_key'`, [schemaName]);
296
+ if (rows.length > 0) {
297
+ throw new error_1.DBOSError(`The DBOS system schema ${schemaName} is sharded. Enable sharding to use it: ` +
298
+ 'set sharding in your DBOS configuration, or pass sharding: true to DBOSClient.');
299
+ }
300
+ }
301
+ /** Whether this database holds a sharded DBOS system schema (this one, if named); false if it can't be reached. */
302
+ async function holdsShardedSchema(databaseUrl, schemaName) {
303
+ let client;
304
+ try {
305
+ // Bounded, so an unreachable host fails as fast as the caller's own connection would.
306
+ client = new pg_1.Client({ ...(0, utils_2.getClientConfig)(databaseUrl), connectionTimeoutMillis: 10_000 });
307
+ }
308
+ catch {
309
+ // A malformed URL, for the caller's own connection to report.
310
+ return false;
311
+ }
312
+ client.on('error', () => { });
313
+ try {
314
+ await client.connect();
315
+ const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
316
+ WHERE table_name = 'workflow_status' AND column_name = 'shard_key'${schemaName !== undefined ? ' AND table_schema = $1' : ''} LIMIT 1`, schemaName !== undefined ? [schemaName] : []);
317
+ return rows.length > 0;
318
+ }
319
+ catch {
320
+ return false;
321
+ }
322
+ finally {
323
+ await client.end().catch(() => { });
324
+ }
325
+ }
326
+ exports.holdsShardedSchema = holdsShardedSchema;
262
327
  class NotificationMap {
263
328
  map = new Map();
264
329
  curCK = 0;
@@ -300,6 +365,23 @@ function mapVersionInfo(row) {
300
365
  };
301
366
  }
302
367
  const SCHEDULE_COLUMNS = 'schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name';
368
+ /** A schedule's own columns, as a new row stores them. */
369
+ function scheduleRow(schedule, owner) {
370
+ return {
371
+ schedule_id: schedule.scheduleId,
372
+ schedule_name: schedule.scheduleName,
373
+ workflow_name: schedule.workflowName,
374
+ workflow_class_name: schedule.workflowClassName,
375
+ schedule: schedule.schedule,
376
+ status: schedule.status,
377
+ context: schedule.context,
378
+ last_fired_at: schedule.lastFiredAt,
379
+ automatic_backfill: schedule.automaticBackfill,
380
+ cron_timezone: schedule.cronTimezone,
381
+ queue_name: schedule.queueName,
382
+ application_name: owner ?? null,
383
+ };
384
+ }
303
385
  function mapWorkflowSchedule(row) {
304
386
  return {
305
387
  scheduleId: row.schedule_id,
@@ -314,6 +396,7 @@ function mapWorkflowSchedule(row) {
314
396
  cronTimezone: row.cron_timezone ?? null,
315
397
  queueName: row.queue_name ?? null,
316
398
  applicationName: row.application_name ?? undefined,
399
+ shardKey: row.workflow_shard_key ?? undefined,
317
400
  };
318
401
  }
319
402
  function mapWorkflowStatus(row) {
@@ -337,7 +420,10 @@ function mapWorkflowStatus(row) {
337
420
  recoveryAttempts: Number(row.recovery_attempts),
338
421
  input: row.inputs ? row.inputs : null,
339
422
  timeoutMS: row.workflow_timeout_ms ? Number(row.workflow_timeout_ms) : undefined,
340
- deadlineEpochMS: row.workflow_deadline_epoch_ms ? Number(row.workflow_deadline_epoch_ms) : undefined,
423
+ // Cap at the largest exact JS integer: a saturated bigint deadline would otherwise round past the bigint maximum and be rejected when a child workflow inherits it.
424
+ deadlineEpochMS: row.workflow_deadline_epoch_ms
425
+ ? Math.min(Number(row.workflow_deadline_epoch_ms), Number.MAX_SAFE_INTEGER)
426
+ : undefined,
341
427
  deduplicationID: row.deduplication_id ?? undefined,
342
428
  priority: row.priority ?? 0,
343
429
  queuePartitionKey: row.queue_partition_key ?? undefined,
@@ -353,8 +439,16 @@ function mapWorkflowStatus(row) {
353
439
  debounceDeadlineEpochMS: row.debounce_deadline_epoch_ms ? Number(row.debounce_deadline_epoch_ms) : undefined,
354
440
  isDebounced: row.is_debounced ?? false,
355
441
  applicationName: row.application_name ?? undefined,
442
+ // Read back only from a sharded system database, whose rows carry them.
443
+ shardKey: row.shard_key ?? undefined,
444
+ parentShardKey: row.parent_shard_key ?? undefined,
356
445
  };
357
446
  }
447
+ /** `column`, qualified by `alias` if one is given. */
448
+ function qualify(alias, column) {
449
+ return alias ? `${alias}.${column}` : column;
450
+ }
451
+ exports.qualify = qualify;
358
452
  // SQLSTATE classes/codes that are generally safe to retry
359
453
  // https://www.postgresql.org/docs/current/errcodes-appendix.html
360
454
  const RETRY_SQLSTATE_PREFIXES = new Set([
@@ -524,6 +618,26 @@ function dbRetry(options = {}) {
524
618
  return descriptor;
525
619
  };
526
620
  }
621
+ /** Build the system database these options describe. */
622
+ function createSystemDatabase(options) {
623
+ if (options.sharding) {
624
+ const { ShardedSystemDatabase } = enterprise.load(enterprise.SHARDING.description);
625
+ return new ShardedSystemDatabase({
626
+ systemDatabaseUrl: options.systemDatabaseUrl,
627
+ logger: options.logger,
628
+ serializer: options.serializer,
629
+ sysDbPoolSize: options.sysDbPoolSize,
630
+ systemDatabasePool: options.systemDatabasePool,
631
+ schemaName: options.schemaName,
632
+ pollingConcurrency: options.pollingConcurrency,
633
+ appName: options.appName,
634
+ observabilityQueryTimeoutMs: options.observabilityQueryTimeoutMs,
635
+ idleTransactionTimeoutMs: options.idleTransactionTimeoutMs,
636
+ });
637
+ }
638
+ return new SystemDatabase(options.systemDatabaseUrl, options.logger, options.serializer, options.sysDbPoolSize, options.systemDatabasePool, options.schemaName, options.useListenNotify, options.pollingConcurrency, options.notificationCoalesceMs, options.appName, options.observabilityQueryTimeoutMs, options.idleTransactionTimeoutMs);
639
+ }
640
+ exports.createSystemDatabase = createSystemDatabase;
527
641
  /**
528
642
  * General notes:
529
643
  * The responsibilities of the `SystemDatabase` are to store data for workflows, and
@@ -592,6 +706,7 @@ class SystemDatabase {
592
706
  pollLimiter;
593
707
  /** Executions running in this process, by workflow ID; a re-dispatch runs alongside its stale execution, which parks at its next write. */
594
708
  runningWorkflowMap = new Map();
709
+ // Keyed by waiter key, as one ID can live under several shard keys.
595
710
  #cancelWatchers = new Map();
596
711
  #cancelPollerLoop = undefined;
597
712
  #cancelPollerWake = null;
@@ -724,25 +839,222 @@ class SystemDatabase {
724
839
  observabilityFilter(column, value, params) {
725
840
  return this.appNameFilter(column, value ?? this.appName, params);
726
841
  }
842
+ // ==================== Shard Keys ====================
843
+ // Statements locate rows through these, so a sharded subclass adds its key; each appends to `params` and returns SQL.
844
+ /** Whether this handle routes every statement by shard key. */
845
+ get sharded() {
846
+ return false;
847
+ }
848
+ #rejectShardKey(workflowID, shardKey) {
849
+ if ((shardKey !== undefined && shardKey !== null) || (0, shard_keys_1.boundShardKey)(workflowID) !== undefined) {
850
+ throw new error_1.DBOSError('A shard key was given, but this system database is not sharded');
851
+ }
852
+ }
853
+ /** Predicate selecting one workflow's rows: those whose `column` is `workflowID`. */
854
+ wf(alias, workflowID, params, column = 'workflow_uuid', shardKey) {
855
+ this.#rejectShardKey(workflowID, shardKey);
856
+ params.push(workflowID);
857
+ return `${qualify(alias, column)} = $${params.length}`;
858
+ }
859
+ /** Predicate selecting several workflows' rows. */
860
+ wfIn(alias, workflowIDs, params) {
861
+ for (const workflowID of workflowIDs) {
862
+ this.#rejectShardKey(workflowID, undefined);
863
+ }
864
+ params.push(workflowIDs);
865
+ return `${qualify(alias, 'workflow_uuid')} = ANY($${params.length}::text[])`;
866
+ }
867
+ /** Condition joining two tables' rows of the same workflow. */
868
+ wfJoin(left, right) {
869
+ return `${left}.workflow_uuid = ${right}.workflow_uuid`;
870
+ }
871
+ /** Extra column values for a new row of this workflow. */
872
+ wfValues(workflowID, shardKey) {
873
+ this.#rejectShardKey(workflowID, shardKey);
874
+ return {};
875
+ }
876
+ /** Extra column values storing the shard key of a new workflow's parent. */
877
+ parentValues(_parentWorkflowID) {
878
+ return {};
879
+ }
880
+ /** Extra column values storing the shard key of the workflow a step refers to. */
881
+ childValues(_childWorkflowID) {
882
+ return {};
883
+ }
884
+ /** Extra (column, expression) pairs for an INSERT ... SELECT of this workflow's rows. */
885
+ wfColumns(_workflowID, _params) {
886
+ return [];
887
+ }
888
+ /** Conflict target for a unique key of a workflow table. */
889
+ keyConflict(...columns) {
890
+ return columns.join(', ');
891
+ }
892
+ /** Insert rows into `table` in one statement, returning the RETURNING rows; a sharded subclass inserts row by row. */
893
+ async insertRows(client, table, columns, rows, suffix = '') {
894
+ if (rows.length === 0)
895
+ return [];
896
+ const params = [];
897
+ const tuples = rows.map((row) => `(${row.map((value) => `$${params.push(value)}`).join(', ')})`);
898
+ const { rows: returned } = await client.query(`INSERT INTO "${this.schemaName}".${table} (${columns.join(', ')}) VALUES ${tuples.join(', ')} ${suffix}`, params);
899
+ return returned;
900
+ }
901
+ /** Predicate a queue claim must also meet, evaluated once before it scans; sharded, it caps claimers per key. */
902
+ claimGate(_queueName, _partitionKey, _params) {
903
+ return 'TRUE';
904
+ }
905
+ /** Predicate confining a statement that names no single workflow to the current shard. */
906
+ scoped(_alias, _params) {
907
+ return 'TRUE';
908
+ }
909
+ /** Extra (column, expression) pairs for an INSERT ... SELECT confined to the current shard. */
910
+ scopedColumns(_params) {
911
+ return [];
912
+ }
913
+ /** Predicate selecting a global table's rows (queues, schedules, versions). */
914
+ global(_alias, _params) {
915
+ return 'TRUE';
916
+ }
917
+ /** Extra column values for a new row of a global table. */
918
+ globalValues() {
919
+ return {};
920
+ }
921
+ /** Predicate selecting rows whose `column` holds one of these shard keys. */
922
+ keyIn(_alias, _shardKeys, _params, _column = 'shard_key') {
923
+ throw new error_1.DBOSError('Filtering by shard key requires a sharded system database');
924
+ }
925
+ /** A shard key column to read back with a row, if this handle has one. */
926
+ keyColumn(_alias, _column = 'shard_key') {
927
+ return undefined;
928
+ }
929
+ /** Extra column values storing the shard key a schedule's workflows run under. */
930
+ scheduleValues(schedule) {
931
+ if (schedule.shardKey !== undefined && schedule.shardKey !== null) {
932
+ throw new error_1.DBOSError('Scheduling under a shard key requires a sharded system database');
933
+ }
934
+ return {};
935
+ }
936
+ /** The column holding the shard key a schedule's workflows run under, if any. */
937
+ scheduleKeyColumn() {
938
+ return undefined;
939
+ }
940
+ /** This workflow's shard key, resolved now for a statement or waiter that needs it later. */
941
+ keyHint(workflowID) {
942
+ this.#rejectShardKey(workflowID, undefined);
943
+ return undefined;
944
+ }
945
+ /** The notification-map key for a workflow's topic or key, distinct per shard key as IDs repeat across keys. */
946
+ waiter(workflowID, name) {
947
+ const key = this.keyHint(workflowID);
948
+ const payload = `${workflowID}::${name}`;
949
+ return key === undefined ? payload : `${[...key].length}:${key}:${payload}`;
950
+ }
951
+ /** Bind a workflow to `key`, else to the running workflow's own key even where a same-ID target is bound. */
952
+ ownKey(workflowID, key, callback) {
953
+ if (key === undefined) {
954
+ const ctx = (0, context_1.getCurrentContextStore)();
955
+ if (ctx !== undefined && ctx.workflowId === workflowID) {
956
+ key = ctx.shardKey;
957
+ }
958
+ }
959
+ return (0, shard_keys_1.bindShardKeys)([[workflowID, key]], callback);
960
+ }
961
+ /** Run `callback` under this application's lock for a background job, passing whether it was taken. */
962
+ async backgroundJobLock(jobName, callback) {
963
+ // CockroachDB grants every advisory lock, so each process runs every job.
964
+ if (await this.#isCockroach()) {
965
+ return await callback(true);
966
+ }
967
+ // Leading 8 bytes of SHA-256, big-endian signed, as for the retention lock.
968
+ const key = advisoryLockKey(`dbos.background_job.${this.schemaName}.${this.appName ?? ''}.${jobName}`);
969
+ return await this.lockedBackgroundJob(jobName, `SELECT pg_try_advisory_xact_lock($1) AS acquired FROM set_config('idle_in_transaction_session_timeout', $2, true)`, [key.toString(), String(exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS)], callback);
970
+ }
971
+ /** Take a transaction-scoped background job lock with `lockQuery`, which returns `acquired`, and run `callback`. */
972
+ async lockedBackgroundJob(jobName, lockQuery, lockParams, callback) {
973
+ const client = await this.connect();
974
+ try {
975
+ await client.query('BEGIN');
976
+ // The local idle timeout ends a frozen holder's session, releasing its lock.
977
+ const { rows } = await client.query(lockQuery, lockParams);
978
+ return await callback(rows[0]?.acquired === true);
979
+ }
980
+ finally {
981
+ try {
982
+ await client.query('ROLLBACK');
983
+ client.release();
984
+ }
985
+ catch (e) {
986
+ // A session the idle timeout ended has released the lock already.
987
+ this.logger.debug(`Could not release the ${jobName} lock: ${String(e)}`);
988
+ client.release(e instanceof Error ? e : true);
989
+ }
990
+ }
991
+ }
992
+ #cockroach = undefined;
993
+ /** Whether this handle's database is CockroachDB, read on first use. */
994
+ #isCockroach() {
995
+ this.#cockroach ??= (async () => {
996
+ const client = await this.connect();
997
+ try {
998
+ return await isCockroachDB(client);
999
+ }
1000
+ finally {
1001
+ client.release();
1002
+ }
1003
+ })();
1004
+ // A failed read is retried on next use, rather than cached.
1005
+ this.#cockroach.catch(() => {
1006
+ this.#cockroach = undefined;
1007
+ });
1008
+ return this.#cockroach;
1009
+ }
1010
+ /** Scopes a queue's workers dequeue from in turn; one unkeyed scope unless sharded. */
1011
+ dequeueScopes(_queueName) {
1012
+ return Promise.resolve([undefined]);
1013
+ }
1014
+ /** Whether a dequeue failed because another worker holds the rows it needs. */
1015
+ isLockConflict(e) {
1016
+ return isPgDatabaseError(e) && e.code === '55P03';
1017
+ }
1018
+ /** Whether a dequeue failed on a serialization conflict with a concurrent one. */
1019
+ isSerializationConflict(e) {
1020
+ return isPgDatabaseError(e) && e.code === '40001';
1021
+ }
1022
+ /** Whether `appVersion` is this application's latest registered version, or none is registered. */
1023
+ async isLatestVersion(appVersion, client) {
1024
+ // Own plus unclaimed: a named peer's deploy must not demote this one.
1025
+ const latest = await this.#latestApplicationVersionName(client ?? this.pool);
1026
+ return latest === undefined || latest === appVersion;
1027
+ }
727
1028
  /**
728
1029
  * The version name a dequeue treats as latest: this application's own plus unclaimed
729
1030
  * ones, so a named peer's deploy does not demote this one.
730
1031
  */
731
1032
  async #latestApplicationVersionName(client) {
732
- const params = [];
733
- const scope = this.appNameFilter('application_name', this.appName, params);
734
- const { rows } = await client.query(`SELECT version_name
735
- FROM "${this.schemaName}".application_versions
736
- WHERE ${scope}
737
- ORDER BY version_timestamp DESC LIMIT 1`, params);
738
- return rows[0]?.version_name;
1033
+ return (await this.#latestApplicationVersion(client, this.appName))?.version_name;
1034
+ }
1035
+ async #latestApplicationVersion(client, applicationName) {
1036
+ const columns = 'version_id, version_name, version_timestamp, created_at, application_name';
1037
+ const table = `"${this.schemaName}".application_versions`;
1038
+ const params = applicationName ? [applicationName] : [];
1039
+ const scope = this.global(undefined, params);
1040
+ // Probe each eligible owner separately; an OR filter can scan an unbounded history of newer peer versions.
1041
+ // Keep the constant owner in the ordering so the IS NULL probe uses the composite index too.
1042
+ const query = applicationName
1043
+ ? `(SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
1044
+ UNION ALL
1045
+ (SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
1046
+ ORDER BY version_timestamp DESC LIMIT 1`
1047
+ : `SELECT ${columns} FROM ${table} WHERE ${scope} ORDER BY version_timestamp DESC LIMIT 1`;
1048
+ const { rows } = await client.query(query, params);
1049
+ return rows[0];
739
1050
  }
740
1051
  /**
741
1052
  * The owner to persist when writing a row that may already exist. A nameless writer
742
1053
  * leaves the owner intact; a named one collides only with a different name.
743
1054
  */
744
1055
  async #resolveRowOwner(client, table, keyColumn, name, owner, kind) {
745
- const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${keyColumn} = $1`, [name]);
1056
+ const params = [name];
1057
+ const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${this.global(undefined, params)} AND ${keyColumn} = $1`, params);
746
1058
  const current = rows[0]?.application_name ?? null;
747
1059
  if (current === null)
748
1060
  return owner;
@@ -759,8 +1071,12 @@ class SystemDatabase {
759
1071
  }
760
1072
  /** Migrates the system database, or, when `runMigrations` is false, verifies it is already migrated. */
761
1073
  async init(runMigrations = true) {
762
- const migrateOrVerify = runMigrations ? ensureSystemDatabase : verifySystemDatabase;
763
- await migrateOrVerify(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
1074
+ if (runMigrations) {
1075
+ await this.runMigrations();
1076
+ }
1077
+ else {
1078
+ await this.verifyMigrations();
1079
+ }
764
1080
  if (this.shouldUseDBNotifications) {
765
1081
  await this.#listenForNotifications();
766
1082
  // Push coalesced stream and event notifications off the write path.
@@ -768,6 +1084,32 @@ class SystemDatabase {
768
1084
  this.#notifierLoop = this.#runNotifier();
769
1085
  }
770
1086
  }
1087
+ /** Create or migrate the system database. */
1088
+ async runMigrations() {
1089
+ await ensureSystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
1090
+ }
1091
+ /** Throw if this unsharded handle's schema is sharded; an unreachable database is left for its first statement. */
1092
+ async checkShardedSchema() {
1093
+ if (this.sharded)
1094
+ return;
1095
+ let client;
1096
+ try {
1097
+ client = await this.connect();
1098
+ }
1099
+ catch {
1100
+ return;
1101
+ }
1102
+ try {
1103
+ await refuseShardedSchema(client, this.schemaName);
1104
+ }
1105
+ finally {
1106
+ client.release();
1107
+ }
1108
+ }
1109
+ /** Check the system database is migrated, creating and changing nothing. */
1110
+ async verifyMigrations() {
1111
+ await verifySystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
1112
+ }
771
1113
  async destroy() {
772
1114
  // Set synchronously, before any await, so no reconnect is scheduled or published after this point.
773
1115
  this.#notificationsStopped = true;
@@ -813,7 +1155,7 @@ class SystemDatabase {
813
1155
  /** Runs on `client` if given, joining its transaction; otherwise in its own retried transaction. */
814
1156
  async initWorkflowStatus(initStatus, creatorXid, client, reusePolicy = 'return-existing') {
815
1157
  if (client !== undefined) {
816
- return await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
1158
+ return await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
817
1159
  }
818
1160
  return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
819
1161
  }
@@ -822,7 +1164,7 @@ class SystemDatabase {
822
1164
  let shouldCommit = false;
823
1165
  try {
824
1166
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
825
- const result = await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
1167
+ const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
826
1168
  // If there is an existing DB record and we aren't here to recover it, leave it be.
827
1169
  shouldCommit = result.shouldExecuteOnThisExecutor;
828
1170
  return result;
@@ -850,10 +1192,12 @@ class SystemDatabase {
850
1192
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
851
1193
  const parentOwnerXid = (0, context_1.currentOwnerXid)(parentWorkflowID);
852
1194
  if (parentOwnerXid !== undefined) {
853
- await this.#checkOwner(client, parentWorkflowID, parentOwnerXid);
1195
+ await this.checkOwner(client, parentWorkflowID, parentOwnerXid);
854
1196
  }
855
- const result = await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
856
- await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs, { childWorkflowID: initStatus.workflowUUID, ownerChecked: true });
1197
+ const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
1198
+ await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs,
1199
+ // Unkeyed, as only a sharded database records the child's key, through its own override.
1200
+ { childWorkflowID: initStatus.workflowUUID, ownerChecked: true, childValues: {} });
857
1201
  shouldCommit = true;
858
1202
  return result;
859
1203
  }
@@ -872,7 +1216,7 @@ class SystemDatabase {
872
1216
  }
873
1217
  }
874
1218
  }
875
- async #initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy) {
1219
+ async initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy) {
876
1220
  const resRow = await this.insertWorkflowStatus(client, initStatus, creatorXid);
877
1221
  if (reusePolicy === 'reject' && (creatorXid === null || resRow.creator_xid !== creatorXid)) {
878
1222
  throw new error_1.DBOSWorkflowIDInUseError(initStatus.workflowUUID, resRow.status, resRow.name);
@@ -904,6 +1248,7 @@ class SystemDatabase {
904
1248
  async deadLetterWorkflows(workflowIDs, minRecoveryAttempts) {
905
1249
  if (workflowIDs.length === 0)
906
1250
  return;
1251
+ const params = [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflow_1.StatusString.PENDING, minRecoveryAttempts];
907
1252
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
908
1253
  SET status = $1,
909
1254
  owner_xid = NULL,
@@ -912,17 +1257,18 @@ class SystemDatabase {
912
1257
  queue_name = NULL,
913
1258
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
914
1259
  completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
915
- WHERE workflow_uuid = ANY($2::text[]) AND status = $3 AND recovery_attempts >= $4`, [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflowIDs, workflow_1.StatusString.PENDING, minRecoveryAttempts]);
1260
+ WHERE ${this.wfIn(undefined, workflowIDs, params)} AND status = $2 AND recovery_attempts >= $3`, params);
916
1261
  }
917
1262
  /** Highest created_at among still-active rows per partition key, used to seed the in-memory cursor. */
918
1263
  async #maxPartitionKeyCreatedAt(keys) {
919
1264
  const maxima = new Map();
920
1265
  if (keys.length === 0)
921
1266
  return maxima;
1267
+ const params = [keys, [workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING]];
922
1268
  const { rows } = await this.pool.query(`SELECT queue_partition_key, MAX(created_at) AS max_created_at
923
1269
  FROM "${this.schemaName}".workflow_status
924
- WHERE queue_partition_key = ANY($1) AND status = ANY($2)
925
- GROUP BY queue_partition_key`, [keys, [workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING]]);
1270
+ WHERE ${this.scoped(undefined, params)} AND queue_partition_key = ANY($1) AND status = ANY($2)
1271
+ GROUP BY queue_partition_key`, params);
926
1272
  for (const row of rows) {
927
1273
  if (row.max_created_at !== null) {
928
1274
  maxima.set(row.queue_partition_key, Number(row.max_created_at));
@@ -1028,39 +1374,60 @@ class SystemDatabase {
1028
1374
  const chunkSize = 500;
1029
1375
  for (let start = 0; start < statuses.length; start += chunkSize) {
1030
1376
  const chunk = statuses.slice(start, start + chunkSize);
1031
- const tuples = [];
1032
- const params = [];
1033
- let paramIdx = 1;
1034
- for (let i = 0; i < chunk.length; i++) {
1035
- const status = chunk[i];
1377
+ const keyed = chunk.map((status) => ({
1378
+ ...this.wfValues(status.workflowUUID, status.shardKey),
1379
+ ...this.parentValues(status.parentWorkflowID),
1380
+ }));
1381
+ const rows = chunk.map((status, i) => {
1036
1382
  const createdAt = createdAts[start + i];
1037
- tuples.push(`(${columns.map(() => `$${paramIdx++}`).join(', ')})`);
1038
- params.push(status.workflowUUID, status.status, status.workflowName,
1039
- // For cross-language compatibility, these MUST be NULL in the database when not set
1040
- status.workflowClassName === '' ? null : status.workflowClassName, status.workflowConfigName === '' ? null : status.workflowConfigName, status.queueName ?? null, status.authenticatedUser, status.assumedRole, JSON.stringify(status.authenticatedRoles), status.executorId, status.applicationVersion ?? null, status.applicationID, createdAt, 0, createdAt, status.timeoutMS ?? null, status.deadlineEpochMS ?? null,
1041
- // Legacy column: the payload lives in workflow_input.
1042
- null, null, status.priority, status.queuePartitionKey ?? null, status.parentWorkflowID ?? null, status.serialization, null, status.delayUntilEpochMS ?? null, status.attributes ? JSON.stringify(status.attributes) : null, status.scheduleName ?? null, status.applicationName ?? null);
1043
- }
1044
- const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
1045
- VALUES ${tuples.join(', ')}
1046
- ON CONFLICT (workflow_uuid) DO NOTHING
1047
- RETURNING workflow_uuid`, params);
1048
- for (const row of rows) {
1383
+ return [
1384
+ ...Object.values(keyed[i]),
1385
+ status.workflowUUID,
1386
+ status.status,
1387
+ status.workflowName,
1388
+ // For cross-language compatibility, these MUST be NULL in the database when not set
1389
+ status.workflowClassName === '' ? null : status.workflowClassName,
1390
+ status.workflowConfigName === '' ? null : status.workflowConfigName,
1391
+ status.queueName ?? null,
1392
+ status.authenticatedUser,
1393
+ status.assumedRole,
1394
+ JSON.stringify(status.authenticatedRoles),
1395
+ status.executorId,
1396
+ status.applicationVersion ?? null,
1397
+ status.applicationID,
1398
+ createdAt,
1399
+ 0,
1400
+ createdAt,
1401
+ status.timeoutMS ?? null,
1402
+ status.deadlineEpochMS ?? null,
1403
+ // Legacy column: the payload lives in workflow_input.
1404
+ null,
1405
+ null,
1406
+ status.priority,
1407
+ status.queuePartitionKey ?? null,
1408
+ status.parentWorkflowID ?? null,
1409
+ status.serialization,
1410
+ null,
1411
+ status.delayUntilEpochMS ?? null,
1412
+ status.attributes ? JSON.stringify(status.attributes) : null,
1413
+ status.scheduleName ?? null,
1414
+ status.applicationName ?? null,
1415
+ ];
1416
+ });
1417
+ const returned = await this.insertRows(client, 'workflow_status', [...Object.keys(keyed[0]), ...columns], rows, `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING RETURNING workflow_uuid`);
1418
+ for (const row of returned) {
1049
1419
  inserted.add(row.workflow_uuid);
1050
1420
  }
1051
1421
  }
1052
1422
  for (let start = 0; start < statuses.length; start += chunkSize) {
1053
1423
  const chunk = statuses.slice(start, start + chunkSize);
1054
- const tuples = [];
1055
- const params = [];
1056
- let paramIdx = 1;
1057
- for (let i = 0; i < chunk.length; i++) {
1058
- tuples.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
1059
- params.push(chunk[i].workflowUUID, chunk[i].input, createdAts[start + i]);
1060
- }
1061
- await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
1062
- VALUES ${tuples.join(', ')}
1063
- ON CONFLICT (workflow_uuid) DO NOTHING`, params);
1424
+ const keyed = chunk.map((status) => this.wfValues(status.workflowUUID, status.shardKey));
1425
+ await this.insertRows(client, 'workflow_input', [...Object.keys(keyed[0]), 'workflow_uuid', 'inputs', 'retention_timestamp'], chunk.map((status, i) => [
1426
+ ...Object.values(keyed[i]),
1427
+ status.workflowUUID,
1428
+ status.input,
1429
+ createdAts[start + i],
1430
+ ]), `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`);
1064
1431
  }
1065
1432
  await client.query('COMMIT');
1066
1433
  }
@@ -1074,22 +1441,10 @@ class SystemDatabase {
1074
1441
  return inserted;
1075
1442
  }
1076
1443
  async recordWorkflowOutput(workflowID, status, ownerXid) {
1077
- const client = await this.connect();
1078
- try {
1079
- return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
1080
- }
1081
- finally {
1082
- client.release();
1083
- }
1444
+ return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
1084
1445
  }
1085
1446
  async recordWorkflowError(workflowID, status, ownerXid) {
1086
- const client = await this.connect();
1087
- try {
1088
- return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
1089
- }
1090
- finally {
1091
- client.release();
1092
- }
1447
+ return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
1093
1448
  }
1094
1449
  // Record a workflow's terminal outcome (SUCCESS or ERROR), reporting whether
1095
1450
  // the write landed. The write applies only to a PENDING row still owned by
@@ -1100,8 +1455,49 @@ class SystemDatabase {
1100
1455
  // terminal, handed to another execution (e.g. by a concurrent resume or
1101
1456
  // recovery), or deleted; the caller resolves which by awaiting the recorded
1102
1457
  // outcome.
1103
- async #recordWorkflowOutcome(client, workflowID, status, outcome, ownerXid) {
1458
+ async #recordWorkflowOutcome(workflowID, status, outcome, ownerXid) {
1104
1459
  ownerXid ??= (0, context_1.currentOwnerXid)(workflowID);
1460
+ if (!(await this.outcomeInOneStatement())) {
1461
+ return await this.#recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid);
1462
+ }
1463
+ // One autocommit round trip; the upsert reads the UPDATE's rows, so the output lands only with the status.
1464
+ const params = [status, workflow_1.StatusString.PENDING, outcome.output ?? null, outcome.error ?? null];
1465
+ const target = this.wf(undefined, workflowID, params);
1466
+ const owned = ownerXid !== undefined ? `AND owner_xid = $${params.push(ownerXid)}` : '';
1467
+ const keys = this.wfValues(workflowID);
1468
+ const keyColumns = Object.keys(keys);
1469
+ const keyExprs = Object.values(keys).map((value) => `$${params.push(value)}::text`);
1470
+ const workflowParam = `$${params.push(workflowID)}::text`;
1471
+ const { rows } = await this.pool.query(`WITH updated AS (
1472
+ UPDATE "${this.schemaName}".workflow_status
1473
+ SET status = $1,
1474
+ -- As the workflow is complete, remove its deduplication ID; a finished workflow has no owner.
1475
+ deduplication_id = NULL,
1476
+ owner_xid = NULL,
1477
+ updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
1478
+ completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1479
+ WHERE ${target} AND status = $2 ${owned}
1480
+ RETURNING 1
1481
+ ), stored AS (
1482
+ INSERT INTO "${this.schemaName}".workflow_output (${[...keyColumns, 'workflow_uuid', 'output', 'error'].join(', ')})
1483
+ SELECT ${[...keyExprs, workflowParam, '$3::text', '$4::text'].join(', ')}
1484
+ WHERE EXISTS (SELECT 1 FROM updated)
1485
+ -- EXCLUDED is the row this INSERT proposed, so these are the values above.
1486
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error
1487
+ -- CockroachDB refuses a data-modifying CTE without RETURNING.
1488
+ RETURNING 1
1489
+ )
1490
+ -- Selecting from the CTEs rather than inserting at the top level lets a router see the shard key.
1491
+ SELECT count(*) AS count FROM updated`, params);
1492
+ return Number(rows[0]?.count ?? 0) > 0;
1493
+ }
1494
+ /** Whether a workflow's outcome is written in one statement, rather than a transaction of two. */
1495
+ outcomeInOneStatement() {
1496
+ return Promise.resolve(true);
1497
+ }
1498
+ /** The outcome write as a transaction of two statements, for a backend that cannot run it as one. */
1499
+ async #recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid) {
1500
+ const client = await this.connect();
1105
1501
  let committed = false;
1106
1502
  try {
1107
1503
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
@@ -1115,19 +1511,31 @@ class SystemDatabase {
1115
1511
  // The outcome was not ours to write, so leave no orphan payload.
1116
1512
  return false;
1117
1513
  }
1118
- await client.query(`INSERT INTO "${this.schemaName}".workflow_output (workflow_uuid, output, error)
1119
- VALUES ($1, $2, $3)
1120
- ON CONFLICT (workflow_uuid) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error`, [workflowID, outcome.output ?? null, outcome.error ?? null]);
1514
+ const values = {
1515
+ ...this.wfValues(workflowID),
1516
+ workflow_uuid: workflowID,
1517
+ output: outcome.output ?? null,
1518
+ error: outcome.error ?? null,
1519
+ };
1520
+ const columns = Object.keys(values);
1521
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_output (${columns.join(', ')})
1522
+ VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
1523
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error`, Object.values(values));
1121
1524
  committed = true;
1122
1525
  return true;
1123
1526
  }
1124
1527
  finally {
1125
- if (committed) {
1126
- await client.query('COMMIT');
1528
+ try {
1529
+ if (committed) {
1530
+ await client.query('COMMIT');
1531
+ }
1532
+ else {
1533
+ // Swallowed: a rollback failure must not mask why we are rolling back.
1534
+ await client.query('ROLLBACK').catch(() => undefined);
1535
+ }
1127
1536
  }
1128
- else {
1129
- // Swallowed: a rollback failure must not mask why we are rolling back.
1130
- await client.query('ROLLBACK').catch(() => undefined);
1537
+ finally {
1538
+ client.release();
1131
1539
  }
1132
1540
  }
1133
1541
  }
@@ -1136,6 +1544,7 @@ class SystemDatabase {
1136
1544
  const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
1137
1545
  // executor_id defaults to "local", so it collides across applications.
1138
1546
  const scope = this.appNameFilter('application_name', this.appName, params);
1547
+ const keyColumn = this.keyColumn(undefined);
1139
1548
  const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1140
1549
  SET started_at_epoch_ms = NULL,
1141
1550
  status = $1,
@@ -1146,25 +1555,31 @@ class SystemDatabase {
1146
1555
  AND executor_id = $4
1147
1556
  AND application_version = $5
1148
1557
  AND ${scope}
1149
- RETURNING workflow_uuid`, params);
1150
- return result.rows.map((row) => row.workflow_uuid);
1558
+ RETURNING workflow_uuid${keyColumn !== undefined ? `, ${keyColumn} AS shard_key` : ''}`, params);
1559
+ return result.rows.map((row) => ({ workflowID: row.workflow_uuid, shardKey: row.shard_key ?? undefined }));
1151
1560
  }
1152
1561
  async getWorkflowStatus(workflowID, callerID, callerFN) {
1153
- const funcGetStatus = async () => {
1154
- const statuses = await this.listWorkflows({ workflowIDs: [workflowID] });
1562
+ // Resolved before binding the caller's own key, which a same-ID target would share.
1563
+ const shardKey = this.keyHint(workflowID);
1564
+ // Inside a workflow this runs on the transaction's client: taking a second pool connection while
1565
+ // holding the first deadlocks the pool once every connection is held by a waiting caller.
1566
+ const funcGetStatus = async (client) => {
1567
+ const statuses = await this.#listWorkflows({ workflowIDs: [workflowID], shardKey }, {}, client);
1155
1568
  const status = statuses.find((s) => s.workflowUUID === workflowID);
1156
1569
  return status ? JSON.stringify(status) : null;
1157
1570
  };
1158
- if (callerID && callerFN) {
1159
- const client = await this.connect();
1160
- try {
1161
- // Check if the operation has been done before for OAOO (only do this inside a workflow).
1162
- const json = await this.#inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, funcGetStatus));
1163
- return parseStatus(json);
1164
- }
1165
- finally {
1166
- client.release();
1167
- }
1571
+ if (callerID && callerFN !== undefined) {
1572
+ const json = await this.ownKey(callerID, undefined, async () => {
1573
+ const client = await this.connect();
1574
+ try {
1575
+ // Check if the operation has been done before for OAOO (only do this inside a workflow).
1576
+ return await this.inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, () => funcGetStatus(client)));
1577
+ }
1578
+ finally {
1579
+ client.release();
1580
+ }
1581
+ });
1582
+ return parseStatus(json);
1168
1583
  }
1169
1584
  else {
1170
1585
  const json = await funcGetStatus();
@@ -1178,7 +1593,7 @@ class SystemDatabase {
1178
1593
  statusFetchChunkSize = 500;
1179
1594
  // Retried per chunk so a reconnect refetches one chunk, not every chunk before it.
1180
1595
  async fetchWorkflowStatusChunk(workflowIDs) {
1181
- return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false });
1596
+ return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false }, { keyedIDs: true });
1182
1597
  }
1183
1598
  /** Fetch many statuses in as few round trips as possible. IDs with no row are omitted. */
1184
1599
  async getWorkflowStatuses(workflowIDs) {
@@ -1209,23 +1624,26 @@ class SystemDatabase {
1209
1624
  }
1210
1625
  }
1211
1626
  // ==================== Step Results ====================
1212
- async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
1213
- const client = await this.connect();
1214
- try {
1215
- return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
1216
- }
1217
- finally {
1218
- client.release();
1219
- }
1627
+ async getOperationResultAndThrowIfCancelled(workflowID, functionID, ownerKey) {
1628
+ return await this.ownKey(workflowID, ownerKey, async () => {
1629
+ const client = await this.connect();
1630
+ try {
1631
+ return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
1632
+ }
1633
+ finally {
1634
+ client.release();
1635
+ }
1636
+ });
1220
1637
  }
1221
1638
  async getAllOperationResults(workflowID, limit, offset, loadOutput = true) {
1222
1639
  // Skip the output/error payload columns when the caller does not need them,
1223
1640
  // mirroring how listWorkflows skips its payload joins when loadOutput is false.
1641
+ const childKey = this.keyColumn(undefined, 'child_shard_key');
1224
1642
  const columns = loadOutput
1225
1643
  ? '*'
1226
- : 'workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp';
1227
- let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE workflow_uuid=$1 ORDER BY function_id`;
1228
- const params = [workflowID];
1644
+ : `workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? `, ${childKey}` : ''}`;
1645
+ const params = [];
1646
+ let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE ${this.wf(undefined, workflowID, params)} ORDER BY function_id`;
1229
1647
  if (limit !== undefined) {
1230
1648
  params.push(limit);
1231
1649
  query += ` LIMIT $${params.length}`;
@@ -1238,14 +1656,18 @@ class SystemDatabase {
1238
1656
  return rows;
1239
1657
  }
1240
1658
  async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
1241
- const client = await this.connect();
1242
- try {
1243
- await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options));
1244
- }
1245
- finally {
1246
- client.release();
1247
- await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
1248
- }
1659
+ // Resolved before binding the caller's own key, which a same-ID child would share.
1660
+ const childValues = this.childValues(options.childWorkflowID);
1661
+ await this.ownKey(workflowID, options.ownerKey, async () => {
1662
+ const client = await this.connect();
1663
+ try {
1664
+ await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, { ...options, childValues }));
1665
+ }
1666
+ finally {
1667
+ client.release();
1668
+ await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
1669
+ }
1670
+ });
1249
1671
  }
1250
1672
  /** Checkpoint an operation's failure so a replay re-throws it instead of re-running. */
1251
1673
  async recordOperationError(workflowID, functionID, functionName, startTimeEpochMs, error, childWorkflowID) {
@@ -1319,9 +1741,10 @@ class SystemDatabase {
1319
1741
  throw new TypeError('functionID must be defined');
1320
1742
  patchName = `DBOS.patch-${patchName}`;
1321
1743
  const readCheckpointName = async (queryable) => {
1744
+ const params = [functionID];
1322
1745
  const { rows } = await queryable.query(`SELECT function_name
1323
1746
  FROM "${this.schemaName}".operation_outputs
1324
- WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
1747
+ WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
1325
1748
  return rows.length === 0 ? undefined : rows[0].function_name;
1326
1749
  };
1327
1750
  if (deprecated) {
@@ -1336,7 +1759,7 @@ class SystemDatabase {
1336
1759
  // If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
1337
1760
  const client = await this.connect();
1338
1761
  try {
1339
- return await this.#inTransaction(client, async () => {
1762
+ return await this.inTransaction(client, async () => {
1340
1763
  const checkpointName = await readCheckpointName(client);
1341
1764
  if (checkpointName !== undefined) {
1342
1765
  const matches = checkpointName === patchName;
@@ -1362,7 +1785,7 @@ class SystemDatabase {
1362
1785
  let frontier = workflowIDs;
1363
1786
  while (frontier.length > 0) {
1364
1787
  await this.#cancelWorkflows(frontier);
1365
- const children = await this.#getDirectChildren(frontier);
1788
+ const children = await this.getDirectChildren(frontier);
1366
1789
  frontier = children.filter((id) => !visited.has(id));
1367
1790
  for (const id of frontier) {
1368
1791
  visited.add(id);
@@ -1370,41 +1793,44 @@ class SystemDatabase {
1370
1793
  }
1371
1794
  }
1372
1795
  async #cancelWorkflows(workflowIDs) {
1796
+ const params = [workflow_1.StatusString.CANCELLED, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR];
1373
1797
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1374
1798
  SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
1375
1799
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
1376
1800
  completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1377
- WHERE workflow_uuid = ANY($2)
1378
- AND status NOT IN ($3, $4)`, [workflow_1.StatusString.CANCELLED, workflowIDs, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR]);
1801
+ WHERE ${this.wfIn(undefined, workflowIDs, params)}
1802
+ AND status NOT IN ($2, $3)`, params);
1379
1803
  }
1380
1804
  async checkIfCanceled(workflowID) {
1381
- await this.#checkIfCanceled(this.pool, workflowID);
1805
+ // The caller's own workflow, even where a same-ID workflow is bound as a target.
1806
+ await this.ownKey(workflowID, undefined, () => this.#checkIfCanceled(this.pool, workflowID));
1382
1807
  }
1383
1808
  async resumeWorkflows(workflowIDs, queueName) {
1384
1809
  const client = await this.connect();
1385
1810
  try {
1386
1811
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1387
1812
  // Check existence separately: a zero-row update also means "already complete", a legal no-op.
1388
- const { rows } = await client.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [workflowIDs]);
1813
+ const existingParams = [];
1814
+ const { rows } = await client.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${this.wfIn(undefined, workflowIDs, existingParams)}`, existingParams);
1389
1815
  const existing = new Set(rows.map((r) => r.workflow_uuid));
1390
1816
  const missing = workflowIDs.filter((id) => !existing.has(id));
1391
1817
  if (missing.length > 0) {
1392
1818
  throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${missing.join(', ')} does not exist`);
1393
1819
  }
1820
+ const updateParams = [
1821
+ workflow_1.StatusString.ENQUEUED,
1822
+ queueName ?? utils_1.INTERNAL_QUEUE_NAME,
1823
+ workflow_1.StatusString.SUCCESS,
1824
+ workflow_1.StatusString.ERROR,
1825
+ ];
1394
1826
  await client.query(`UPDATE "${this.schemaName}".workflow_status
1395
1827
  SET status = $1, owner_xid = NULL, queue_name = $2, recovery_attempts = 0,
1396
1828
  workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
1397
1829
  started_at_epoch_ms = NULL,
1398
1830
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
1399
1831
  completed_at = NULL
1400
- WHERE workflow_uuid = ANY($3)
1401
- AND status NOT IN ($4, $5)`, [
1402
- workflow_1.StatusString.ENQUEUED,
1403
- queueName ?? utils_1.INTERNAL_QUEUE_NAME,
1404
- workflowIDs,
1405
- workflow_1.StatusString.SUCCESS,
1406
- workflow_1.StatusString.ERROR,
1407
- ]);
1832
+ WHERE ${this.wfIn(undefined, workflowIDs, updateParams)}
1833
+ AND status NOT IN ($3, $4)`, updateParams);
1408
1834
  await client.query('COMMIT');
1409
1835
  }
1410
1836
  catch (e) {
@@ -1417,16 +1843,18 @@ class SystemDatabase {
1417
1843
  }
1418
1844
  }
1419
1845
  async setWorkflowPriority(workflowID, priority) {
1846
+ const params = [priority, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED];
1420
1847
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1421
1848
  SET priority = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1422
- WHERE workflow_uuid = $2
1423
- AND status IN ($3, $4)`, [priority, workflowID, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED]);
1849
+ WHERE ${this.wf(undefined, workflowID, params)}
1850
+ AND status IN ($2, $3)`, params);
1424
1851
  }
1425
1852
  async setWorkflowDelay(workflowID, delayUntilEpochMS) {
1853
+ const params = [delayUntilEpochMS, workflow_1.StatusString.DELAYED];
1426
1854
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1427
1855
  SET delay_until_epoch_ms = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1428
- WHERE workflow_uuid = $2
1429
- AND status = $3`, [delayUntilEpochMS, workflowID, workflow_1.StatusString.DELAYED]);
1856
+ WHERE ${this.wf(undefined, workflowID, params)}
1857
+ AND status = $2`, params);
1430
1858
  }
1431
1859
  /**
1432
1860
  * Extend an existing debounced DELAYED workflow's delay and update its inputs, atomically.
@@ -1492,7 +1920,7 @@ class SystemDatabase {
1492
1920
  if (updated.rows.length > 0) {
1493
1921
  await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
1494
1922
  VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
1495
- ON CONFLICT (workflow_uuid) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
1923
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
1496
1924
  return {
1497
1925
  bouncedWorkflowID: updated.rows[0].workflow_uuid,
1498
1926
  holderWorkflowID: null,
@@ -1502,10 +1930,11 @@ class SystemDatabase {
1502
1930
  holderApplicationName: null,
1503
1931
  };
1504
1932
  }
1505
- // Unscoped, so a holder that blocked the update above is reportable.
1933
+ // No application filter, so a holder that blocked the update above is reportable.
1934
+ const holderParams = [params.queueName, params.deduplicationID];
1506
1935
  const holder = await client.query(`SELECT workflow_uuid, is_debounced, name, class_name, application_name
1507
1936
  FROM "${this.schemaName}".workflow_status
1508
- WHERE queue_name = $1 AND deduplication_id = $2`, [params.queueName, params.deduplicationID]);
1937
+ WHERE ${this.scoped(undefined, holderParams)} AND queue_name = $1 AND deduplication_id = $2`, holderParams);
1509
1938
  if (holder.rows.length === 0) {
1510
1939
  return {
1511
1940
  bouncedWorkflowID: null,
@@ -1526,7 +1955,7 @@ class SystemDatabase {
1526
1955
  };
1527
1956
  }
1528
1957
  // Get the immediate (one-level) child workflow IDs for a set of workflows.
1529
- async #getDirectChildren(workflowIDs) {
1958
+ async getDirectChildren(workflowIDs) {
1530
1959
  if (workflowIDs.length === 0) {
1531
1960
  return [];
1532
1961
  }
@@ -1540,7 +1969,7 @@ class SystemDatabase {
1540
1969
  const descendants = new Set();
1541
1970
  let frontier = [workflowID];
1542
1971
  while (frontier.length > 0) {
1543
- const children = await this.#getDirectChildren(frontier);
1972
+ const children = await this.getDirectChildren(frontier);
1544
1973
  frontier = children.filter((id) => !descendants.has(id));
1545
1974
  for (const id of frontier) {
1546
1975
  descendants.add(id);
@@ -1548,6 +1977,7 @@ class SystemDatabase {
1548
1977
  }
1549
1978
  return [...descendants];
1550
1979
  }
1980
+ /** Delete workflows and all associated data, and with deleteChildren every workflow they started, recursively. */
1551
1981
  async deleteWorkflows(workflowIDs, deleteChildren = false) {
1552
1982
  const allIds = [...workflowIDs];
1553
1983
  if (deleteChildren) {
@@ -1555,15 +1985,18 @@ class SystemDatabase {
1555
1985
  allIds.push(...(await this.getWorkflowChildren(wfid)));
1556
1986
  }
1557
1987
  }
1988
+ await this.deleteWorkflowsInternal(allIds);
1989
+ }
1990
+ async deleteWorkflowsInternal(allIds) {
1558
1991
  const client = await this.connect();
1559
1992
  try {
1560
1993
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1561
1994
  // The payload tables carry no foreign key, so the status delete does not cascade
1562
1995
  // into them; without this they survive as orphans.
1563
- for (const table of ['workflow_input', 'workflow_output', 'operation_outputs']) {
1564
- await client.query(`DELETE FROM "${this.schemaName}".${table} WHERE workflow_uuid = ANY($1)`, [allIds]);
1996
+ for (const table of ['workflow_input', 'workflow_output', 'operation_outputs', 'workflow_status']) {
1997
+ const params = [];
1998
+ await client.query(`DELETE FROM "${this.schemaName}".${table} WHERE ${this.wfIn(undefined, allIds, params)}`, params);
1565
1999
  }
1566
- await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [allIds]);
1567
2000
  await client.query('COMMIT');
1568
2001
  }
1569
2002
  catch (e) {
@@ -1604,7 +2037,8 @@ class SystemDatabase {
1604
2037
  const client = await this.connect();
1605
2038
  try {
1606
2039
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1607
- const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE workflow_uuid = $1`, [workflowID]);
2040
+ const statusParams = [];
2041
+ const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE ${this.wf(undefined, workflowID, statusParams)}`, statusParams);
1608
2042
  if (statusRows.length === 0) {
1609
2043
  throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
1610
2044
  }
@@ -1615,56 +2049,61 @@ class SystemDatabase {
1615
2049
  // Roll workflow_events back to the last value published before the cut, using
1616
2050
  // workflow_events_history as an undo log, then rewind that history too.
1617
2051
  // `publishedPastCut` is the correlated EXISTS both halves key off.
1618
- const publishedPastCut = `EXISTS (
2052
+ const publishedPastCut = (key, params) => `EXISTS (
1619
2053
  SELECT 1 FROM "${schema}".workflow_events_history discarded
1620
- WHERE discarded.workflow_uuid = $1 AND discarded.key = %KEY% AND discarded.function_id >= $2
2054
+ WHERE ${this.wf('discarded', workflowID, params)} AND discarded.key = ${key} AND discarded.function_id >= $1
1621
2055
  )`;
1622
2056
  // First unpublish everything the discarded run touched.
2057
+ const unpublishParams = [startStep];
1623
2058
  await client.query(`DELETE FROM "${schema}".workflow_events
1624
- WHERE workflow_uuid = $1 AND ${publishedPastCut.replace('%KEY%', '"' + schema + '".workflow_events.key')}`, [workflowID, startStep]);
2059
+ WHERE ${this.wf(undefined, workflowID, unpublishParams)}
2060
+ AND ${publishedPastCut('"' + schema + '".workflow_events.key', unpublishParams)}`, unpublishParams);
1625
2061
  // Then restore those keys, if any, to the value published below the cut.
1626
- await client.query(`INSERT INTO "${schema}".workflow_events (workflow_uuid, key, value, serialization)
1627
- SELECT surviving.workflow_uuid, surviving.key, surviving.value, surviving.serialization
2062
+ const restoreParams = [startStep];
2063
+ const keyColumns = this.wfColumns(workflowID, restoreParams);
2064
+ await client.query(`INSERT INTO "${schema}".workflow_events (${[...keyColumns.map(([c]) => c), 'workflow_uuid', 'key', 'value', 'serialization'].join(', ')})
2065
+ SELECT ${[...keyColumns.map(([, e]) => e), 'surviving.workflow_uuid', 'surviving.key', 'surviving.value', 'surviving.serialization'].join(', ')}
1628
2066
  FROM (
1629
2067
  SELECT weh.workflow_uuid, weh.key, weh.value, weh.serialization,
1630
2068
  ROW_NUMBER() OVER (PARTITION BY weh.key ORDER BY weh.function_id DESC) AS rn
1631
2069
  FROM "${schema}".workflow_events_history weh
1632
- WHERE weh.workflow_uuid = $1 AND weh.function_id < $2
1633
- AND ${publishedPastCut.replace('%KEY%', 'weh.key')}
2070
+ WHERE ${this.wf('weh', workflowID, restoreParams)} AND weh.function_id < $1
2071
+ AND ${publishedPastCut('weh.key', restoreParams)}
1634
2072
  ) surviving
1635
- WHERE surviving.rn = 1`, [workflowID, startStep]);
2073
+ WHERE surviving.rn = 1`, restoreParams);
1636
2074
  // Clear the streams' close sentinels so the replay can append again.
2075
+ const sentinelParams = [startStep, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name()];
1637
2076
  await client.query(`DELETE FROM "${schema}".streams
1638
- WHERE workflow_uuid = $1 AND function_id >= $2 AND value = $3 AND serialization = $4`, [workflowID, startStep, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name()]);
2077
+ WHERE ${this.wf(undefined, workflowID, sentinelParams)} AND function_id >= $1 AND value = $2 AND serialization = $3`, sentinelParams);
1639
2078
  // Discard the steps and the events history past the cut.
1640
2079
  for (const table of ['operation_outputs', 'workflow_events_history']) {
1641
- await client.query(`DELETE FROM "${schema}".${table} WHERE workflow_uuid = $1 AND function_id >= $2`, [
1642
- workflowID,
1643
- startStep,
1644
- ]);
2080
+ const params = [startStep];
2081
+ await client.query(`DELETE FROM "${schema}".${table} WHERE ${this.wf(undefined, workflowID, params)} AND function_id >= $1`, params);
1645
2082
  }
1646
2083
  // Delete messages consumed or received after the rewind point
1647
- await client.query(`DELETE FROM "${schema}".notifications WHERE destination_uuid = $1 AND (consumed_by_function_id >= $2 OR consumed = false)`, [workflowID, startStep]);
1648
- await client.query(`DELETE FROM "${schema}".workflow_output WHERE workflow_uuid = $1`, [workflowID]);
2084
+ const messageParams = [startStep];
2085
+ await client.query(`DELETE FROM "${schema}".notifications
2086
+ WHERE ${this.wf(undefined, workflowID, messageParams, 'destination_uuid')}
2087
+ AND (consumed_by_function_id >= $1 OR consumed = false)`, messageParams);
2088
+ const outputParams = [];
2089
+ await client.query(`DELETE FROM "${schema}".workflow_output WHERE ${this.wf(undefined, workflowID, outputParams)}`, outputParams);
1649
2090
  // Re-enqueue the workflow. Re-asserting the status we read keeps a workflow that
1650
2091
  // moved on underneath us from being resurrected.
1651
- const setVersion = options.applicationVersion !== undefined ? ', application_version = $6' : '';
1652
2092
  const params = [
1653
2093
  workflow_1.StatusString.ENQUEUED,
1654
2094
  options.queueName ?? utils_1.INTERNAL_QUEUE_NAME,
1655
2095
  options.queuePartitionKey ?? null,
1656
- workflowID,
1657
2096
  status,
1658
2097
  ];
1659
- if (options.applicationVersion !== undefined) {
1660
- params.push(options.applicationVersion);
1661
- }
2098
+ const setVersion = options.applicationVersion !== undefined
2099
+ ? `, application_version = $${params.push(options.applicationVersion)}`
2100
+ : '';
1662
2101
  const { rowCount } = await client.query(`UPDATE "${schema}".workflow_status
1663
2102
  SET status = $1, owner_xid = NULL, queue_name = $2, queue_partition_key = $3, recovery_attempts = 0,
1664
2103
  workflow_deadline_epoch_ms = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
1665
2104
  completed_at = NULL, output = NULL, error = NULL,
1666
2105
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint${setVersion}
1667
- WHERE workflow_uuid = $4 AND status = $5`, params);
2106
+ WHERE ${this.wf(undefined, workflowID, params)} AND status = $4`, params);
1668
2107
  if (rowCount !== 1) {
1669
2108
  throw new error_1.DBOSError(`Workflow ${workflowID} changed status while being rewound; retry the rewind`);
1670
2109
  }
@@ -1694,13 +2133,14 @@ class SystemDatabase {
1694
2133
  try {
1695
2134
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1696
2135
  // Fetch the status of all original workflows inside the transaction.
2136
+ const statusParams = [];
1697
2137
  const { rows: statusRows } = await client.query(`SELECT ws.workflow_uuid, ws.name, ws.class_name, ws.config_name, ws.application_id,
1698
2138
  ws.authenticated_user, ws.authenticated_roles, ws.assumed_role,
1699
2139
  COALESCE(wi.inputs, ws.inputs) AS inputs, ws.serialization,
1700
2140
  ws.application_version, ws.attributes, ws.application_name
1701
2141
  FROM "${this.schemaName}".workflow_status ws
1702
- LEFT JOIN "${this.schemaName}".workflow_input wi ON wi.workflow_uuid = ws.workflow_uuid
1703
- WHERE ws.workflow_uuid = ANY($1)`, [originalWorkflowIDs]);
2142
+ LEFT JOIN "${this.schemaName}".workflow_input wi ON ${this.wfJoin('ws', 'wi')}
2143
+ WHERE ${this.wfIn('ws', originalWorkflowIDs, statusParams)}`, statusParams);
1704
2144
  const statusByID = new Map(statusRows.map((r) => [r.workflow_uuid, r]));
1705
2145
  for (const wid of originalWorkflowIDs) {
1706
2146
  if (!statusByID.has(wid)) {
@@ -1736,35 +2176,42 @@ class SystemDatabase {
1736
2176
  forkedWorkflowIDs[i],
1737
2177
  statusByID.get(origID).application_name ?? this.appName ?? null,
1738
2178
  ]));
1739
- const valuesPlaceholders = [];
1740
- const params = [];
1741
- let paramIdx = 1;
1742
- for (let i = 0; i < originalWorkflowIDs.length; i++) {
1743
- const origID = originalWorkflowIDs[i];
2179
+ const forkKeys = forkedWorkflowIDs.map((forkID) => this.wfValues(forkID));
2180
+ await this.insertRows(client, 'workflow_status', [...Object.keys(forkKeys[0]), ...insertCols], originalWorkflowIDs.map((origID, i) => {
1744
2181
  const forkID = forkedWorkflowIDs[i];
1745
2182
  const ws = statusByID.get(origID);
1746
- const placeholders = insertCols.map(() => `$${paramIdx++}`).join(', ');
1747
- valuesPlaceholders.push(`(${placeholders})`);
1748
- params.push(forkID, workflow_1.StatusString.ENQUEUED, ws.name, ws.class_name ?? null, ws.config_name ?? null, queueName, ws.authenticated_user, ws.assumed_role, ws.authenticated_roles, options.applicationVersion ?? ws.application_version ?? null, ws.application_id,
1749
- // Legacy column: the payload lives in workflow_input.
1750
- null, options.queuePartitionKey ?? null, origID, ws.serialization, ws.attributes ? JSON.stringify(ws.attributes) : null, forkOwners.get(forkID) ?? null);
1751
- if (options.timeoutMS !== undefined) {
1752
- params.push(options.timeoutMS);
1753
- }
1754
- }
1755
- await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${insertCols.join(', ')})
1756
- VALUES ${valuesPlaceholders.join(', ')}`, params);
1757
- const inputPlaceholders = [];
1758
- const inputParams = [];
1759
- let inputIdx = 1;
1760
- for (let i = 0; i < originalWorkflowIDs.length; i++) {
1761
- inputPlaceholders.push(`($${inputIdx++}, $${inputIdx++})`);
1762
- inputParams.push(forkedWorkflowIDs[i], statusByID.get(originalWorkflowIDs[i]).inputs);
1763
- }
1764
- await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs)
1765
- VALUES ${inputPlaceholders.join(', ')}`, inputParams);
2183
+ return [
2184
+ ...Object.values(forkKeys[i]),
2185
+ forkID,
2186
+ workflow_1.StatusString.ENQUEUED,
2187
+ ws.name,
2188
+ ws.class_name ?? null,
2189
+ ws.config_name ?? null,
2190
+ queueName,
2191
+ ws.authenticated_user,
2192
+ ws.assumed_role,
2193
+ ws.authenticated_roles,
2194
+ options.applicationVersion ?? ws.application_version ?? null,
2195
+ ws.application_id,
2196
+ // Legacy column: the payload lives in workflow_input.
2197
+ null,
2198
+ options.queuePartitionKey ?? null,
2199
+ origID,
2200
+ ws.serialization,
2201
+ ws.attributes ? JSON.stringify(ws.attributes) : null,
2202
+ forkOwners.get(forkID) ?? null,
2203
+ ...(options.timeoutMS !== undefined ? [options.timeoutMS] : []),
2204
+ ];
2205
+ }));
2206
+ await this.insertRows(client, 'workflow_input', [...Object.keys(forkKeys[0]), 'workflow_uuid', 'inputs'], originalWorkflowIDs.map((origID, i) => [
2207
+ ...Object.values(forkKeys[i]),
2208
+ forkedWorkflowIDs[i],
2209
+ statusByID.get(origID).inputs,
2210
+ ]));
1766
2211
  // Mark all original workflows as having been forked from.
1767
- await client.query(`UPDATE "${this.schemaName}".workflow_status SET was_forked_from = TRUE WHERE workflow_uuid = ANY($1)`, [originalWorkflowIDs]);
2212
+ const forkedFromParams = [];
2213
+ await client.query(`UPDATE "${this.schemaName}".workflow_status SET was_forked_from = TRUE
2214
+ WHERE ${this.wfIn(undefined, originalWorkflowIDs, forkedFromParams)}`, forkedFromParams);
1768
2215
  // For workflows with start_step > 0, copy checkpoints/events/streams.
1769
2216
  // Build a mapping CTE so each copy is a single SQL statement.
1770
2217
  const forkMappings = originalWorkflowIDs
@@ -1781,55 +2228,85 @@ class SystemDatabase {
1781
2228
  mIdx += 4;
1782
2229
  }
1783
2230
  const mappingCTE = `WITH mapping(orig_id, fork_id, start_step, owner) AS (VALUES ${mappingValues.join(', ')})`;
2231
+ // The forks live under their originals' shard key, read from the scope; a copy's params extend the mapping's.
2232
+ const copyParams = () => [...mappingParams];
2233
+ const keyed = (params) => {
2234
+ const columns = this.scopedColumns(params);
2235
+ return { names: columns.map(([c]) => `${c}, `).join(''), exprs: columns.map(([, e]) => `${e}, `).join('') };
2236
+ };
2237
+ // A replacement child is a fork, so it shares the replaced child's key.
2238
+ const childKey = this.keyColumn('oo', 'child_shard_key');
2239
+ if (options.replacementChildren && childKey !== undefined) {
2240
+ // A replacement names a child by ID, which can't pick one of several keys.
2241
+ const ambiguityParams = copyParams();
2242
+ const replaced = `$${ambiguityParams.push(Object.keys(options.replacementChildren))}::text[]`;
2243
+ const { rows: ambiguous } = await client.query(`${mappingCTE}
2244
+ SELECT oo.child_workflow_id
2245
+ FROM mapping m
2246
+ JOIN "${this.schemaName}".operation_outputs oo
2247
+ ON ${this.scoped('oo', ambiguityParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step
2248
+ WHERE oo.child_workflow_id = ANY(${replaced})
2249
+ GROUP BY oo.child_workflow_id
2250
+ HAVING COUNT(DISTINCT ${childKey}) > 1
2251
+ LIMIT 1`, ambiguityParams);
2252
+ if (ambiguous.length > 0) {
2253
+ throw new error_1.DBOSError(`Can't replace child workflow ${ambiguous[0].child_workflow_id}: the forked steps refer to children with that ID under more than one shard key`);
2254
+ }
2255
+ }
1784
2256
  // Build the child_workflow_id expression, applying replacements if provided.
2257
+ const ooParams = copyParams();
1785
2258
  let childWfExpr = 'oo.child_workflow_id';
1786
- const ooParams = [...mappingParams];
1787
2259
  if (options.replacementChildren && Object.keys(options.replacementChildren).length > 0) {
1788
2260
  const whenClauses = [];
1789
2261
  for (const [oldId, newId] of Object.entries(options.replacementChildren)) {
1790
- whenClauses.push(`WHEN oo.child_workflow_id = $${mIdx} THEN $${mIdx + 1}::text`);
1791
- ooParams.push(oldId, newId);
1792
- mIdx += 2;
2262
+ whenClauses.push(`WHEN oo.child_workflow_id = $${ooParams.push(oldId)} THEN $${ooParams.push(newId)}::text`);
1793
2263
  }
1794
2264
  childWfExpr = `CASE ${whenClauses.join(' ')} ELSE oo.child_workflow_id END`;
1795
2265
  }
2266
+ const ooKeys = keyed(ooParams);
1796
2267
  // Copy operation outputs
1797
2268
  await client.query(`${mappingCTE}
1798
2269
  INSERT INTO "${this.schemaName}".operation_outputs
1799
- (workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp)
1800
- SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint
2270
+ (${ooKeys.names}workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? ', child_shard_key' : ''})
2271
+ SELECT ${ooKeys.exprs}m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint${childKey !== undefined ? `, ${childKey}` : ''}
1801
2272
  FROM mapping m
1802
2273
  JOIN "${this.schemaName}".operation_outputs oo
1803
- ON oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
2274
+ ON ${this.scoped('oo', ooParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
1804
2275
  // Copy streams
2276
+ const streamParams = copyParams();
2277
+ const streamKeys = keyed(streamParams);
1805
2278
  await client.query(`${mappingCTE}
1806
2279
  INSERT INTO "${this.schemaName}".streams
1807
- (workflow_uuid, key, value, serialization, "offset", function_id)
1808
- SELECT m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
2280
+ (${streamKeys.names}workflow_uuid, key, value, serialization, "offset", function_id)
2281
+ SELECT ${streamKeys.exprs}m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
1809
2282
  FROM mapping m
1810
2283
  JOIN "${this.schemaName}".streams s
1811
- ON s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`, mappingParams);
2284
+ ON ${this.scoped('s', streamParams)} AND s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`, streamParams);
1812
2285
  // Copy events history
2286
+ const historyParams = copyParams();
2287
+ const historyKeys = keyed(historyParams);
1813
2288
  await client.query(`${mappingCTE}
1814
2289
  INSERT INTO "${this.schemaName}".workflow_events_history
1815
- (workflow_uuid, function_id, key, value, serialization)
1816
- SELECT m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
2290
+ (${historyKeys.names}workflow_uuid, function_id, key, value, serialization)
2291
+ SELECT ${historyKeys.exprs}m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
1817
2292
  FROM mapping m
1818
2293
  JOIN "${this.schemaName}".workflow_events_history weh
1819
- ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`, mappingParams);
2294
+ ON ${this.scoped('weh', historyParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`, historyParams);
1820
2295
  // Copy only the latest version of each event using a window function
2296
+ const eventParams = copyParams();
2297
+ const eventKeys = keyed(eventParams);
1821
2298
  await client.query(`${mappingCTE}
1822
2299
  INSERT INTO "${this.schemaName}".workflow_events
1823
- (workflow_uuid, key, value, serialization)
1824
- SELECT ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
2300
+ (${eventKeys.names}workflow_uuid, key, value, serialization)
2301
+ SELECT ${eventKeys.exprs}ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
1825
2302
  FROM (
1826
2303
  SELECT m.fork_id AS workflow_uuid, weh.key, weh.value, weh.serialization,
1827
- ROW_NUMBER() OVER (PARTITION BY weh.workflow_uuid, weh.key ORDER BY weh.function_id DESC) AS rn
2304
+ ROW_NUMBER() OVER (PARTITION BY m.fork_id, weh.key ORDER BY weh.function_id DESC) AS rn
1828
2305
  FROM mapping m
1829
2306
  JOIN "${this.schemaName}".workflow_events_history weh
1830
- ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
2307
+ ON ${this.scoped('weh', eventParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
1831
2308
  ) ranked
1832
- WHERE ranked.rn = 1`, mappingParams);
2309
+ WHERE ranked.rn = 1`, eventParams);
1833
2310
  }
1834
2311
  await client.query('COMMIT');
1835
2312
  return forkedWorkflowIDs;
@@ -1952,27 +2429,36 @@ class SystemDatabase {
1952
2429
  }
1953
2430
  /** Cancellation check for polling waits: goes through the limiter so readers cannot starve the pool. */
1954
2431
  async checkIfCanceledLimited(workflowID) {
1955
- await this.#pollWithLimiter(() => this.#checkIfCanceled(this.pool, workflowID));
2432
+ // The caller's own workflow, even where a same-ID workflow is bound as a target.
2433
+ const params = [];
2434
+ const target = this.ownKey(workflowID, undefined, () => this.wf(undefined, workflowID, params));
2435
+ const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${target}`, params));
2436
+ if (rows[0]?.status === workflow_1.StatusString.CANCELLED) {
2437
+ throw new error_1.DBOSWorkflowCancelledError(workflowID);
2438
+ }
1956
2439
  }
1957
2440
  watchForCancellation(workflowID) {
1958
2441
  const controller = new AbortController();
1959
2442
  let state = 'idle';
2443
+ // The running workflow's own key, resolved now for the poller that reads it later outside this context.
2444
+ const shardKey = this.ownKey(workflowID, undefined, () => this.keyHint(workflowID));
2445
+ const waiterKey = this.ownKey(workflowID, undefined, () => this.waiter(workflowID, ''));
1960
2446
  const start = () => {
1961
2447
  state = 'watching';
1962
2448
  if (this.#destroyed)
1963
2449
  return;
1964
- let watchers = this.#cancelWatchers.get(workflowID);
2450
+ let watchers = this.#cancelWatchers.get(waiterKey);
1965
2451
  if (watchers === undefined) {
1966
- watchers = new Set();
1967
- this.#cancelWatchers.set(workflowID, watchers);
2452
+ watchers = { workflowID, shardKey, controllers: new Set() };
2453
+ this.#cancelWatchers.set(waiterKey, watchers);
1968
2454
  }
1969
- watchers.add(controller);
1970
- this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => this.#runCancelPoller());
2455
+ watchers.controllers.add(controller);
2456
+ this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => (0, shard_keys_1.withClearedShardKeyState)(() => this.#runCancelPoller()));
1971
2457
  };
1972
2458
  const stop = () => {
1973
- const watchers = this.#cancelWatchers.get(workflowID);
1974
- if (watchers?.delete(controller) && watchers.size === 0) {
1975
- this.#cancelWatchers.delete(workflowID);
2459
+ const watchers = this.#cancelWatchers.get(waiterKey);
2460
+ if (watchers?.controllers.delete(controller) && watchers.controllers.size === 0) {
2461
+ this.#cancelWatchers.delete(waiterKey);
1976
2462
  if (this.#cancelWatchers.size === 0)
1977
2463
  this.#cancelPollerWake?.();
1978
2464
  }
@@ -2011,15 +2497,36 @@ class SystemDatabase {
2011
2497
  this.#cancelPollerLoop = undefined;
2012
2498
  }
2013
2499
  async #abortCancelledWatchers() {
2014
- const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1) AND status = $2`, [Array.from(this.#cancelWatchers.keys()), workflow_1.StatusString.CANCELLED]));
2015
- for (const { workflow_uuid: workflowID } of rows) {
2016
- const watchers = this.#cancelWatchers.get(workflowID);
2017
- if (watchers === undefined)
2500
+ // One read per shard key, each bound to its own workflows.
2501
+ const byKey = new Map();
2502
+ for (const { workflowID, shardKey } of this.#cancelWatchers.values()) {
2503
+ const ids = byKey.get(shardKey) ?? [];
2504
+ ids.push(workflowID);
2505
+ byKey.set(shardKey, ids);
2506
+ }
2507
+ for (const [shardKey, workflowIDs] of byKey) {
2508
+ // Each key's read on its own, so one key's failure doesn't hide another key's cancellations.
2509
+ let rows;
2510
+ try {
2511
+ ({ rows } = await (0, shard_keys_1.bindShardKeys)(workflowIDs.map((id) => [id, shardKey]), () => {
2512
+ const params = [workflow_1.StatusString.CANCELLED];
2513
+ const ids = this.wfIn(undefined, workflowIDs, params);
2514
+ return this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${ids} AND status = $1`, params));
2515
+ }));
2516
+ }
2517
+ catch (e) {
2518
+ this.logger.warn(`Error polling status for cancellable steps: ${String(e)}`);
2018
2519
  continue;
2019
- this.#cancelWatchers.delete(workflowID);
2020
- const reason = new error_1.DBOSWorkflowCancelledError(workflowID);
2021
- for (const controller of watchers)
2022
- controller.abort(reason);
2520
+ }
2521
+ const cancelled = new Set(rows.map((row) => row.workflow_uuid));
2522
+ for (const [waiterKey, watchers] of this.#cancelWatchers) {
2523
+ if (watchers.shardKey !== shardKey || !cancelled.has(watchers.workflowID))
2524
+ continue;
2525
+ this.#cancelWatchers.delete(waiterKey);
2526
+ const reason = new error_1.DBOSWorkflowCancelledError(watchers.workflowID);
2527
+ for (const controller of watchers.controllers)
2528
+ controller.abort(reason);
2529
+ }
2023
2530
  }
2024
2531
  }
2025
2532
  // A missing row normally means the workflow has not been inserted yet, so
@@ -2031,21 +2538,23 @@ class SystemDatabase {
2031
2538
  const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
2032
2539
  let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
2033
2540
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
2034
- // Record the durable timeout deadline once before polling. #durableSleep persists it on the
2541
+ // Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
2035
2542
  // first call and reads back the same value on recovery, so it never changes across iterations.
2036
2543
  if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
2037
- finishTime = await this.#durableSleep(callerID, timerFuncID, timeoutms);
2544
+ finishTime = await this.durableSleepInternal(callerID, timerFuncID, timeoutms);
2038
2545
  }
2039
2546
  while (true) {
2040
2547
  if (callerID)
2041
2548
  await this.checkIfCanceledLimited(callerID);
2042
2549
  let rows;
2043
2550
  try {
2551
+ const params = [];
2552
+ const target = this.wf('ws', workflowID, params);
2044
2553
  ({ rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT ws.status, COALESCE(wo.output, ws.output) AS output,
2045
2554
  COALESCE(wo.error, ws.error) AS error, ws.serialization
2046
2555
  FROM "${this.schemaName}".workflow_status ws
2047
- LEFT JOIN "${this.schemaName}".workflow_output wo ON wo.workflow_uuid = ws.workflow_uuid
2048
- WHERE ws.workflow_uuid=$1`, [workflowID])));
2556
+ LEFT JOIN "${this.schemaName}".workflow_output wo ON ${this.wfJoin('ws', 'wo')}
2557
+ WHERE ${target}`, params)));
2049
2558
  }
2050
2559
  catch (e) {
2051
2560
  const err = e;
@@ -2082,21 +2591,27 @@ class SystemDatabase {
2082
2591
  }
2083
2592
  }
2084
2593
  async awaitFirstWorkflowId(workflowIds, callerID, pollingIntervalMs) {
2085
- const placeholders = workflowIds.map((_, i) => `$${i + 1}`).join(', ');
2086
2594
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
2087
2595
  while (true) {
2088
2596
  if (callerID)
2089
2597
  await this.checkIfCanceledLimited(callerID);
2090
- const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2091
- WHERE workflow_uuid IN (${placeholders})
2092
- AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
2093
- LIMIT 1`, workflowIds));
2094
- if (rows.length > 0) {
2095
- return rows[0].workflow_uuid;
2598
+ const first = await this.checkFirstWorkflowId(workflowIds);
2599
+ if (first !== undefined) {
2600
+ return first;
2096
2601
  }
2097
2602
  await (0, utils_1.sleepms)(pollIntervalMs);
2098
2603
  }
2099
2604
  }
2605
+ /** The first of these workflows found complete, if any. */
2606
+ async checkFirstWorkflowId(workflowIds) {
2607
+ const params = [];
2608
+ const ids = this.wfIn(undefined, workflowIds, params);
2609
+ const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2610
+ WHERE ${ids}
2611
+ AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
2612
+ LIMIT 1`, params));
2613
+ return rows[0]?.workflow_uuid;
2614
+ }
2100
2615
  async awaitWorkflowIds(workflowIds, callerID, pollingIntervalMs) {
2101
2616
  const remainingWorkflowIds = new Set(workflowIds);
2102
2617
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
@@ -2104,11 +2619,8 @@ class SystemDatabase {
2104
2619
  const currentWorkflowIds = [...remainingWorkflowIds];
2105
2620
  if (callerID)
2106
2621
  await this.checkIfCanceledLimited(callerID);
2107
- const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2108
- WHERE workflow_uuid = ANY($1::text[])
2109
- AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, [currentWorkflowIds]));
2110
- for (const row of rows) {
2111
- remainingWorkflowIds.delete(row.workflow_uuid);
2622
+ for (const workflowID of await this.checkCompletedWorkflowIds(currentWorkflowIds)) {
2623
+ remainingWorkflowIds.delete(workflowID);
2112
2624
  }
2113
2625
  if (remainingWorkflowIds.size === 0) {
2114
2626
  return;
@@ -2116,9 +2628,18 @@ class SystemDatabase {
2116
2628
  await (0, utils_1.sleepms)(pollIntervalMs);
2117
2629
  }
2118
2630
  }
2631
+ /** Those of these workflows found complete. */
2632
+ async checkCompletedWorkflowIds(workflowIds) {
2633
+ const params = [];
2634
+ const ids = this.wfIn(undefined, workflowIds, params);
2635
+ const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2636
+ WHERE ${ids}
2637
+ AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, params));
2638
+ return rows.map((row) => row.workflow_uuid);
2639
+ }
2119
2640
  // ==================== Sleep ====================
2120
2641
  async durableSleepms(workflowID, functionID, durationMS) {
2121
- const endTime = await this.#durableSleep(workflowID, functionID, durationMS, true);
2642
+ const endTime = await this.durableSleepInternal(workflowID, functionID, durationMS, true);
2122
2643
  while (Date.now() < endTime) {
2123
2644
  await (0, utils_1.sleepms)(Math.min(endTime - Date.now(), utils_1.sleepConfig.maxTimeoutMS));
2124
2645
  }
@@ -2126,31 +2647,28 @@ class SystemDatabase {
2126
2647
  }
2127
2648
  // ==================== Messaging ====================
2128
2649
  nullTopic = '__null__topic__';
2129
- async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey) {
2130
- topic = topic ?? this.nullTopic;
2131
- const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
2650
+ async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey,
2651
+ // The destination's shard key; required on a sharded system database.
2652
+ shardKey) {
2132
2653
  const startTime = Date.now();
2133
2654
  let missing;
2134
2655
  const client = await this.connect();
2135
2656
  try {
2136
2657
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
2137
2658
  await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
2138
- await client.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
2139
- VALUES ($1, $2, $3, $4, $5)
2140
- ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
2659
+ await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2141
2660
  return undefined;
2142
2661
  });
2143
2662
  await client.query('COMMIT');
2144
2663
  }
2145
2664
  catch (error) {
2146
2665
  await client.query('ROLLBACK');
2147
- const err = error;
2148
- if (err.code === '23503') {
2666
+ if (error instanceof error_1.DBOSNonExistentWorkflowError) {
2149
2667
  // Foreign key constraint violation (only expected for the INSERT query)
2150
- missing = new error_1.DBOSNonExistentWorkflowError(`Sent to non-existent destination workflow UUID: ${destinationID}`);
2668
+ missing = error;
2151
2669
  }
2152
2670
  else {
2153
- throw err;
2671
+ throw error;
2154
2672
  }
2155
2673
  }
2156
2674
  finally {
@@ -2163,22 +2681,22 @@ class SystemDatabase {
2163
2681
  }
2164
2682
  }
2165
2683
  /** Runs on `client` if given, joining its transaction; otherwise on the pool with retries. */
2166
- async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client) {
2684
+ async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client, shardKey) {
2167
2685
  if (client !== undefined) {
2168
- return await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
2686
+ return await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2169
2687
  }
2170
- return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey);
2688
+ return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey);
2171
2689
  }
2172
2690
  /** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
2173
- async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
2691
+ async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
2174
2692
  const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
2175
2693
  const client = await this.connect();
2176
2694
  try {
2177
- await this.#inTransaction(client, async () => {
2178
- await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
2695
+ await this.inTransaction(client, async () => {
2696
+ await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2179
2697
  // After the insert, in the order the recorded send locks, so they cannot deadlock.
2180
2698
  if (ownerXid !== undefined) {
2181
- await this.#checkOwner(client, workflowID, ownerXid);
2699
+ await this.checkOwner(client, workflowID, ownerXid);
2182
2700
  }
2183
2701
  });
2184
2702
  }
@@ -2186,17 +2704,23 @@ class SystemDatabase {
2186
2704
  client.release();
2187
2705
  }
2188
2706
  }
2189
- async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey) {
2190
- return await this.#sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey);
2707
+ async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey) {
2708
+ return await this.sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2191
2709
  }
2192
- async #sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey) {
2710
+ async sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
2193
2711
  topic = topic ?? this.nullTopic;
2194
- // Same per-destination scoping as send() above.
2712
+ // An idempotency key is scoped per destination: suffix it with the recipient's workflow ID.
2195
2713
  const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
2714
+ const values = {
2715
+ ...this.wfValues(destinationID, shardKey),
2716
+ destination_uuid: destinationID,
2717
+ topic,
2718
+ message,
2719
+ serialization,
2720
+ message_uuid: messageUUID,
2721
+ };
2196
2722
  try {
2197
- await db.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
2198
- VALUES ($1, $2, $3, $4, $5)
2199
- ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
2723
+ await this.insertRows(db, 'notifications', Object.keys(values), [Object.values(values)], `ON CONFLICT (${this.keyConflict('message_uuid')}) DO NOTHING`);
2200
2724
  }
2201
2725
  catch (error) {
2202
2726
  const err = error;
@@ -2220,10 +2744,10 @@ class SystemDatabase {
2220
2744
  const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
2221
2745
  let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
2222
2746
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
2223
- // Record the durable timeout deadline once before polling. #durableSleep persists it on the
2747
+ // Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
2224
2748
  // first call and reads back the same value on recovery, so it never changes across iterations.
2225
2749
  if (timeoutms) {
2226
- finishTime = await this.#durableSleep(workflowID, timeoutFunctionID, timeoutms);
2750
+ finishTime = await this.durableSleepInternal(workflowID, timeoutFunctionID, timeoutms);
2227
2751
  }
2228
2752
  while (true) {
2229
2753
  // register the key with the global notifications listener.
@@ -2231,12 +2755,14 @@ class SystemDatabase {
2231
2755
  const messagePromise = new Promise((resolve) => {
2232
2756
  resolveNotification = resolve;
2233
2757
  });
2234
- const payload = `${workflowID}::${topic}`;
2758
+ const payload = this.waiter(workflowID, topic);
2235
2759
  const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
2236
2760
  try {
2237
2761
  await this.checkIfCanceledLimited(workflowID);
2238
2762
  // Check if the key is already in the DB, then wait for the notification if it isn't.
2239
- const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE destination_uuid=$1 AND topic=$2 AND consumed = false;`, [workflowID, topic]))).rows;
2763
+ const params = [topic];
2764
+ const target = this.wf(undefined, workflowID, params, 'destination_uuid');
2765
+ const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE ${target} AND topic=$1 AND consumed = false;`, params))).rows;
2240
2766
  if (initRecvRows.length !== 0)
2241
2767
  break;
2242
2768
  const ct = Date.now();
@@ -2263,21 +2789,22 @@ class SystemDatabase {
2263
2789
  const client = await this.connect();
2264
2790
  try {
2265
2791
  await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
2792
+ const consumeParams = [topic, functionID];
2266
2793
  const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
2267
- SET consumed = true, consumed_by_function_id = $3
2268
- WHERE destination_uuid = $1
2269
- AND topic = $2
2794
+ SET consumed = true, consumed_by_function_id = $2
2795
+ WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
2796
+ AND topic = $1
2270
2797
  AND consumed = false
2271
2798
  AND message_uuid = (
2272
2799
  SELECT message_uuid
2273
2800
  FROM "${this.schemaName}".notifications
2274
- WHERE destination_uuid = $1
2275
- AND topic = $2
2801
+ WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
2802
+ AND topic = $1
2276
2803
  AND consumed = false
2277
2804
  ORDER BY created_at_epoch_ms ASC
2278
2805
  LIMIT 1
2279
2806
  )
2280
- RETURNING notifications.message, notifications.serialization;`, [workflowID, topic, functionID])).rows;
2807
+ RETURNING notifications.message, notifications.serialization;`, consumeParams)).rows;
2281
2808
  if (finalRecvRows.length > 0) {
2282
2809
  message = finalRecvRows[0].message;
2283
2810
  serialization = finalRecvRows[0].serialization;
@@ -2306,16 +2833,19 @@ class SystemDatabase {
2306
2833
  // Only a real write (not a replay) should wake readers.
2307
2834
  let didWrite = false;
2308
2835
  await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SETEVENT, workflowID, functionID, async () => {
2309
- await client.query(`INSERT INTO "${this.schemaName}".workflow_events (workflow_uuid, key, value, serialization)
2310
- VALUES ($1, $2, $3, $4)
2311
- ON CONFLICT (workflow_uuid, key)
2836
+ const keys = this.wfValues(workflowID);
2837
+ const keyColumns = Object.keys(keys).map((c) => `${c}, `);
2838
+ const keyPlaceholders = (from) => Object.keys(keys).map((_, i) => `$${from + i}, `);
2839
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_events (${keyColumns.join('')}workflow_uuid, key, value, serialization)
2840
+ VALUES (${keyPlaceholders(5).join('')}$1, $2, $3, $4)
2841
+ ON CONFLICT (${this.keyConflict('workflow_uuid', 'key')})
2312
2842
  DO UPDATE SET value = $3, serialization = $4
2313
- RETURNING workflow_uuid;`, [workflowID, key, message, serialization]);
2843
+ RETURNING workflow_uuid;`, [workflowID, key, message, serialization, ...Object.values(keys)]);
2314
2844
  // Also write to the immutable history table for fork support
2315
- await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (workflow_uuid, function_id, key, value, serialization)
2316
- VALUES ($1, $2, $3, $4, $5)
2317
- ON CONFLICT (workflow_uuid, function_id, key)
2318
- DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization]);
2845
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (${keyColumns.join('')}workflow_uuid, function_id, key, value, serialization)
2846
+ VALUES (${keyPlaceholders(6).join('')}$1, $2, $3, $4, $5)
2847
+ ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id', 'key')})
2848
+ DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization, ...Object.values(keys)]);
2319
2849
  didWrite = true;
2320
2850
  return undefined;
2321
2851
  });
@@ -2349,15 +2879,15 @@ class SystemDatabase {
2349
2879
  // Get the return the value. if it's in the DB, otherwise return null.
2350
2880
  let value = null;
2351
2881
  let valueSer = null;
2352
- const payloadKey = `${workflowID}::${key}`;
2882
+ const payloadKey = this.waiter(workflowID, key);
2353
2883
  const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
2354
2884
  let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
2355
2885
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
2356
2886
  // If we have a callerWorkflow, we want a durable sleep, otherwise, not. Record the durable
2357
- // timeout deadline once before polling. #durableSleep persists it on the first call and reads
2887
+ // timeout deadline once before polling. durableSleepInternal persists it on the first call and reads
2358
2888
  // back the same value on recovery, so it never changes across iterations.
2359
2889
  if (callerWorkflow && timeoutms) {
2360
- finishTime = await this.#durableSleep(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms);
2890
+ finishTime = await this.durableSleepInternal(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms);
2361
2891
  }
2362
2892
  // Register the key with the global notifications listener first... we do not want to look in the DB first
2363
2893
  // or that would cause a timing hole.
@@ -2371,9 +2901,11 @@ class SystemDatabase {
2371
2901
  if (callerWorkflow?.workflowID)
2372
2902
  await this.checkIfCanceledLimited(callerWorkflow?.workflowID);
2373
2903
  // Check if the key is already in the DB, then wait for the notification if it isn't.
2904
+ const params = [key];
2905
+ const target = this.wf(undefined, workflowID, params);
2374
2906
  const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT key, value, serialization
2375
2907
  FROM "${this.schemaName}".workflow_events
2376
- WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key]))).rows;
2908
+ WHERE ${target} AND key=$1;`, params))).rows;
2377
2909
  if (initRecvRows.length > 0) {
2378
2910
  value = initRecvRows[0].value;
2379
2911
  valueSer = initRecvRows[0].serialization;
@@ -2412,15 +2944,12 @@ class SystemDatabase {
2412
2944
  try {
2413
2945
  while (true) {
2414
2946
  try {
2415
- await this.#inTransaction(client, async () => {
2947
+ await this.inTransaction(client, async () => {
2416
2948
  // Derives the first unused offset inside the insert; two writers can still pick the same one.
2417
- await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
2418
- SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
2419
- FROM "${this.schemaName}".streams s
2420
- WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
2949
+ await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
2421
2950
  // After the insert, in the order the workflow-level writes lock, so they cannot deadlock.
2422
2951
  if (ownerXid !== undefined) {
2423
- await this.#checkOwner(client, workflowID, ownerXid);
2952
+ await this.checkOwner(client, workflowID, ownerXid);
2424
2953
  }
2425
2954
  });
2426
2955
  }
@@ -2454,10 +2983,7 @@ class SystemDatabase {
2454
2983
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
2455
2984
  await this.#runAndRecordResult(client, functionName, workflowID, functionID, async () => {
2456
2985
  // Derives the first unused offset inside the insert; two writers can still pick the same one.
2457
- await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
2458
- SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
2459
- FROM "${this.schemaName}".streams s
2460
- WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
2986
+ await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
2461
2987
  didWrite = true;
2462
2988
  return undefined;
2463
2989
  });
@@ -2489,6 +3015,15 @@ class SystemDatabase {
2489
3015
  client.release();
2490
3016
  }
2491
3017
  }
3018
+ /** Append a value to a stream, at the first unused offset, derived inside the insert. */
3019
+ async #insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization) {
3020
+ const params = [key, serializedValue, functionID, serialization, workflowID];
3021
+ const keyColumns = this.wfColumns(workflowID, params);
3022
+ await client.query(`INSERT INTO "${this.schemaName}".streams (${keyColumns.map(([c]) => `${c}, `).join('')}workflow_uuid, key, value, "offset", function_id, serialization)
3023
+ SELECT ${keyColumns.map(([, e]) => `${e}, `).join('')}$5::text, $1::text, $2::text, COALESCE(MAX(s."offset"), -1) + 1, $3::int, $4::text
3024
+ FROM "${this.schemaName}".streams s
3025
+ WHERE ${this.wf('s', workflowID, params)} AND s.key = $1`, params);
3026
+ }
2492
3027
  async closeStreamFromWorkflow(workflowID, functionID, key) {
2493
3028
  await this.writeStreamFromWorkflow(workflowID, functionID, key, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name(), exports.DBOS_FUNCNAME_CLOSESTREAM);
2494
3029
  }
@@ -2498,13 +3033,15 @@ class SystemDatabase {
2498
3033
  // Read the value at `offset` and the workflow's status in one query: status null = no such workflow, value undefined = nothing at that offset.
2499
3034
  async readStreamValue(workflowID, key, offset) {
2500
3035
  // LEFT JOIN so a workflow with nothing at offset still reports its status (single PK lookup); under the poll limiter, inside @dbRetry so the permit frees across backoff.
3036
+ const params = [key, offset];
3037
+ const target = this.wf('ws', workflowID, params);
2501
3038
  const result = await this.#pollWithLimiter(() => this.pool.query(
2502
3039
  // "offset" is a reserved word, so alias it (stream_offset) to read it back plainly.
2503
3040
  `SELECT ws.status AS status, s.value AS value, s.serialization AS serialization, s."offset" AS stream_offset
2504
3041
  FROM "${this.schemaName}".workflow_status ws
2505
3042
  LEFT OUTER JOIN "${this.schemaName}".streams s
2506
- ON s.workflow_uuid = ws.workflow_uuid AND s.key = $2 AND s."offset" = $3
2507
- WHERE ws.workflow_uuid = $1`, [workflowID, key, offset]));
3043
+ ON ${this.wfJoin('s', 'ws')} AND s.key = $1 AND s."offset" = $2
3044
+ WHERE ${target}`, params));
2508
3045
  if (result.rows.length === 0) {
2509
3046
  return { status: null, value: undefined };
2510
3047
  }
@@ -2641,7 +3178,7 @@ class SystemDatabase {
2641
3178
  // Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v3, so cost scales with the number of partitions rather than the backlog depth.
2642
3179
  const params = [queueName, workflow_1.StatusString.ENQUEUED];
2643
3180
  // Only partitions this application can actually dequeue from.
2644
- const scope = this.appNameFilter('application_name', this.appName, params);
3181
+ const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
2645
3182
  const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
2646
3183
  (SELECT MIN(queue_partition_key) AS pk
2647
3184
  FROM "${this.schemaName}".workflow_status
@@ -2658,7 +3195,14 @@ class SystemDatabase {
2658
3195
  }
2659
3196
  async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0,
2660
3197
  // Written to every claimed row: the dispatches of this claim own their executions.
2661
- ownerXid = null) {
3198
+ ownerXid = null, isLatestVersion) {
3199
+ const { claimedIDs } = await this.claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount, partitionLocalRunningCount, ownerXid, isLatestVersion);
3200
+ return claimedIDs;
3201
+ }
3202
+ /** Claim this queue's next workflows, plus the limit that left no room to claim any (undefined if every limit had room). */
3203
+ async claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0, ownerXid = null, isLatestVersion) {
3204
+ // Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
3205
+ const startTimeMs = Date.now();
2662
3206
  const claimedIDs = [];
2663
3207
  const partitionParams = queuePartitionKey !== undefined ? [queuePartitionKey] : [];
2664
3208
  // Shares a concurrency or rate limit budget with other executors.
@@ -2668,6 +3212,10 @@ class SystemDatabase {
2668
3212
  queue.partitionRateLimit !== undefined;
2669
3213
  // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
2670
3214
  const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
3215
+ // A sharded database keeps the version row under another key, which the claim's transaction cannot span.
3216
+ if (isLatestVersion === undefined && this.sharded) {
3217
+ isLatestVersion = await this.isLatestVersion(appVersion);
3218
+ }
2671
3219
  const client = await this.connect();
2672
3220
  try {
2673
3221
  // Default to READ COMMITTED except with a budget shared across executors
@@ -2681,7 +3229,7 @@ class SystemDatabase {
2681
3229
  const rateLimitRemaining = async (rateLimit, partitionScoped) => {
2682
3230
  const params = [queue.name, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED, rateLimit.periodSec * 1000];
2683
3231
  // Count only what this application would dequeue, matching the select below.
2684
- const scope = this.appNameFilter('application_name', this.appName, params);
3232
+ const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
2685
3233
  const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
2686
3234
  const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
2687
3235
  WHERE queue_name = $1
@@ -2700,7 +3248,7 @@ class SystemDatabase {
2700
3248
  */
2701
3249
  const pendingCount = async (partitionScoped) => {
2702
3250
  const params = [queue.name, workflow_1.StatusString.PENDING];
2703
- const scope = this.appNameFilter('application_name', this.appName, params);
3251
+ const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
2704
3252
  const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
2705
3253
  const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
2706
3254
  WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
@@ -2708,27 +3256,38 @@ class SystemDatabase {
2708
3256
  };
2709
3257
  // Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
2710
3258
  let maxTasks = Infinity;
3259
+ // The first limit found with no room, which the caller reads as why nothing was claimed.
3260
+ let stopReason = undefined;
3261
+ const limit = (room, reason) => {
3262
+ maxTasks = Math.min(maxTasks, room);
3263
+ if (maxTasks <= 0)
3264
+ stopReason ??= reason;
3265
+ };
2711
3266
  if (queue.workerConcurrency !== undefined) {
2712
3267
  // Use the in-memory registry for this worker's running count — avoids a DB round trip.
2713
- maxTasks = Math.min(maxTasks, Math.max(0, queue.workerConcurrency - localRunningCount));
3268
+ limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
2714
3269
  }
2715
3270
  if (queue.partitionWorkerConcurrency !== undefined) {
2716
- maxTasks = Math.min(maxTasks, Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount));
3271
+ limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
2717
3272
  }
2718
3273
  if (maxTasks <= 0) {
2719
3274
  await client.query('COMMIT');
2720
- return claimedIDs;
3275
+ return { claimedIDs, stopReason };
2721
3276
  }
2722
3277
  if (queue.rateLimit !== undefined) {
2723
3278
  // Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
2724
- maxTasks = Math.min(maxTasks, await rateLimitRemaining(queue.rateLimit, false));
3279
+ limit(await rateLimitRemaining(queue.rateLimit, false), 'limiter');
3280
+ if (maxTasks <= 0) {
3281
+ await client.query('COMMIT');
3282
+ return { claimedIDs, stopReason };
3283
+ }
2725
3284
  }
2726
3285
  if (queue.partitionRateLimit !== undefined) {
2727
- maxTasks = Math.min(maxTasks, await rateLimitRemaining(queue.partitionRateLimit, true));
3286
+ limit(await rateLimitRemaining(queue.partitionRateLimit, true), 'partition_limiter');
2728
3287
  }
2729
3288
  if (maxTasks <= 0) {
2730
3289
  await client.query('COMMIT');
2731
- return claimedIDs;
3290
+ return { claimedIDs, stopReason };
2732
3291
  }
2733
3292
  if (queue.concurrency !== undefined) {
2734
3293
  // Global concurrency still requires a DB query since other workers may be running workflows too.
@@ -2736,23 +3295,26 @@ class SystemDatabase {
2736
3295
  if (totalRunningTasks > queue.concurrency) {
2737
3296
  this.logger.warn(`Total running tasks (${totalRunningTasks}) exceeds the global concurrency limit (${queue.concurrency})`);
2738
3297
  }
2739
- maxTasks = Math.min(maxTasks, Math.max(0, queue.concurrency - totalRunningTasks));
3298
+ limit(Math.max(0, queue.concurrency - totalRunningTasks), 'concurrency');
3299
+ if (maxTasks <= 0) {
3300
+ await client.query('COMMIT');
3301
+ return { claimedIDs, stopReason };
3302
+ }
2740
3303
  }
2741
3304
  if (queue.partitionConcurrency !== undefined) {
2742
3305
  const partitionRunningTasks = await pendingCount(true);
2743
3306
  if (partitionRunningTasks > queue.partitionConcurrency) {
2744
3307
  this.logger.warn(`Total running tasks (${partitionRunningTasks}) on partition ${queuePartitionKey} of queue ${queue.name} exceeds the partition concurrency limit (${queue.partitionConcurrency})`);
2745
3308
  }
2746
- maxTasks = Math.min(maxTasks, Math.max(0, queue.partitionConcurrency - partitionRunningTasks));
3309
+ limit(Math.max(0, queue.partitionConcurrency - partitionRunningTasks), 'partition_concurrency');
2747
3310
  }
2748
3311
  // Return immediately if there are no available tasks due to flow control limits
2749
3312
  if (maxTasks <= 0) {
2750
3313
  await client.query('COMMIT');
2751
- return claimedIDs;
3314
+ return { claimedIDs, stopReason };
2752
3315
  }
2753
3316
  // Retrieve the first max_tasks workflows in the queue.
2754
- const latestVersion = await this.#latestApplicationVersionName(client);
2755
- const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
3317
+ isLatestVersion ??= await this.isLatestVersion(appVersion, client);
2756
3318
  const versionClause = isLatestVersion
2757
3319
  ? '(application_version = $3 OR application_version IS NULL)'
2758
3320
  : 'application_version = $3';
@@ -2762,13 +3324,17 @@ class SystemDatabase {
2762
3324
  const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
2763
3325
  const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
2764
3326
  const selectScope = this.appNameFilter('application_name', this.appName, selectParams);
3327
+ const keyScope = this.scoped(undefined, selectParams);
3328
+ const claimGate = this.claimGate(queue.name, queuePartitionKey, selectParams);
2765
3329
  const selectQuery = `
2766
3330
  SELECT workflow_uuid
2767
3331
  FROM "${this.schemaName}".workflow_status
2768
- WHERE status = $1
3332
+ WHERE ${keyScope}
3333
+ AND status = $1
2769
3334
  AND queue_name = $2
2770
3335
  AND ${versionClause}
2771
3336
  AND ${selectScope}
3337
+ AND ${claimGate}
2772
3338
  ${queuePartitionKey !== undefined ? 'AND queue_partition_key = $4' : ''}
2773
3339
  ORDER BY priority ASC, created_at ASC
2774
3340
  ${limitClause}
@@ -2787,31 +3353,36 @@ class SystemDatabase {
2787
3353
  executorID,
2788
3354
  appVersion,
2789
3355
  queue.rateLimit !== undefined || queue.partitionRateLimit !== undefined,
2790
- workflowIDs,
2791
3356
  workflow_1.StatusString.ENQUEUED,
2792
3357
  // Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
2793
3358
  this.appName ?? null,
2794
3359
  ownerXid,
3360
+ startTimeMs,
2795
3361
  ];
3362
+ const claimedIDsFilter = this.wfIn(undefined, workflowIDs, updateParams);
2796
3363
  // Re-check ownership alongside status, as the partitioned claim guard does.
2797
3364
  const claimScope = this.appNameFilter('application_name', this.appName, updateParams);
2798
3365
  // RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
2799
3366
  const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
2800
3367
  SET status = $1,
2801
- owner_xid = $8,
3368
+ owner_xid = $7,
2802
3369
  executor_id = $2,
2803
3370
  application_version = $3,
2804
3371
  started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2805
3372
  rate_limited = $4,
2806
- application_name = COALESCE(application_name, $7),
3373
+ application_name = COALESCE(application_name, $6),
2807
3374
  recovery_attempts = recovery_attempts + 1,
2808
3375
  updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2809
3376
  workflow_deadline_epoch_ms = CASE
2810
- WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
2811
- THEN (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
3377
+ WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
3378
+ CASE
3379
+ WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
3380
+ THEN 9223372036854775807
3381
+ ELSE $8::bigint + workflow_timeout_ms
3382
+ END
2812
3383
  ELSE workflow_deadline_epoch_ms
2813
3384
  END
2814
- WHERE workflow_uuid = ANY($5::text[]) AND status = $6 AND ${claimScope}
3385
+ WHERE ${claimedIDsFilter} AND status = $5 AND ${claimScope}
2815
3386
  RETURNING workflow_uuid`, updateParams);
2816
3387
  const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
2817
3388
  for (const id of workflowIDs) {
@@ -2829,14 +3400,16 @@ class SystemDatabase {
2829
3400
  client.release();
2830
3401
  }
2831
3402
  // Return the IDs of all functions we marked started
2832
- return claimedIDs;
3403
+ return { claimedIDs };
2833
3404
  }
2834
3405
  /** Max heads admitted per sweep: bounds dispatch, not the partition walk; an unconstrained sweep takes the lowest keys, so higher keys can wait under sustained load. */
2835
3406
  partitionedDequeueSweepCap = 8192;
2836
3407
  /** Dequeue each partition's head-of-line workflow in one transaction, at most {@link partitionedDequeueSweepCap} per sweep; only valid for partition-concurrency-1 queues with no queue-wide limit. */
2837
3408
  async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity,
2838
3409
  // Written to every claimed row: the dispatches of this claim own their executions.
2839
- ownerXid = null) {
3410
+ ownerXid = null, isLatestVersion) {
3411
+ // Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
3412
+ const startTimeMs = Date.now();
2840
3413
  if (queue.partitionConcurrency !== 1 ||
2841
3414
  queue.concurrency !== undefined ||
2842
3415
  queue.rateLimit !== undefined ||
@@ -2844,11 +3417,14 @@ class SystemDatabase {
2844
3417
  throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
2845
3418
  }
2846
3419
  // partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
3420
+ // A sharded database keeps the version row under another key, which the claim's transaction cannot span.
3421
+ if (isLatestVersion === undefined && this.sharded) {
3422
+ isLatestVersion = await this.isLatestVersion(appVersion);
3423
+ }
2847
3424
  const client = await this.connect();
2848
3425
  try {
2849
3426
  await client.query('BEGIN');
2850
- const latestVersion = await this.#latestApplicationVersionName(client);
2851
- const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
3427
+ isLatestVersion ??= await this.isLatestVersion(appVersion, client);
2852
3428
  const versionClause = (n) => isLatestVersion
2853
3429
  ? `(application_version = $${n} OR application_version IS NULL)`
2854
3430
  : `application_version = $${n}`;
@@ -2863,7 +3439,8 @@ class SystemDatabase {
2863
3439
  workflow_1.StatusString.PENDING,
2864
3440
  sweepLimit,
2865
3441
  ];
2866
- const candidateScope = this.appNameFilter('application_name', this.appName, candidateParams);
3442
+ const candidateScope = `${this.scoped(undefined, candidateParams)} AND ${this.appNameFilter('application_name', this.appName, candidateParams)}`;
3443
+ const probeScope = this.scoped(undefined, candidateParams);
2867
3444
  // Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
2868
3445
  const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
2869
3446
  (SELECT MIN(queue_partition_key) AS pk
@@ -2880,11 +3457,11 @@ class SystemDatabase {
2880
3457
  SELECT partitions.pk
2881
3458
  FROM partitions
2882
3459
  WHERE partitions.pk IS NOT NULL
2883
- -- Unscoped by design: a mutual-exclusion probe must block on any owner's row.
3460
+ -- No application filter by design: a mutual-exclusion probe must block on any owner's row.
2884
3461
  AND NOT EXISTS (
2885
3462
  SELECT 1
2886
3463
  FROM "${this.schemaName}".workflow_status
2887
- WHERE queue_name = $1 AND status = $4
3464
+ WHERE ${probeScope} AND queue_name = $1 AND status = $4
2888
3465
  AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
2889
3466
  )
2890
3467
  ORDER BY ${sweepOrder}
@@ -2913,18 +3490,19 @@ class SystemDatabase {
2913
3490
  }
2914
3491
  await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
2915
3492
  // Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
2916
- const claimGuard = (ids, status, name, version, scope) => `workflow_uuid = ANY($${ids}::text[])
3493
+ const claimGuard = (ids, status, name, version, scope) => `${ids}
2917
3494
  AND status = $${status}
2918
3495
  AND queue_name = $${name}
2919
3496
  AND queue_partition_key IS NOT NULL
2920
3497
  AND ${versionClause(version)}
2921
3498
  AND ${scope}`;
2922
- const lockedParams = [candidateIDs, workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
3499
+ const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
3500
+ const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
2923
3501
  const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
2924
3502
  // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
2925
3503
  const lockedResult = await client.query(`SELECT workflow_uuid
2926
3504
  FROM "${this.schemaName}".workflow_status
2927
- WHERE ${claimGuard(1, 2, 3, 4, lockedScope)}
3505
+ WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
2928
3506
  FOR UPDATE SKIP LOCKED`, lockedParams);
2929
3507
  const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
2930
3508
  // Preserve partition order for submission.
@@ -2936,32 +3514,37 @@ class SystemDatabase {
2936
3514
  const flipParams = [
2937
3515
  workflow_1.StatusString.PENDING,
2938
3516
  executorID,
2939
- claimIDs,
2940
3517
  workflow_1.StatusString.ENQUEUED,
2941
3518
  queue.name,
2942
3519
  appVersion,
2943
3520
  // Claim the row, as the unpartitioned dequeue does.
2944
3521
  this.appName ?? null,
2945
3522
  ownerXid,
3523
+ startTimeMs,
2946
3524
  ];
3525
+ const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
2947
3526
  const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
2948
3527
  // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
2949
3528
  const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
2950
3529
  SET status = $1,
2951
- owner_xid = $8,
3530
+ owner_xid = $7,
2952
3531
  executor_id = $2,
2953
- application_version = $6,
3532
+ application_version = $5,
2954
3533
  started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2955
3534
  rate_limited = FALSE,
2956
- application_name = COALESCE(application_name, $7),
3535
+ application_name = COALESCE(application_name, $6),
2957
3536
  recovery_attempts = recovery_attempts + 1,
2958
3537
  updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2959
3538
  workflow_deadline_epoch_ms = CASE
2960
- WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
2961
- THEN (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
3539
+ WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
3540
+ CASE
3541
+ WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
3542
+ THEN 9223372036854775807
3543
+ ELSE $8::bigint + workflow_timeout_ms
3544
+ END
2962
3545
  ELSE workflow_deadline_epoch_ms
2963
3546
  END
2964
- WHERE ${claimGuard(3, 4, 5, 6, flipScope)}
3547
+ WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
2965
3548
  RETURNING workflow_uuid`, flipParams);
2966
3549
  await client.query('COMMIT');
2967
3550
  const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
@@ -2980,7 +3563,15 @@ class SystemDatabase {
2980
3563
  }
2981
3564
  }
2982
3565
  // ==================== Queries & Maintenance ====================
2983
- async listWorkflows(input) {
3566
+ /** With `keyedIDs`, the listed workflow IDs are addressed by their shard keys, as a batch read of known workflows. */
3567
+ async listWorkflows(input, options = {}) {
3568
+ return await this.#listWorkflows(input, options);
3569
+ }
3570
+ /**
3571
+ * Unretried body of {@link listWorkflows}. An ID-keyed read runs on `client` when given, so a caller
3572
+ * holding a transaction never waits on a second pool connection; the caller owns the retry.
3573
+ */
3574
+ async #listWorkflows(input, options = {}, client) {
2984
3575
  const schemaName = this.schemaName;
2985
3576
  const selectColumns = [
2986
3577
  'workflow_uuid',
@@ -3022,16 +3613,23 @@ class SystemDatabase {
3022
3613
  const payloadJoins = [];
3023
3614
  if (input.loadInput) {
3024
3615
  payloadColumns.push('COALESCE(wi.inputs, workflow_status.inputs) AS inputs');
3025
- payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON wi.workflow_uuid = workflow_status.workflow_uuid`);
3616
+ payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON ${this.wfJoin('wi', 'workflow_status')}`);
3026
3617
  }
3027
3618
  if (input.loadOutput) {
3028
3619
  payloadColumns.push('COALESCE(wo.output, workflow_status.output) AS output');
3029
3620
  payloadColumns.push('COALESCE(wo.error, workflow_status.error) AS error');
3030
- payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON wo.workflow_uuid = workflow_status.workflow_uuid`);
3621
+ payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON ${this.wfJoin('wo', 'workflow_status')}`);
3031
3622
  }
3032
3623
  if (input.loadInput || input.loadOutput) {
3033
3624
  selectColumns.push('serialization');
3034
3625
  }
3626
+ const keyColumns = [];
3627
+ const keyColumn = this.keyColumn('workflow_status');
3628
+ if (keyColumn !== undefined)
3629
+ keyColumns.push(`${keyColumn} AS shard_key`);
3630
+ const parentKeyColumn = this.keyColumn('workflow_status', 'parent_shard_key');
3631
+ if (parentKeyColumn !== undefined)
3632
+ keyColumns.push(`${parentKeyColumn} AS parent_shard_key`);
3035
3633
  input.sortDesc = input.sortDesc ?? false; // By default, sort in ascending order
3036
3634
  // Build WHERE clauses
3037
3635
  const whereClauses = [];
@@ -3069,9 +3667,23 @@ class SystemDatabase {
3069
3667
  // application, as on every other observability query.
3070
3668
  // Falsy, not just undefined: a Conductor request carries an omitted list as JSON null.
3071
3669
  const idKeyed = (input.workflowIDs?.length ?? 0) > 0;
3670
+ if (client && !idKeyed)
3671
+ throw new TypeError('listWorkflows takes a client only for an ID-keyed read');
3072
3672
  whereClauses.push(idKeyed
3073
3673
  ? this.appNameFilter('application_name', input.applicationName, params)
3074
3674
  : this.observabilityFilter('application_name', input.applicationName, params));
3675
+ // An empty key list matches nothing, rather than every key.
3676
+ if (input.shardKey !== undefined && input.shardKey !== null) {
3677
+ const keys = Array.isArray(input.shardKey) ? input.shardKey : [input.shardKey];
3678
+ whereClauses.push(this.keyIn('workflow_status', keys, params));
3679
+ }
3680
+ if (input.parentShardKey !== undefined && input.parentShardKey !== null) {
3681
+ const keys = Array.isArray(input.parentShardKey) ? input.parentShardKey : [input.parentShardKey];
3682
+ whereClauses.push(this.keyIn('workflow_status', keys, params, 'parent_shard_key'));
3683
+ }
3684
+ if (options.keyedIDs && input.workflowIDs) {
3685
+ whereClauses.push(this.wfIn('workflow_status', input.workflowIDs, params));
3686
+ }
3075
3687
  paramCounter = params.length + 1;
3076
3688
  if (input.workflow_id_prefix) {
3077
3689
  if (Array.isArray(input.workflow_id_prefix)) {
@@ -3086,7 +3698,7 @@ class SystemDatabase {
3086
3698
  paramCounter++;
3087
3699
  }
3088
3700
  }
3089
- if (input.workflowIDs) {
3701
+ if (input.workflowIDs && !options.keyedIDs) {
3090
3702
  const placeholders = input.workflowIDs.map((_, i) => `$${paramCounter + i}`).join(', ');
3091
3703
  whereClauses.push(`workflow_status.workflow_uuid IN (${placeholders})`);
3092
3704
  params.push(...input.workflowIDs);
@@ -3168,7 +3780,7 @@ class SystemDatabase {
3168
3780
  params.push(input.offset);
3169
3781
  paramCounter++;
3170
3782
  }
3171
- const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns].join(', ');
3783
+ const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns, ...keyColumns].join(', ');
3172
3784
  const query = `
3173
3785
  SELECT ${projection}
3174
3786
  FROM "${schemaName}".workflow_status
@@ -3180,31 +3792,24 @@ class SystemDatabase {
3180
3792
  `;
3181
3793
  // An ID-keyed read is bounded by its ID list, so it takes no cap: a status lookup must not fail its own caller.
3182
3794
  const result = idKeyed
3183
- ? await this.pool.query(query, params)
3184
- : await this.observabilityQuery((client) => client.query(query, params));
3795
+ ? await (client ?? this.pool).query(query, params)
3796
+ : await this.observabilityQuery((c) => c.query(query, params));
3185
3797
  return result.rows.map(mapWorkflowStatus);
3186
3798
  }
3187
3799
  // ==================== Scheduling ====================
3188
3800
  async createSchedule(schedule, client) {
3189
3801
  const q = client ?? this.pool;
3190
3802
  const owner = await this.#resolveRowOwner(q, 'workflow_schedules', 'schedule_name', schedule.scheduleName, schedule.applicationName, 'Schedule');
3803
+ const values = {
3804
+ ...this.globalValues(),
3805
+ ...this.scheduleValues(schedule),
3806
+ ...scheduleRow(schedule, owner),
3807
+ };
3191
3808
  try {
3192
- await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules
3193
- (schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name)
3194
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, [
3195
- schedule.scheduleId,
3196
- schedule.scheduleName,
3197
- schedule.workflowName,
3198
- schedule.workflowClassName,
3199
- schedule.schedule,
3200
- schedule.status,
3201
- schedule.context,
3202
- schedule.lastFiredAt,
3203
- schedule.automaticBackfill,
3204
- schedule.cronTimezone,
3205
- schedule.queueName,
3206
- owner ?? null,
3207
- ]);
3809
+ await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
3810
+ VALUES (${Object.keys(values)
3811
+ .map((_, i) => `$${i + 1}`)
3812
+ .join(', ')})`, Object.values(values));
3208
3813
  }
3209
3814
  catch (e) {
3210
3815
  if (isPgDatabaseError(e) && e.code === '23505') {
@@ -3251,31 +3856,37 @@ class SystemDatabase {
3251
3856
  // Unset scopes to this application, as on every other observability query.
3252
3857
  conditions.push(this.observabilityFilter('application_name', filters?.applicationName, params));
3253
3858
  paramIdx = params.length + 1;
3859
+ conditions.push(this.global(undefined, params));
3254
3860
  const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
3255
- const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
3861
+ const result = await q.query(`SELECT ${this.#scheduleColumns()}
3256
3862
  FROM "${this.schemaName}".workflow_schedules${where}
3257
3863
  ORDER BY schedule_name`, params);
3258
3864
  return result.rows.map((row) => mapWorkflowSchedule(row));
3259
3865
  }
3260
3866
  async getSchedule(name, client) {
3261
3867
  const q = client ?? this.pool;
3262
- const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
3868
+ const params = [name];
3869
+ const result = await q.query(`SELECT ${this.#scheduleColumns()}
3263
3870
  FROM "${this.schemaName}".workflow_schedules
3264
- WHERE schedule_name = $1`, [name]);
3871
+ WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
3265
3872
  if (result.rows.length === 0)
3266
3873
  return null;
3267
3874
  return mapWorkflowSchedule(result.rows[0]);
3268
3875
  }
3876
+ /** The columns a schedule is read back with, its workflows' shard key among them on a sharded database. */
3877
+ #scheduleColumns() {
3878
+ const keyColumn = this.scheduleKeyColumn();
3879
+ return keyColumn !== undefined ? `${SCHEDULE_COLUMNS}, ${keyColumn}` : SCHEDULE_COLUMNS;
3880
+ }
3269
3881
  async deleteSchedule(name, client) {
3270
3882
  const q = client ?? this.pool;
3271
- await q.query(`DELETE FROM "${this.schemaName}".workflow_schedules WHERE schedule_name = $1`, [name]);
3883
+ const params = [name];
3884
+ await q.query(`DELETE FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
3272
3885
  }
3273
3886
  async setScheduleStatus(name, status, client) {
3274
3887
  const q = client ?? this.pool;
3275
- await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET status = $1 WHERE schedule_name = $2`, [
3276
- status,
3277
- name,
3278
- ]);
3888
+ const params = [status, name];
3889
+ await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET status = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
3279
3890
  }
3280
3891
  async updateSchedule(name, updates, client) {
3281
3892
  const q = client ?? this.pool;
@@ -3298,22 +3909,22 @@ class SystemDatabase {
3298
3909
  }
3299
3910
  if (setClauses.length === 0) {
3300
3911
  // Nothing to change, but still surface a missing schedule as an error.
3301
- const existing = await q.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules WHERE schedule_name = $1`, [
3302
- name,
3303
- ]);
3912
+ const existingParams = [name];
3913
+ const existing = await q.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, existingParams)} AND schedule_name = $1`, existingParams);
3304
3914
  if (existing.rows.length === 0) {
3305
3915
  throw new error_1.DBOSError(`Schedule '${name}' not found`);
3306
3916
  }
3307
3917
  return;
3308
3918
  }
3309
3919
  params.push(name);
3310
- const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx}`, params);
3920
+ const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx} AND ${this.global(undefined, params)}`, params);
3311
3921
  if (result.rowCount === 0) {
3312
3922
  throw new error_1.DBOSError(`Schedule '${name}' not found`);
3313
3923
  }
3314
3924
  }
3315
3925
  async updateLastFiredAt(name, lastFiredAt) {
3316
- await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE schedule_name = $2`, [lastFiredAt, name]);
3926
+ const params = [lastFiredAt, name];
3927
+ await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
3317
3928
  }
3318
3929
  async applySchedules(schedules) {
3319
3930
  const client = await this.connect();
@@ -3321,12 +3932,21 @@ class SystemDatabase {
3321
3932
  await client.query('BEGIN');
3322
3933
  for (const sched of schedules) {
3323
3934
  const owner = await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
3935
+ const scheduleKeys = this.scheduleValues(sched);
3936
+ const values = {
3937
+ ...this.globalValues(),
3938
+ ...scheduleKeys,
3939
+ ...scheduleRow(sched, owner),
3940
+ };
3324
3941
  // Upsert on schedule_name; on conflict, preserve schedule_id and runtime state (status, last_fired_at) and update only the declared definition fields, so an unchanged re-apply is a no-op.
3325
- await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules
3326
- (schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name)
3327
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
3328
- ON CONFLICT (schedule_name) DO UPDATE SET
3329
- workflow_name = EXCLUDED.workflow_name,
3942
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
3943
+ VALUES (${Object.keys(values)
3944
+ .map((_, i) => `$${i + 1}`)
3945
+ .join(', ')})
3946
+ ON CONFLICT (${this.keyConflict('schedule_name')}) DO UPDATE SET
3947
+ ${Object.keys(scheduleKeys)
3948
+ .map((c) => `${c} = EXCLUDED.${c}, `)
3949
+ .join('')}workflow_name = EXCLUDED.workflow_name,
3330
3950
  workflow_class_name = EXCLUDED.workflow_class_name,
3331
3951
  schedule = EXCLUDED.schedule,
3332
3952
  context = EXCLUDED.context,
@@ -3334,20 +3954,7 @@ class SystemDatabase {
3334
3954
  cron_timezone = EXCLUDED.cron_timezone,
3335
3955
  queue_name = EXCLUDED.queue_name,
3336
3956
  -- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it took.
3337
- application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, [
3338
- sched.scheduleId,
3339
- sched.scheduleName,
3340
- sched.workflowName,
3341
- sched.workflowClassName,
3342
- sched.schedule,
3343
- sched.status,
3344
- sched.context,
3345
- sched.lastFiredAt,
3346
- sched.automaticBackfill,
3347
- sched.cronTimezone,
3348
- sched.queueName,
3349
- owner ?? null,
3350
- ]);
3957
+ application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, Object.values(values));
3351
3958
  // Read back, since the guard above is silent about why it declined to claim.
3352
3959
  await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
3353
3960
  }
@@ -3372,14 +3979,23 @@ class SystemDatabase {
3372
3979
  try {
3373
3980
  await client.query('BEGIN');
3374
3981
  // Claim a pre-upgrade row in place, so the version is not recreated or retimed.
3982
+ const claimParams = [owner ?? null, versionName];
3375
3983
  const claimed = await client.query(`UPDATE "${this.schemaName}".application_versions
3376
3984
  SET application_name = $1
3377
- WHERE version_name = $2 AND application_name IS NULL`, [owner ?? null, versionName]);
3985
+ WHERE ${this.global(undefined, claimParams)} AND version_name = $2 AND application_name IS NULL`, claimParams);
3378
3986
  if ((claimed.rowCount ?? 0) === 0) {
3987
+ const values = {
3988
+ ...this.globalValues(),
3989
+ version_id: (0, crypto_1.randomUUID)(),
3990
+ version_name: versionName,
3991
+ application_name: owner ?? null,
3992
+ };
3379
3993
  // Targetless DO NOTHING: names no arbiter, so it survives version_name's uniqueness being dropped while still absorbing a concurrent registrar.
3380
- await client.query(`INSERT INTO "${this.schemaName}".application_versions (version_id, version_name, application_name)
3381
- VALUES ($1, $2, $3)
3382
- ON CONFLICT DO NOTHING`, [(0, crypto_1.randomUUID)(), versionName, owner ?? null]);
3994
+ await client.query(`INSERT INTO "${this.schemaName}".application_versions (${Object.keys(values).join(', ')})
3995
+ VALUES (${Object.keys(values)
3996
+ .map((_, i) => `$${i + 1}`)
3997
+ .join(', ')})
3998
+ ON CONFLICT DO NOTHING`, Object.values(values));
3383
3999
  }
3384
4000
  // Read back, since the writes above are silent about why they declined to claim.
3385
4001
  await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
@@ -3410,7 +4026,7 @@ class SystemDatabase {
3410
4026
  params.push(resolved);
3411
4027
  await client.query(`UPDATE "${this.schemaName}".application_versions
3412
4028
  SET version_timestamp = $1, application_name = ${resolved === undefined ? 'application_name' : '$3'}
3413
- WHERE version_name = $2 AND ${scope}`, params);
4029
+ WHERE ${this.global(undefined, params)} AND version_name = $2 AND ${scope}`, params);
3414
4030
  await client.query('COMMIT');
3415
4031
  }
3416
4032
  catch (e) {
@@ -3423,7 +4039,7 @@ class SystemDatabase {
3423
4039
  }
3424
4040
  async listApplicationVersions() {
3425
4041
  const params = [];
3426
- const scope = this.appNameFilter('application_name', this.appName, params);
4042
+ const scope = `${this.global(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
3427
4043
  const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
3428
4044
  FROM "${this.schemaName}".application_versions
3429
4045
  WHERE ${scope}
@@ -3435,24 +4051,18 @@ class SystemDatabase {
3435
4051
  * caller acting for another one — firing its schedule — must name it.
3436
4052
  */
3437
4053
  async getLatestApplicationVersion(applicationName) {
3438
- const owner = applicationName ?? this.appName;
3439
- const params = [];
3440
- const scope = this.appNameFilter('application_name', owner, params);
3441
- const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
3442
- FROM "${this.schemaName}".application_versions
3443
- WHERE ${scope}
3444
- ORDER BY version_timestamp DESC
3445
- LIMIT 1`, params);
3446
- if (rows.length === 0) {
4054
+ const latest = await this.#latestApplicationVersion(this.pool, applicationName ?? this.appName);
4055
+ if (latest === undefined) {
3447
4056
  throw new error_1.DBOSInitializationError('No application versions found');
3448
4057
  }
3449
- return mapVersionInfo(rows[0]);
4058
+ return mapVersionInfo(latest);
3450
4059
  }
3451
4060
  // ==================== Queues ====================
3452
4061
  async getQueue(name) {
4062
+ const params = [name];
3453
4063
  const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
3454
4064
  FROM "${this.schemaName}".queues
3455
- WHERE name = $1`, [name]);
4065
+ WHERE ${this.global(undefined, params)} AND name = $1`, params);
3456
4066
  return rows.length === 0 ? null : queueRecordFromRow(rows[0]);
3457
4067
  }
3458
4068
  /**
@@ -3461,14 +4071,15 @@ class SystemDatabase {
3461
4071
  */
3462
4072
  async listQueues(applicationName) {
3463
4073
  const params = [];
3464
- const scope = this.observabilityFilter('application_name', applicationName, params);
4074
+ const scope = `${this.global(undefined, params)} AND ${this.observabilityFilter('application_name', applicationName, params)}`;
3465
4075
  const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
3466
4076
  FROM "${this.schemaName}".queues
3467
4077
  WHERE ${scope}`, params);
3468
4078
  return rows.map(queueRecordFromRow);
3469
4079
  }
3470
4080
  async deleteQueue(name) {
3471
- await this.pool.query(`DELETE FROM "${this.schemaName}".queues WHERE name = $1`, [name]);
4081
+ const params = [name];
4082
+ await this.pool.query(`DELETE FROM "${this.schemaName}".queues WHERE ${this.global(undefined, params)} AND name = $1`, params);
3472
4083
  }
3473
4084
  async updateQueue(name, fields) {
3474
4085
  const setClauses = [];
@@ -3484,7 +4095,7 @@ class SystemDatabase {
3484
4095
  setClauses.push(`"updated_at" = $${idx++}`);
3485
4096
  params.push(Date.now());
3486
4097
  params.push(name);
3487
- await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx}`, params);
4098
+ await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx} AND ${this.global(undefined, params)}`, params);
3488
4099
  }
3489
4100
  /** Returns true iff this call inserted a new row (i.e. the queue did not
3490
4101
  * previously exist). False if the row already existed, regardless of
@@ -3492,7 +4103,7 @@ class SystemDatabase {
3492
4103
  async upsertQueue(record, updateExisting) {
3493
4104
  const now = Date.now();
3494
4105
  const onConflict = updateExisting
3495
- ? `ON CONFLICT (name) DO UPDATE SET
4106
+ ? `ON CONFLICT (${this.keyConflict('name')}) DO UPDATE SET
3496
4107
  concurrency = EXCLUDED.concurrency,
3497
4108
  worker_concurrency = EXCLUDED.worker_concurrency,
3498
4109
  rate_limit_max = EXCLUDED.rate_limit_max,
@@ -3507,21 +4118,27 @@ class SystemDatabase {
3507
4118
  updated_at = EXCLUDED.updated_at,
3508
4119
  -- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it just took.
3509
4120
  application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
3510
- : `ON CONFLICT (name) DO NOTHING`;
4121
+ : `ON CONFLICT (${this.keyConflict('name')}) DO NOTHING`;
3511
4122
  const owner = record.applicationName ?? this.appName;
3512
4123
  const client = await this.connect();
3513
4124
  try {
3514
4125
  await client.query('BEGIN');
3515
- const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE name = $1`, [record.name]);
4126
+ const existedParams = [record.name];
4127
+ const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE ${this.global(undefined, existedParams)} AND name = $1`, existedParams);
3516
4128
  // A name collision is a conflict in every mode: the name is the queue's address.
3517
4129
  const resolvedOwner = await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
4130
+ const keys = this.globalValues();
3518
4131
  await client.query(`INSERT INTO "${this.schemaName}".queues
3519
- (name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
4132
+ (${Object.keys(keys)
4133
+ .map((c) => `${c}, `)
4134
+ .join('')}name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
3520
4135
  priority_enabled, partition_queue, partition_concurrency, partition_worker_concurrency,
3521
4136
  partition_rate_limit_max, partition_rate_limit_period_sec,
3522
4137
  polling_interval_sec, updated_at, application_name)
3523
4138
  -- priority_enabled is vestigial: every queue dispatches in priority order.
3524
- VALUES ($1, $2, $3, $4, $5, TRUE, $6, $7, $8, $9, $10, $11, $12, $13)
4139
+ VALUES (${Object.keys(keys)
4140
+ .map((_, i) => `$${14 + i}, `)
4141
+ .join('')}$1, $2, $3, $4, $5, TRUE, $6, $7, $8, $9, $10, $11, $12, $13)
3525
4142
  ${onConflict}`, [
3526
4143
  record.name,
3527
4144
  record.concurrency,
@@ -3536,6 +4153,7 @@ class SystemDatabase {
3536
4153
  record.pollingIntervalSec,
3537
4154
  now,
3538
4155
  resolvedOwner ?? null,
4156
+ ...Object.values(keys),
3539
4157
  ]);
3540
4158
  // Read back, since the guard above is silent about why it declined to claim.
3541
4159
  await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
@@ -3645,6 +4263,9 @@ class SystemDatabase {
3645
4263
  params.push(statuses);
3646
4264
  where = `${where} AND status = ANY($${params.length})`;
3647
4265
  }
4266
+ else {
4267
+ where = `${where} AND ${this.global(undefined, params)}`;
4268
+ }
3648
4269
  const res = await client.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${where}`, params);
3649
4270
  return res.rowCount ?? 0;
3650
4271
  };
@@ -3669,84 +4290,64 @@ class SystemDatabase {
3669
4290
  }
3670
4291
  // ==================== Internal ====================
3671
4292
  async insertWorkflowStatus(client, initStatus, creatorXid) {
4293
+ const values = {
4294
+ ...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
4295
+ workflow_uuid: initStatus.workflowUUID,
4296
+ status: initStatus.status,
4297
+ name: initStatus.workflowName,
4298
+ // For cross-language compatibility, these variables MUST be NULL in the database when not set
4299
+ class_name: initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
4300
+ config_name: initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
4301
+ queue_name: initStatus.queueName ?? null,
4302
+ authenticated_user: initStatus.authenticatedUser,
4303
+ assumed_role: initStatus.assumedRole,
4304
+ authenticated_roles: JSON.stringify(initStatus.authenticatedRoles),
4305
+ executor_id: initStatus.executorId,
4306
+ application_version: initStatus.applicationVersion ?? null,
4307
+ application_id: initStatus.applicationID,
4308
+ recovery_attempts: initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
4309
+ workflow_timeout_ms: initStatus.timeoutMS ?? null,
4310
+ workflow_deadline_epoch_ms: initStatus.deadlineEpochMS ?? null,
4311
+ // Legacy column: the payload lives in workflow_input.
4312
+ inputs: null,
4313
+ deduplication_id: initStatus.deduplicationID ?? null,
4314
+ priority: initStatus.priority,
4315
+ queue_partition_key: initStatus.queuePartitionKey ?? null,
4316
+ forked_from: initStatus.forkedFrom ?? null,
4317
+ parent_workflow_id: initStatus.parentWorkflowID ?? null,
4318
+ ...this.parentValues(initStatus.parentWorkflowID),
4319
+ serialization: initStatus.serialization,
4320
+ creator_xid: creatorXid,
4321
+ delay_until_epoch_ms: initStatus.delayUntilEpochMS ?? null,
4322
+ attributes: initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
4323
+ schedule_name: initStatus.scheduleName ?? null,
4324
+ debounce_deadline_epoch_ms: initStatus.debounceDeadlineEpochMS ?? null,
4325
+ is_debounced: initStatus.isDebounced ?? false,
4326
+ application_name: initStatus.applicationName ?? null,
4327
+ // A direct start runs at once, so its creator token also owns the execution.
4328
+ owner_xid: initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
4329
+ };
4330
+ const columns = Object.keys(values);
4331
+ const inputValues = {
4332
+ ...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
4333
+ workflow_uuid: initStatus.workflowUUID,
4334
+ inputs: initStatus.input ?? null,
4335
+ };
4336
+ const inputColumns = Object.keys(inputValues);
3672
4337
  try {
3673
- const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
3674
- workflow_uuid,
3675
- status,
3676
- name,
3677
- class_name,
3678
- config_name,
3679
- queue_name,
3680
- authenticated_user,
3681
- assumed_role,
3682
- authenticated_roles,
3683
- executor_id,
3684
- application_version,
3685
- application_id,
3686
- recovery_attempts,
3687
- workflow_timeout_ms,
3688
- workflow_deadline_epoch_ms,
3689
- inputs,
3690
- deduplication_id,
3691
- priority,
3692
- queue_partition_key,
3693
- forked_from,
3694
- parent_workflow_id,
3695
- serialization,
3696
- creator_xid,
3697
- delay_until_epoch_ms,
3698
- attributes,
3699
- schedule_name,
3700
- debounce_deadline_epoch_ms,
3701
- is_debounced,
3702
- application_name,
3703
- owner_xid
3704
- ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)
3705
- ON CONFLICT (workflow_uuid)
4338
+ const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
4339
+ VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
4340
+ ON CONFLICT (${this.keyConflict('workflow_uuid')})
3706
4341
  -- A no-op update, so an existing row comes back unchanged for the caller to inspect.
3707
4342
  DO UPDATE SET creator_xid = workflow_status.creator_xid
3708
- RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`, [
3709
- initStatus.workflowUUID,
3710
- initStatus.status,
3711
- initStatus.workflowName,
3712
- // For cross-language compatibility, these variables MUST be NULL in the database when not set
3713
- initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
3714
- initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
3715
- initStatus.queueName ?? null,
3716
- initStatus.authenticatedUser,
3717
- initStatus.assumedRole,
3718
- JSON.stringify(initStatus.authenticatedRoles),
3719
- initStatus.executorId,
3720
- initStatus.applicationVersion ?? null,
3721
- initStatus.applicationID,
3722
- initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
3723
- initStatus.timeoutMS ?? null,
3724
- initStatus.deadlineEpochMS ?? null,
3725
- // Legacy column: the payload lives in workflow_input.
3726
- null,
3727
- initStatus.deduplicationID ?? null,
3728
- initStatus.priority,
3729
- initStatus.queuePartitionKey ?? null,
3730
- initStatus.forkedFrom ?? null,
3731
- initStatus.parentWorkflowID ?? null,
3732
- initStatus.serialization,
3733
- creatorXid,
3734
- initStatus.delayUntilEpochMS ?? null,
3735
- initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
3736
- initStatus.scheduleName ?? null,
3737
- initStatus.debounceDeadlineEpochMS ?? null,
3738
- initStatus.isDebounced ?? false,
3739
- initStatus.applicationName ?? null,
3740
- // A direct start runs at once, so its creator token also owns the execution.
3741
- initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
3742
- ]);
4343
+ RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`, Object.values(values));
3743
4344
  if (rows.length === 0) {
3744
4345
  throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
3745
4346
  }
3746
4347
  // Two statements, not a data-modifying CTE: at scale the CTE costs more than the round trip it saves.
3747
- await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs)
3748
- VALUES ($1, $2)
3749
- ON CONFLICT (workflow_uuid) DO NOTHING`, [initStatus.workflowUUID, initStatus.input ?? null]);
4348
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_input (${inputColumns.join(', ')})
4349
+ VALUES (${inputColumns.map((_, i) => `$${i + 1}`).join(', ')})
4350
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`, Object.values(inputValues));
3750
4351
  const ret = rows[0];
3751
4352
  ret.class_name = ret.class_name ?? '';
3752
4353
  ret.config_name = ret.config_name ?? '';
@@ -3762,16 +4363,17 @@ class SystemDatabase {
3762
4363
  }
3763
4364
  }
3764
4365
  async getWorkflowStatusValue(client, workflowID) {
3765
- const { rows } = await client.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE workflow_uuid=$1`, [workflowID]);
4366
+ const params = [];
4367
+ const { rows } = await client.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
3766
4368
  return rows.length === 0 ? undefined : rows[0].status;
3767
4369
  }
3768
4370
  async updateWorkflowStatus(client, workflowID, status, options = {}) {
3769
4371
  // Use SQL now() so updated_at and completed_at (when set together) are
3770
4372
  // computed in the same statement against the same clock.
3771
4373
  const nowMsExpr = `(EXTRACT(EPOCH FROM now()) * 1000)::bigint`;
3772
- let setClause = `SET status=$2, updated_at=${nowMsExpr}`;
3773
- let whereClause = `WHERE workflow_uuid=$1`;
3774
- const args = [workflowID, status];
4374
+ let setClause = `SET status=$1, updated_at=${nowMsExpr}`;
4375
+ const args = [status];
4376
+ let whereClause = `WHERE ${this.wf(undefined, workflowID, args)}`;
3775
4377
  const update = options.update ?? {};
3776
4378
  if (update.resetRecoveryAttempts) {
3777
4379
  setClause += `, recovery_attempts = 0`;
@@ -3830,27 +4432,31 @@ class SystemDatabase {
3830
4432
  async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
3831
4433
  const ownerXid = options.ownerChecked ? undefined : (0, context_1.currentOwnerXid)(workflowID);
3832
4434
  if (ownerXid !== undefined) {
3833
- await this.#checkOwner(client, workflowID, ownerXid);
3834
- }
4435
+ await this.checkOwner(client, workflowID, ownerXid);
4436
+ }
4437
+ const values = {
4438
+ ...this.wfValues(workflowID),
4439
+ workflow_uuid: workflowID,
4440
+ function_id: functionID,
4441
+ output: options.output ?? null,
4442
+ error: options.error ?? null,
4443
+ function_name: functionName,
4444
+ child_workflow_id: options.childWorkflowID ?? null,
4445
+ ...(options.childValues ?? this.childValues(options.childWorkflowID)),
4446
+ started_at_epoch_ms: startTimeEpochMs,
4447
+ completed_at_epoch_ms: endTimeEpochMs,
4448
+ serialization: options.serialization ?? null,
4449
+ // Mirrors the parent: only the running application records its steps.
4450
+ application_name: this.appName ?? null,
4451
+ };
4452
+ const columns = Object.keys(values);
3835
4453
  try {
3836
4454
  const out = await client.query(`INSERT INTO "${this.schemaName}".operation_outputs
3837
- (workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization, application_name, retention_timestamp)
3838
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
3839
- ON CONFLICT (workflow_uuid, function_id) DO UPDATE
4455
+ (${columns.join(', ')}, retention_timestamp)
4456
+ VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')}, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
4457
+ ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id')}) DO UPDATE
3840
4458
  SET completed_at_epoch_ms = operation_outputs.completed_at_epoch_ms
3841
- RETURNING completed_at_epoch_ms;`, [
3842
- workflowID,
3843
- functionID,
3844
- options.output ?? null,
3845
- options.error ?? null,
3846
- functionName,
3847
- options.childWorkflowID ?? null,
3848
- startTimeEpochMs,
3849
- endTimeEpochMs,
3850
- options.serialization ?? null,
3851
- // Mirrors the parent: only the running application records its steps.
3852
- this.appName ?? null,
3853
- ]);
4459
+ RETURNING completed_at_epoch_ms;`, Object.values(values));
3854
4460
  // A row already there with another completion time is this execution's own doing
3855
4461
  // (a retry carries the same time), so it is nondeterminism, not a duplicate.
3856
4462
  if (checkConflict &&
@@ -3867,7 +4473,7 @@ class SystemDatabase {
3867
4473
  }
3868
4474
  }
3869
4475
  /** Run `fn` in a READ COMMITTED transaction on `client`, so an ownership check holds its row lock until the write commits. */
3870
- async #inTransaction(client, fn) {
4476
+ async inTransaction(client, fn) {
3871
4477
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
3872
4478
  let result;
3873
4479
  try {
@@ -3882,22 +4488,26 @@ class SystemDatabase {
3882
4488
  return result;
3883
4489
  }
3884
4490
  /** Throw DBOSWorkflowConflictError unless ownerXid still owns the workflow; the row lock holds until commit, so no hand-off lands in between. */
3885
- async #checkOwner(client, workflowID, ownerXid) {
3886
- const { rows } = await client.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = $1 FOR NO KEY UPDATE`, [workflowID]);
4491
+ async checkOwner(client, workflowID, ownerXid) {
4492
+ const params = [];
4493
+ const { rows } = await client.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)} FOR NO KEY UPDATE`, params);
3887
4494
  if (rows[0]?.owner_xid !== ownerXid) {
3888
4495
  throw new error_1.DBOSWorkflowConflictError(workflowID);
3889
4496
  }
3890
4497
  }
3891
4498
  /** The workflow's current ownership token; null if unowned or missing. */
3892
4499
  async getWorkflowOwner(workflowID) {
3893
- const { rows } = await this.pool.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = $1`, [workflowID]);
4500
+ const params = [];
4501
+ const { rows } = await this.pool.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
3894
4502
  return rows[0]?.owner_xid ?? null;
3895
4503
  }
3896
4504
  async #getOperationResultAndThrowIfCancelled(client, workflowID, functionID) {
3897
4505
  await this.#checkIfCanceled(client, workflowID);
3898
- const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name, serialization
4506
+ const childKey = this.keyColumn(undefined, 'child_shard_key');
4507
+ const params = [functionID];
4508
+ const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name, serialization${childKey !== undefined ? `, ${childKey}` : ''}
3899
4509
  FROM "${this.schemaName}".operation_outputs
3900
- WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
4510
+ WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
3901
4511
  if (rows.length === 0) {
3902
4512
  return undefined;
3903
4513
  }
@@ -3906,6 +4516,7 @@ class SystemDatabase {
3906
4516
  output: rows[0].output,
3907
4517
  error: rows[0].error,
3908
4518
  childWorkflowID: rows[0].child_workflow_id,
4519
+ childShardKey: rows[0].child_shard_key ?? undefined,
3909
4520
  functionName: rows[0].function_name,
3910
4521
  // Return serialization so recv/getEvent replay deserializes with the stored format, not the default.
3911
4522
  serialization: rows[0].serialization,
@@ -3943,7 +4554,11 @@ class SystemDatabase {
3943
4554
  // been cancelled.
3944
4555
  // For an actual sleep, completed_at is the wake deadline so the step's duration reflects the
3945
4556
  // sleep; a timeout marker records zero duration since its deadline may never be reached.
3946
- async #durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline = false) {
4557
+ async durableSleepInternal(workflowID, functionID, durationMS, recordCompletionAtDeadline = false) {
4558
+ // The caller's own workflow, even where a same-ID workflow is bound as a target.
4559
+ return await this.ownKey(workflowID, undefined, () => this.#durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline));
4560
+ }
4561
+ async #durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline) {
3947
4562
  const startTimeMs = Date.now();
3948
4563
  // Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
3949
4564
  const endTimeMs = startTimeMs + Math.ceil(durationMS);
@@ -3956,7 +4571,7 @@ class SystemDatabase {
3956
4571
  }
3957
4572
  return JSON.parse(res.output);
3958
4573
  }
3959
- await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
4574
+ await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
3960
4575
  output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
3961
4576
  serialization: serialization_1.DBOSPortableJSON.name(),
3962
4577
  }));
@@ -4136,7 +4751,7 @@ __decorate([
4136
4751
  __decorate([
4137
4752
  dbRetry(),
4138
4753
  __metadata("design:type", Function),
4139
- __metadata("design:paramtypes", [String, Number]),
4754
+ __metadata("design:paramtypes", [String, Number, String]),
4140
4755
  __metadata("design:returntype", Promise)
4141
4756
  ], SystemDatabase.prototype, "getOperationResultAndThrowIfCancelled", null);
4142
4757
  __decorate([
@@ -4196,9 +4811,9 @@ __decorate([
4196
4811
  __decorate([
4197
4812
  dbRetry(),
4198
4813
  __metadata("design:type", Function),
4199
- __metadata("design:paramtypes", [Array, Boolean]),
4814
+ __metadata("design:paramtypes", [Array]),
4200
4815
  __metadata("design:returntype", Promise)
4201
- ], SystemDatabase.prototype, "deleteWorkflows", null);
4816
+ ], SystemDatabase.prototype, "deleteWorkflowsInternal", null);
4202
4817
  __decorate([
4203
4818
  dbRetry()
4204
4819
  // A missing row normally means the workflow has not been inserted yet, so
@@ -4232,19 +4847,19 @@ __decorate([
4232
4847
  __decorate([
4233
4848
  dbRetry(),
4234
4849
  __metadata("design:type", Function),
4235
- __metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String]),
4850
+ __metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String, String]),
4236
4851
  __metadata("design:returntype", Promise)
4237
4852
  ], SystemDatabase.prototype, "send", null);
4238
4853
  __decorate([
4239
4854
  dbRetry(),
4240
4855
  __metadata("design:type", Function),
4241
- __metadata("design:paramtypes", [String, String, Object, Object, Object, String]),
4856
+ __metadata("design:paramtypes", [String, String, Object, Object, Object, String, String]),
4242
4857
  __metadata("design:returntype", Promise)
4243
4858
  ], SystemDatabase.prototype, "sendFromStep", null);
4244
4859
  __decorate([
4245
4860
  dbRetry(),
4246
4861
  __metadata("design:type", Function),
4247
- __metadata("design:paramtypes", [String, Object, Object, Object, String]),
4862
+ __metadata("design:paramtypes", [String, Object, Object, Object, String, String]),
4248
4863
  __metadata("design:returntype", Promise)
4249
4864
  ], SystemDatabase.prototype, "sendDirectStandalone", null);
4250
4865
  __decorate([
@@ -4298,7 +4913,7 @@ __decorate([
4298
4913
  __decorate([
4299
4914
  dbRetry(),
4300
4915
  __metadata("design:type", Function),
4301
- __metadata("design:paramtypes", [Object]),
4916
+ __metadata("design:paramtypes", [Object, Object]),
4302
4917
  __metadata("design:returntype", Promise)
4303
4918
  ], SystemDatabase.prototype, "listWorkflows", null);
4304
4919
  __decorate([