@dbos-inc/dbos-sdk 5.4.10-preview → 5.5.1-preview

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/dist/src/cli/cli.js +56 -13
  2. package/dist/src/cli/cli.js.map +1 -1
  3. package/dist/src/client.d.ts +20 -10
  4. package/dist/src/client.d.ts.map +1 -1
  5. package/dist/src/client.js +76 -44
  6. package/dist/src/client.js.map +1 -1
  7. package/dist/src/config.d.ts +1 -0
  8. package/dist/src/config.d.ts.map +1 -1
  9. package/dist/src/config.js +1 -0
  10. package/dist/src/config.js.map +1 -1
  11. package/dist/src/context.d.ts +6 -0
  12. package/dist/src/context.d.ts.map +1 -1
  13. package/dist/src/context.js +15 -1
  14. package/dist/src/context.js.map +1 -1
  15. package/dist/src/datasource.d.ts +2 -0
  16. package/dist/src/datasource.d.ts.map +1 -1
  17. package/dist/src/datasource.js +7 -1
  18. package/dist/src/datasource.js.map +1 -1
  19. package/dist/src/dbos-executor.d.ts +7 -4
  20. package/dist/src/dbos-executor.d.ts.map +1 -1
  21. package/dist/src/dbos-executor.js +56 -30
  22. package/dist/src/dbos-executor.js.map +1 -1
  23. package/dist/src/dbos.d.ts +30 -10
  24. package/dist/src/dbos.d.ts.map +1 -1
  25. package/dist/src/dbos.js +136 -64
  26. package/dist/src/dbos.js.map +1 -1
  27. package/dist/src/debugpoint.d.ts +1 -0
  28. package/dist/src/debugpoint.d.ts.map +1 -1
  29. package/dist/src/debugpoint.js +4 -2
  30. package/dist/src/debugpoint.js.map +1 -1
  31. package/dist/src/enqueue_options.d.ts +2 -0
  32. package/dist/src/enqueue_options.d.ts.map +1 -1
  33. package/dist/src/enqueue_options.js +3 -0
  34. package/dist/src/enqueue_options.js.map +1 -1
  35. package/dist/src/enqueue_workflow.d.ts.map +1 -1
  36. package/dist/src/enqueue_workflow.js +11 -5
  37. package/dist/src/enqueue_workflow.js.map +1 -1
  38. package/dist/src/enterprise.d.ts +23 -3
  39. package/dist/src/enterprise.d.ts.map +1 -1
  40. package/dist/src/enterprise.js +81 -44
  41. package/dist/src/enterprise.js.map +1 -1
  42. package/dist/src/eventreceiver.d.ts +2 -0
  43. package/dist/src/eventreceiver.d.ts.map +1 -1
  44. package/dist/src/eventreceiver.js +7 -1
  45. package/dist/src/eventreceiver.js.map +1 -1
  46. package/dist/src/index.d.ts +1 -1
  47. package/dist/src/index.d.ts.map +1 -1
  48. package/dist/src/index.js.map +1 -1
  49. package/dist/src/scheduler/scheduler.d.ts +5 -0
  50. package/dist/src/scheduler/scheduler.d.ts.map +1 -1
  51. package/dist/src/scheduler/scheduler.js +14 -4
  52. package/dist/src/scheduler/scheduler.js.map +1 -1
  53. package/dist/src/shard_keys.d.ts +13 -0
  54. package/dist/src/shard_keys.d.ts.map +1 -0
  55. package/dist/src/shard_keys.js +55 -0
  56. package/dist/src/shard_keys.js.map +1 -0
  57. package/dist/src/streams.d.ts +2 -0
  58. package/dist/src/streams.d.ts.map +1 -1
  59. package/dist/src/streams.js +5 -2
  60. package/dist/src/streams.js.map +1 -1
  61. package/dist/src/system_database.d.ts +150 -10
  62. package/dist/src/system_database.d.ts.map +1 -1
  63. package/dist/src/system_database.js +1250 -589
  64. package/dist/src/system_database.js.map +1 -1
  65. package/dist/src/wfqueue.d.ts +1 -2
  66. package/dist/src/wfqueue.d.ts.map +1 -1
  67. package/dist/src/wfqueue.js +83 -63
  68. package/dist/src/wfqueue.js.map +1 -1
  69. package/dist/src/workflow.d.ts +12 -2
  70. package/dist/src/workflow.d.ts.map +1 -1
  71. package/dist/src/workflow.js +14 -9
  72. package/dist/src/workflow.js.map +1 -1
  73. package/dist/src/workflow_management.d.ts +5 -3
  74. package/dist/src/workflow_management.d.ts.map +1 -1
  75. package/dist/src/workflow_management.js +45 -14
  76. package/dist/src/workflow_management.js.map +1 -1
  77. package/dist/tsconfig.tsbuildinfo +1 -1
  78. package/package.json +1 -1
@@ -1,15 +1,38 @@
1
1
  "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
2
18
  var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
19
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
20
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
21
  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
22
  return c > 3 && r && Object.defineProperty(target, key, r), r;
7
23
  };
24
+ var __importStar = (this && this.__importStar) || function (mod) {
25
+ if (mod && mod.__esModule) return mod;
26
+ var result = {};
27
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
28
+ __setModuleDefault(result, mod);
29
+ return result;
30
+ };
8
31
  var __metadata = (this && this.__metadata) || function (k, v) {
9
32
  if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
33
  };
11
34
  Object.defineProperty(exports, "__esModule", { value: true });
12
- exports.SystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
35
+ exports.SystemDatabase = exports.createSystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.qualify = exports.holdsShardedSchema = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
13
36
  const dbos_executor_1 = require("./dbos-executor");
14
37
  const pg_1 = require("pg");
15
38
  const error_1 = require("./error");
@@ -18,6 +41,8 @@ const utils_1 = require("./utils");
18
41
  const context_1 = require("./context");
19
42
  const async_hooks_1 = require("async_hooks");
20
43
  const crypto_1 = require("crypto");
44
+ const shard_keys_1 = require("./shard_keys");
45
+ const enterprise = __importStar(require("./enterprise"));
21
46
  const utils_2 = require("./utils");
22
47
  const database_utils_1 = require("./database_utils");
23
48
  const migration_runner_1 = require("./sysdb_migrations/migration_runner");
@@ -70,6 +95,8 @@ function validateObservabilityQueryTimeoutMs(value) {
70
95
  exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutMs;
71
96
  // Bounds how long a frozen or unreachable client can hold system database locks.
72
97
  exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = 60_000;
98
+ // How long a background job's lock may sit idle before Postgres ends its holder's session.
99
+ exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = 5000;
73
100
  /** The timeout is rendered into a `SET idle_in_transaction_session_timeout`, which takes only an in-range integer. */
74
101
  function validateIdleTransactionTimeoutMs(value) {
75
102
  if (value !== undefined && (!Number.isFinite(value) || value > MAX_STATEMENT_TIMEOUT_MS)) {
@@ -157,6 +184,7 @@ async function grantDbosSchemaPermissions(databaseUrl, roleName, logger, schemaN
157
184
  await client.end();
158
185
  }
159
186
  }
187
+ exports.grantDbosSchemaPermissions = grantDbosSchemaPermissions;
160
188
  /**
161
189
  * Check out a connection carrying our own error handler, so a socket death while we hold it is
162
190
  * logged instead of thrown at an emitter with no listener. `release()` takes the handler back off,
@@ -219,6 +247,7 @@ async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = '
219
247
  }
220
248
  const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
221
249
  try {
250
+ await refuseShardedSchema(client, schemaName);
222
251
  const isCockroach = await isCockroachDB(client);
223
252
  await (0, migration_runner_1.runSysMigrationsPg)(client, (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }), schemaName, {
224
253
  onWarn: (e) => logger.info(e),
@@ -242,6 +271,7 @@ exports.migrateSystemDatabase = migrateSystemDatabase;
242
271
  async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
243
272
  const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
244
273
  try {
274
+ await refuseShardedSchema(client, schemaName);
245
275
  const isCockroach = await isCockroachDB(client);
246
276
  const requiredVersion = (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }).length;
247
277
  const currentVersion = await (0, migration_runner_1.getCurrentSysDBVersion)(client, schemaName);
@@ -259,6 +289,41 @@ async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = '
259
289
  }
260
290
  }
261
291
  exports.verifySystemDatabase = verifySystemDatabase;
292
+ /** Throw if this schema is sharded: an unsharded handle's statements would span every tenant's rows. */
293
+ async function refuseShardedSchema(client, schemaName) {
294
+ const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
295
+ WHERE table_schema = $1 AND table_name = 'workflow_status' AND column_name = 'shard_key'`, [schemaName]);
296
+ if (rows.length > 0) {
297
+ throw new error_1.DBOSError(`The DBOS system schema ${schemaName} is sharded. Enable sharding to use it: ` +
298
+ 'set sharding in your DBOS configuration, or pass sharding: true to DBOSClient.');
299
+ }
300
+ }
301
+ /** Whether this database holds a sharded DBOS system schema (this one, if named); false if it can't be reached. */
302
+ async function holdsShardedSchema(databaseUrl, schemaName) {
303
+ let client;
304
+ try {
305
+ // Bounded, so an unreachable host fails as fast as the caller's own connection would.
306
+ client = new pg_1.Client({ ...(0, utils_2.getClientConfig)(databaseUrl), connectionTimeoutMillis: 10_000 });
307
+ }
308
+ catch {
309
+ // A malformed URL, for the caller's own connection to report.
310
+ return false;
311
+ }
312
+ client.on('error', () => { });
313
+ try {
314
+ await client.connect();
315
+ const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
316
+ WHERE table_name = 'workflow_status' AND column_name = 'shard_key'${schemaName !== undefined ? ' AND table_schema = $1' : ''} LIMIT 1`, schemaName !== undefined ? [schemaName] : []);
317
+ return rows.length > 0;
318
+ }
319
+ catch {
320
+ return false;
321
+ }
322
+ finally {
323
+ await client.end().catch(() => { });
324
+ }
325
+ }
326
+ exports.holdsShardedSchema = holdsShardedSchema;
262
327
  class NotificationMap {
263
328
  map = new Map();
264
329
  curCK = 0;
@@ -300,6 +365,23 @@ function mapVersionInfo(row) {
300
365
  };
301
366
  }
302
367
  const SCHEDULE_COLUMNS = 'schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name';
368
+ /** A schedule's own columns, as a new row stores them. */
369
+ function scheduleRow(schedule, owner) {
370
+ return {
371
+ schedule_id: schedule.scheduleId,
372
+ schedule_name: schedule.scheduleName,
373
+ workflow_name: schedule.workflowName,
374
+ workflow_class_name: schedule.workflowClassName,
375
+ schedule: schedule.schedule,
376
+ status: schedule.status,
377
+ context: schedule.context,
378
+ last_fired_at: schedule.lastFiredAt,
379
+ automatic_backfill: schedule.automaticBackfill,
380
+ cron_timezone: schedule.cronTimezone,
381
+ queue_name: schedule.queueName,
382
+ application_name: owner ?? null,
383
+ };
384
+ }
303
385
  function mapWorkflowSchedule(row) {
304
386
  return {
305
387
  scheduleId: row.schedule_id,
@@ -314,6 +396,7 @@ function mapWorkflowSchedule(row) {
314
396
  cronTimezone: row.cron_timezone ?? null,
315
397
  queueName: row.queue_name ?? null,
316
398
  applicationName: row.application_name ?? undefined,
399
+ shardKey: row.workflow_shard_key ?? undefined,
317
400
  };
318
401
  }
319
402
  function mapWorkflowStatus(row) {
@@ -356,8 +439,16 @@ function mapWorkflowStatus(row) {
356
439
  debounceDeadlineEpochMS: row.debounce_deadline_epoch_ms ? Number(row.debounce_deadline_epoch_ms) : undefined,
357
440
  isDebounced: row.is_debounced ?? false,
358
441
  applicationName: row.application_name ?? undefined,
442
+ // Read back only from a sharded system database, whose rows carry them.
443
+ shardKey: row.shard_key ?? undefined,
444
+ parentShardKey: row.parent_shard_key ?? undefined,
359
445
  };
360
446
  }
447
+ /** `column`, qualified by `alias` if one is given. */
448
+ function qualify(alias, column) {
449
+ return alias ? `${alias}.${column}` : column;
450
+ }
451
+ exports.qualify = qualify;
361
452
  // SQLSTATE classes/codes that are generally safe to retry
362
453
  // https://www.postgresql.org/docs/current/errcodes-appendix.html
363
454
  const RETRY_SQLSTATE_PREFIXES = new Set([
@@ -527,6 +618,26 @@ function dbRetry(options = {}) {
527
618
  return descriptor;
528
619
  };
529
620
  }
621
+ /** Build the system database these options describe. */
622
+ function createSystemDatabase(options) {
623
+ if (options.sharding) {
624
+ const { ShardedSystemDatabase } = enterprise.load(enterprise.SHARDING.description);
625
+ return new ShardedSystemDatabase({
626
+ systemDatabaseUrl: options.systemDatabaseUrl,
627
+ logger: options.logger,
628
+ serializer: options.serializer,
629
+ sysDbPoolSize: options.sysDbPoolSize,
630
+ systemDatabasePool: options.systemDatabasePool,
631
+ schemaName: options.schemaName,
632
+ pollingConcurrency: options.pollingConcurrency,
633
+ appName: options.appName,
634
+ observabilityQueryTimeoutMs: options.observabilityQueryTimeoutMs,
635
+ idleTransactionTimeoutMs: options.idleTransactionTimeoutMs,
636
+ });
637
+ }
638
+ return new SystemDatabase(options.systemDatabaseUrl, options.logger, options.serializer, options.sysDbPoolSize, options.systemDatabasePool, options.schemaName, options.useListenNotify, options.pollingConcurrency, options.notificationCoalesceMs, options.appName, options.observabilityQueryTimeoutMs, options.idleTransactionTimeoutMs);
639
+ }
640
+ exports.createSystemDatabase = createSystemDatabase;
530
641
  /**
531
642
  * General notes:
532
643
  * The responsibilities of the `SystemDatabase` are to store data for workflows, and
@@ -595,6 +706,7 @@ class SystemDatabase {
595
706
  pollLimiter;
596
707
  /** Executions running in this process, by workflow ID; a re-dispatch runs alongside its stale execution, which parks at its next write. */
597
708
  runningWorkflowMap = new Map();
709
+ // Keyed by waiter key, as one ID can live under several shard keys.
598
710
  #cancelWatchers = new Map();
599
711
  #cancelPollerLoop = undefined;
600
712
  #cancelPollerWake = null;
@@ -727,6 +839,209 @@ class SystemDatabase {
727
839
  observabilityFilter(column, value, params) {
728
840
  return this.appNameFilter(column, value ?? this.appName, params);
729
841
  }
842
+ // ==================== Shard Keys ====================
843
+ // Statements locate rows through these, so a sharded subclass adds its key; each appends to `params` and returns SQL.
844
+ /** Whether this handle routes every statement by shard key. */
845
+ get sharded() {
846
+ return false;
847
+ }
848
+ #rejectShardKey(workflowID, shardKey) {
849
+ if ((shardKey !== undefined && shardKey !== null) || (0, shard_keys_1.boundShardKey)(workflowID) !== undefined) {
850
+ throw new error_1.DBOSError('A shard key was given, but this system database is not sharded');
851
+ }
852
+ }
853
+ /** Predicate selecting one workflow's rows: those whose `column` is `workflowID`. */
854
+ wf(alias, workflowID, params, column = 'workflow_uuid', shardKey) {
855
+ this.#rejectShardKey(workflowID, shardKey);
856
+ params.push(workflowID);
857
+ return `${qualify(alias, column)} = $${params.length}`;
858
+ }
859
+ /** Predicate selecting several workflows' rows. */
860
+ wfIn(alias, workflowIDs, params) {
861
+ for (const workflowID of workflowIDs) {
862
+ this.#rejectShardKey(workflowID, undefined);
863
+ }
864
+ params.push(workflowIDs);
865
+ return `${qualify(alias, 'workflow_uuid')} = ANY($${params.length}::text[])`;
866
+ }
867
+ /** Condition joining two tables' rows of the same workflow. */
868
+ wfJoin(left, right) {
869
+ return `${left}.workflow_uuid = ${right}.workflow_uuid`;
870
+ }
871
+ /** Extra column values for a new row of this workflow. */
872
+ wfValues(workflowID, shardKey) {
873
+ this.#rejectShardKey(workflowID, shardKey);
874
+ return {};
875
+ }
876
+ /** Extra column values storing the shard key of a new workflow's parent. */
877
+ parentValues(_parentWorkflowID) {
878
+ return {};
879
+ }
880
+ /** Extra column values storing the shard key of the workflow a step refers to. */
881
+ childValues(_childWorkflowID) {
882
+ return {};
883
+ }
884
+ /** Extra (column, expression) pairs for an INSERT ... SELECT of this workflow's rows. */
885
+ wfColumns(_workflowID, _params) {
886
+ return [];
887
+ }
888
+ /** Conflict target for a unique key of a workflow table. */
889
+ keyConflict(...columns) {
890
+ return columns.join(', ');
891
+ }
892
+ /** Insert rows into `table` in one statement, returning the RETURNING rows; a sharded subclass inserts row by row. */
893
+ async insertRows(client, table, columns, rows, suffix = '') {
894
+ if (rows.length === 0)
895
+ return [];
896
+ const params = [];
897
+ const tuples = rows.map((row) => `(${row.map((value) => `$${params.push(value)}`).join(', ')})`);
898
+ const { rows: returned } = await client.query(`INSERT INTO "${this.schemaName}".${table} (${columns.join(', ')}) VALUES ${tuples.join(', ')} ${suffix}`, params);
899
+ return returned;
900
+ }
901
+ /** Predicate a queue claim must also meet, evaluated once before it scans; sharded, it caps claimers per key. */
902
+ claimGate(_queueName, _partitionKey, _params) {
903
+ return 'TRUE';
904
+ }
905
+ /** Statement whose `acquired` says whether a claim took its budget scope's transaction lock; no row means nothing to claim. */
906
+ claimLockQuery(queueName, partitionKey, params) {
907
+ const name = JSON.stringify(['dbos.queue_claim', this.schemaName, queueName, partitionKey ?? null]);
908
+ params.push(advisoryLockKey(name).toString());
909
+ return `SELECT pg_try_advisory_xact_lock($${params.length}) AS acquired`;
910
+ }
911
+ /** Statement whose `held` lists which of `partitionKeys` a sweep's transaction locked; no row means nothing to claim. */
912
+ partitionLockQuery(queueName, partitionKeys, params) {
913
+ const name = JSON.stringify(['dbos.partition_claim', this.schemaName, queueName]);
914
+ // Two-int keys: the queue, then the partition, hashed so every worker computes the same lock.
915
+ const lockIDs = partitionKeys.map((pk) => (0, crypto_1.createHash)('sha256').update(pk).digest().readInt32BE(0));
916
+ params.push(partitionKeys, lockIDs, Number(BigInt.asIntN(32, advisoryLockKey(name))));
917
+ const n = params.length;
918
+ return `SELECT coalesce(array_agg(p.pk), '{}') AS held
919
+ FROM unnest($${n - 2}::text[], $${n - 1}::int4[]) AS p(pk, lock_id)
920
+ WHERE pg_try_advisory_xact_lock($${n}::int4, p.lock_id)`;
921
+ }
922
+ /** Predicate confining a statement that names no single workflow to the current shard. */
923
+ scoped(_alias, _params) {
924
+ return 'TRUE';
925
+ }
926
+ /** Extra (column, expression) pairs for an INSERT ... SELECT confined to the current shard. */
927
+ scopedColumns(_params) {
928
+ return [];
929
+ }
930
+ /** Predicate selecting a global table's rows (queues, schedules, versions). */
931
+ global(_alias, _params) {
932
+ return 'TRUE';
933
+ }
934
+ /** Extra column values for a new row of a global table. */
935
+ globalValues() {
936
+ return {};
937
+ }
938
+ /** Predicate selecting rows whose `column` holds one of these shard keys. */
939
+ keyIn(_alias, _shardKeys, _params, _column = 'shard_key') {
940
+ throw new error_1.DBOSError('Filtering by shard key requires a sharded system database');
941
+ }
942
+ /** A shard key column to read back with a row, if this handle has one. */
943
+ keyColumn(_alias, _column = 'shard_key') {
944
+ return undefined;
945
+ }
946
+ /** Extra column values storing the shard key a schedule's workflows run under. */
947
+ scheduleValues(schedule) {
948
+ if (schedule.shardKey !== undefined && schedule.shardKey !== null) {
949
+ throw new error_1.DBOSError('Scheduling under a shard key requires a sharded system database');
950
+ }
951
+ return {};
952
+ }
953
+ /** The column holding the shard key a schedule's workflows run under, if any. */
954
+ scheduleKeyColumn() {
955
+ return undefined;
956
+ }
957
+ /** This workflow's shard key, resolved now for a statement or waiter that needs it later. */
958
+ keyHint(workflowID) {
959
+ this.#rejectShardKey(workflowID, undefined);
960
+ return undefined;
961
+ }
962
+ /** The notification-map key for a workflow's topic or key, distinct per shard key as IDs repeat across keys. */
963
+ waiter(workflowID, name) {
964
+ const key = this.keyHint(workflowID);
965
+ const payload = `${workflowID}::${name}`;
966
+ return key === undefined ? payload : `${[...key].length}:${key}:${payload}`;
967
+ }
968
+ /** Bind a workflow to `key`, else to the running workflow's own key even where a same-ID target is bound. */
969
+ ownKey(workflowID, key, callback) {
970
+ if (key === undefined) {
971
+ const ctx = (0, context_1.getCurrentContextStore)();
972
+ if (ctx !== undefined && ctx.workflowId === workflowID) {
973
+ key = ctx.shardKey;
974
+ }
975
+ }
976
+ return (0, shard_keys_1.bindShardKeys)([[workflowID, key]], callback);
977
+ }
978
+ /** Run `callback` under this application's lock for a background job, passing whether it was taken. */
979
+ async backgroundJobLock(jobName, callback) {
980
+ // CockroachDB grants every advisory lock, so each process runs every job.
981
+ if (await this.#isCockroach()) {
982
+ return await callback(true);
983
+ }
984
+ // Leading 8 bytes of SHA-256, big-endian signed, as for the retention lock.
985
+ const key = advisoryLockKey(`dbos.background_job.${this.schemaName}.${this.appName ?? ''}.${jobName}`);
986
+ return await this.lockedBackgroundJob(jobName, `SELECT pg_try_advisory_xact_lock($1) AS acquired FROM set_config('idle_in_transaction_session_timeout', $2, true)`, [key.toString(), String(exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS)], callback);
987
+ }
988
+ /** Take a transaction-scoped background job lock with `lockQuery`, which returns `acquired`, and run `callback`. */
989
+ async lockedBackgroundJob(jobName, lockQuery, lockParams, callback) {
990
+ const client = await this.connect();
991
+ try {
992
+ await client.query('BEGIN');
993
+ // The local idle timeout ends a frozen holder's session, releasing its lock.
994
+ const { rows } = await client.query(lockQuery, lockParams);
995
+ return await callback(rows[0]?.acquired === true);
996
+ }
997
+ finally {
998
+ try {
999
+ await client.query('ROLLBACK');
1000
+ client.release();
1001
+ }
1002
+ catch (e) {
1003
+ // A session the idle timeout ended has released the lock already.
1004
+ this.logger.debug(`Could not release the ${jobName} lock: ${String(e)}`);
1005
+ client.release(e instanceof Error ? e : true);
1006
+ }
1007
+ }
1008
+ }
1009
+ #cockroach = undefined;
1010
+ /** Whether this handle's database is CockroachDB, read on first use. */
1011
+ #isCockroach() {
1012
+ this.#cockroach ??= (async () => {
1013
+ const client = await this.connect();
1014
+ try {
1015
+ return await isCockroachDB(client);
1016
+ }
1017
+ finally {
1018
+ client.release();
1019
+ }
1020
+ })();
1021
+ // A failed read is retried on next use, rather than cached.
1022
+ this.#cockroach.catch(() => {
1023
+ this.#cockroach = undefined;
1024
+ });
1025
+ return this.#cockroach;
1026
+ }
1027
+ /** Scopes a queue's workers dequeue from in turn; one unkeyed scope unless sharded. */
1028
+ dequeueScopes(_queueName) {
1029
+ return Promise.resolve([undefined]);
1030
+ }
1031
+ /** Whether a dequeue failed because another worker holds the rows it needs. */
1032
+ isLockConflict(e) {
1033
+ return isPgDatabaseError(e) && e.code === '55P03';
1034
+ }
1035
+ /** Whether a dequeue failed on a serialization conflict with a concurrent one. */
1036
+ isSerializationConflict(e) {
1037
+ return isPgDatabaseError(e) && e.code === '40001';
1038
+ }
1039
+ /** Whether `appVersion` is this application's latest registered version, or none is registered. */
1040
+ async isLatestVersion(appVersion, client) {
1041
+ // Own plus unclaimed: a named peer's deploy must not demote this one.
1042
+ const latest = await this.#latestApplicationVersionName(client ?? this.pool);
1043
+ return latest === undefined || latest === appVersion;
1044
+ }
730
1045
  /**
731
1046
  * The version name a dequeue treats as latest: this application's own plus unclaimed
732
1047
  * ones, so a named peer's deploy does not demote this one.
@@ -737,15 +1052,17 @@ class SystemDatabase {
737
1052
  async #latestApplicationVersion(client, applicationName) {
738
1053
  const columns = 'version_id, version_name, version_timestamp, created_at, application_name';
739
1054
  const table = `"${this.schemaName}".application_versions`;
1055
+ const params = applicationName ? [applicationName] : [];
1056
+ const scope = this.global(undefined, params);
740
1057
  // Probe each eligible owner separately; an OR filter can scan an unbounded history of newer peer versions.
741
1058
  // Keep the constant owner in the ordering so the IS NULL probe uses the composite index too.
742
1059
  const query = applicationName
743
- ? `(SELECT ${columns} FROM ${table} WHERE application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
1060
+ ? `(SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
744
1061
  UNION ALL
745
- (SELECT ${columns} FROM ${table} WHERE application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
1062
+ (SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
746
1063
  ORDER BY version_timestamp DESC LIMIT 1`
747
- : `SELECT ${columns} FROM ${table} ORDER BY version_timestamp DESC LIMIT 1`;
748
- const { rows } = await client.query(query, applicationName ? [applicationName] : []);
1064
+ : `SELECT ${columns} FROM ${table} WHERE ${scope} ORDER BY version_timestamp DESC LIMIT 1`;
1065
+ const { rows } = await client.query(query, params);
749
1066
  return rows[0];
750
1067
  }
751
1068
  /**
@@ -753,7 +1070,8 @@ class SystemDatabase {
753
1070
  * leaves the owner intact; a named one collides only with a different name.
754
1071
  */
755
1072
  async #resolveRowOwner(client, table, keyColumn, name, owner, kind) {
756
- const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${keyColumn} = $1`, [name]);
1073
+ const params = [name];
1074
+ const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${this.global(undefined, params)} AND ${keyColumn} = $1`, params);
757
1075
  const current = rows[0]?.application_name ?? null;
758
1076
  if (current === null)
759
1077
  return owner;
@@ -770,8 +1088,12 @@ class SystemDatabase {
770
1088
  }
771
1089
  /** Migrates the system database, or, when `runMigrations` is false, verifies it is already migrated. */
772
1090
  async init(runMigrations = true) {
773
- const migrateOrVerify = runMigrations ? ensureSystemDatabase : verifySystemDatabase;
774
- await migrateOrVerify(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
1091
+ if (runMigrations) {
1092
+ await this.runMigrations();
1093
+ }
1094
+ else {
1095
+ await this.verifyMigrations();
1096
+ }
775
1097
  if (this.shouldUseDBNotifications) {
776
1098
  await this.#listenForNotifications();
777
1099
  // Push coalesced stream and event notifications off the write path.
@@ -779,6 +1101,32 @@ class SystemDatabase {
779
1101
  this.#notifierLoop = this.#runNotifier();
780
1102
  }
781
1103
  }
1104
+ /** Create or migrate the system database. */
1105
+ async runMigrations() {
1106
+ await ensureSystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
1107
+ }
1108
+ /** Throw if this unsharded handle's schema is sharded; an unreachable database is left for its first statement. */
1109
+ async checkShardedSchema() {
1110
+ if (this.sharded)
1111
+ return;
1112
+ let client;
1113
+ try {
1114
+ client = await this.connect();
1115
+ }
1116
+ catch {
1117
+ return;
1118
+ }
1119
+ try {
1120
+ await refuseShardedSchema(client, this.schemaName);
1121
+ }
1122
+ finally {
1123
+ client.release();
1124
+ }
1125
+ }
1126
+ /** Check the system database is migrated, creating and changing nothing. */
1127
+ async verifyMigrations() {
1128
+ await verifySystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
1129
+ }
782
1130
  async destroy() {
783
1131
  // Set synchronously, before any await, so no reconnect is scheduled or published after this point.
784
1132
  this.#notificationsStopped = true;
@@ -824,7 +1172,7 @@ class SystemDatabase {
824
1172
  /** Runs on `client` if given, joining its transaction; otherwise in its own retried transaction. */
825
1173
  async initWorkflowStatus(initStatus, creatorXid, client, reusePolicy = 'return-existing') {
826
1174
  if (client !== undefined) {
827
- return await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
1175
+ return await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
828
1176
  }
829
1177
  return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
830
1178
  }
@@ -833,7 +1181,7 @@ class SystemDatabase {
833
1181
  let shouldCommit = false;
834
1182
  try {
835
1183
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
836
- const result = await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
1184
+ const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
837
1185
  // If there is an existing DB record and we aren't here to recover it, leave it be.
838
1186
  shouldCommit = result.shouldExecuteOnThisExecutor;
839
1187
  return result;
@@ -861,10 +1209,12 @@ class SystemDatabase {
861
1209
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
862
1210
  const parentOwnerXid = (0, context_1.currentOwnerXid)(parentWorkflowID);
863
1211
  if (parentOwnerXid !== undefined) {
864
- await this.#checkOwner(client, parentWorkflowID, parentOwnerXid);
1212
+ await this.checkOwner(client, parentWorkflowID, parentOwnerXid);
865
1213
  }
866
- const result = await this.#initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
867
- await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs, { childWorkflowID: initStatus.workflowUUID, ownerChecked: true });
1214
+ const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
1215
+ await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs,
1216
+ // Unkeyed, as only a sharded database records the child's key, through its own override.
1217
+ { childWorkflowID: initStatus.workflowUUID, ownerChecked: true, childValues: {} });
868
1218
  shouldCommit = true;
869
1219
  return result;
870
1220
  }
@@ -883,7 +1233,7 @@ class SystemDatabase {
883
1233
  }
884
1234
  }
885
1235
  }
886
- async #initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy) {
1236
+ async initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy) {
887
1237
  const resRow = await this.insertWorkflowStatus(client, initStatus, creatorXid);
888
1238
  if (reusePolicy === 'reject' && (creatorXid === null || resRow.creator_xid !== creatorXid)) {
889
1239
  throw new error_1.DBOSWorkflowIDInUseError(initStatus.workflowUUID, resRow.status, resRow.name);
@@ -915,6 +1265,7 @@ class SystemDatabase {
915
1265
  async deadLetterWorkflows(workflowIDs, minRecoveryAttempts) {
916
1266
  if (workflowIDs.length === 0)
917
1267
  return;
1268
+ const params = [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflow_1.StatusString.PENDING, minRecoveryAttempts];
918
1269
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
919
1270
  SET status = $1,
920
1271
  owner_xid = NULL,
@@ -923,17 +1274,18 @@ class SystemDatabase {
923
1274
  queue_name = NULL,
924
1275
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
925
1276
  completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
926
- WHERE workflow_uuid = ANY($2::text[]) AND status = $3 AND recovery_attempts >= $4`, [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflowIDs, workflow_1.StatusString.PENDING, minRecoveryAttempts]);
1277
+ WHERE ${this.wfIn(undefined, workflowIDs, params)} AND status = $2 AND recovery_attempts >= $3`, params);
927
1278
  }
928
1279
  /** Highest created_at among still-active rows per partition key, used to seed the in-memory cursor. */
929
1280
  async #maxPartitionKeyCreatedAt(keys) {
930
1281
  const maxima = new Map();
931
1282
  if (keys.length === 0)
932
1283
  return maxima;
1284
+ const params = [keys, [workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING]];
933
1285
  const { rows } = await this.pool.query(`SELECT queue_partition_key, MAX(created_at) AS max_created_at
934
1286
  FROM "${this.schemaName}".workflow_status
935
- WHERE queue_partition_key = ANY($1) AND status = ANY($2)
936
- GROUP BY queue_partition_key`, [keys, [workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING]]);
1287
+ WHERE ${this.scoped(undefined, params)} AND queue_partition_key = ANY($1) AND status = ANY($2)
1288
+ GROUP BY queue_partition_key`, params);
937
1289
  for (const row of rows) {
938
1290
  if (row.max_created_at !== null) {
939
1291
  maxima.set(row.queue_partition_key, Number(row.max_created_at));
@@ -1039,39 +1391,60 @@ class SystemDatabase {
1039
1391
  const chunkSize = 500;
1040
1392
  for (let start = 0; start < statuses.length; start += chunkSize) {
1041
1393
  const chunk = statuses.slice(start, start + chunkSize);
1042
- const tuples = [];
1043
- const params = [];
1044
- let paramIdx = 1;
1045
- for (let i = 0; i < chunk.length; i++) {
1046
- const status = chunk[i];
1394
+ const keyed = chunk.map((status) => ({
1395
+ ...this.wfValues(status.workflowUUID, status.shardKey),
1396
+ ...this.parentValues(status.parentWorkflowID),
1397
+ }));
1398
+ const rows = chunk.map((status, i) => {
1047
1399
  const createdAt = createdAts[start + i];
1048
- tuples.push(`(${columns.map(() => `$${paramIdx++}`).join(', ')})`);
1049
- params.push(status.workflowUUID, status.status, status.workflowName,
1050
- // For cross-language compatibility, these MUST be NULL in the database when not set
1051
- status.workflowClassName === '' ? null : status.workflowClassName, status.workflowConfigName === '' ? null : status.workflowConfigName, status.queueName ?? null, status.authenticatedUser, status.assumedRole, JSON.stringify(status.authenticatedRoles), status.executorId, status.applicationVersion ?? null, status.applicationID, createdAt, 0, createdAt, status.timeoutMS ?? null, status.deadlineEpochMS ?? null,
1052
- // Legacy column: the payload lives in workflow_input.
1053
- null, null, status.priority, status.queuePartitionKey ?? null, status.parentWorkflowID ?? null, status.serialization, null, status.delayUntilEpochMS ?? null, status.attributes ? JSON.stringify(status.attributes) : null, status.scheduleName ?? null, status.applicationName ?? null);
1054
- }
1055
- const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
1056
- VALUES ${tuples.join(', ')}
1057
- ON CONFLICT (workflow_uuid) DO NOTHING
1058
- RETURNING workflow_uuid`, params);
1059
- for (const row of rows) {
1400
+ return [
1401
+ ...Object.values(keyed[i]),
1402
+ status.workflowUUID,
1403
+ status.status,
1404
+ status.workflowName,
1405
+ // For cross-language compatibility, these MUST be NULL in the database when not set
1406
+ status.workflowClassName === '' ? null : status.workflowClassName,
1407
+ status.workflowConfigName === '' ? null : status.workflowConfigName,
1408
+ status.queueName ?? null,
1409
+ status.authenticatedUser,
1410
+ status.assumedRole,
1411
+ JSON.stringify(status.authenticatedRoles),
1412
+ status.executorId,
1413
+ status.applicationVersion ?? null,
1414
+ status.applicationID,
1415
+ createdAt,
1416
+ 0,
1417
+ createdAt,
1418
+ status.timeoutMS ?? null,
1419
+ status.deadlineEpochMS ?? null,
1420
+ // Legacy column: the payload lives in workflow_input.
1421
+ null,
1422
+ null,
1423
+ status.priority,
1424
+ status.queuePartitionKey ?? null,
1425
+ status.parentWorkflowID ?? null,
1426
+ status.serialization,
1427
+ null,
1428
+ status.delayUntilEpochMS ?? null,
1429
+ status.attributes ? JSON.stringify(status.attributes) : null,
1430
+ status.scheduleName ?? null,
1431
+ status.applicationName ?? null,
1432
+ ];
1433
+ });
1434
+ const returned = await this.insertRows(client, 'workflow_status', [...Object.keys(keyed[0]), ...columns], rows, `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING RETURNING workflow_uuid`);
1435
+ for (const row of returned) {
1060
1436
  inserted.add(row.workflow_uuid);
1061
1437
  }
1062
1438
  }
1063
1439
  for (let start = 0; start < statuses.length; start += chunkSize) {
1064
1440
  const chunk = statuses.slice(start, start + chunkSize);
1065
- const tuples = [];
1066
- const params = [];
1067
- let paramIdx = 1;
1068
- for (let i = 0; i < chunk.length; i++) {
1069
- tuples.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
1070
- params.push(chunk[i].workflowUUID, chunk[i].input, createdAts[start + i]);
1071
- }
1072
- await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
1073
- VALUES ${tuples.join(', ')}
1074
- ON CONFLICT (workflow_uuid) DO NOTHING`, params);
1441
+ const keyed = chunk.map((status) => this.wfValues(status.workflowUUID, status.shardKey));
1442
+ await this.insertRows(client, 'workflow_input', [...Object.keys(keyed[0]), 'workflow_uuid', 'inputs', 'retention_timestamp'], chunk.map((status, i) => [
1443
+ ...Object.values(keyed[i]),
1444
+ status.workflowUUID,
1445
+ status.input,
1446
+ createdAts[start + i],
1447
+ ]), `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`);
1075
1448
  }
1076
1449
  await client.query('COMMIT');
1077
1450
  }
@@ -1085,22 +1458,10 @@ class SystemDatabase {
1085
1458
  return inserted;
1086
1459
  }
1087
1460
  async recordWorkflowOutput(workflowID, status, ownerXid) {
1088
- const client = await this.connect();
1089
- try {
1090
- return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
1091
- }
1092
- finally {
1093
- client.release();
1094
- }
1461
+ return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
1095
1462
  }
1096
1463
  async recordWorkflowError(workflowID, status, ownerXid) {
1097
- const client = await this.connect();
1098
- try {
1099
- return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
1100
- }
1101
- finally {
1102
- client.release();
1103
- }
1464
+ return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
1104
1465
  }
1105
1466
  // Record a workflow's terminal outcome (SUCCESS or ERROR), reporting whether
1106
1467
  // the write landed. The write applies only to a PENDING row still owned by
@@ -1111,8 +1472,49 @@ class SystemDatabase {
1111
1472
  // terminal, handed to another execution (e.g. by a concurrent resume or
1112
1473
  // recovery), or deleted; the caller resolves which by awaiting the recorded
1113
1474
  // outcome.
1114
- async #recordWorkflowOutcome(client, workflowID, status, outcome, ownerXid) {
1475
+ async #recordWorkflowOutcome(workflowID, status, outcome, ownerXid) {
1115
1476
  ownerXid ??= (0, context_1.currentOwnerXid)(workflowID);
1477
+ if (!(await this.outcomeInOneStatement())) {
1478
+ return await this.#recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid);
1479
+ }
1480
+ // One autocommit round trip; the upsert reads the UPDATE's rows, so the output lands only with the status.
1481
+ const params = [status, workflow_1.StatusString.PENDING, outcome.output ?? null, outcome.error ?? null];
1482
+ const target = this.wf(undefined, workflowID, params);
1483
+ const owned = ownerXid !== undefined ? `AND owner_xid = $${params.push(ownerXid)}` : '';
1484
+ const keys = this.wfValues(workflowID);
1485
+ const keyColumns = Object.keys(keys);
1486
+ const keyExprs = Object.values(keys).map((value) => `$${params.push(value)}::text`);
1487
+ const workflowParam = `$${params.push(workflowID)}::text`;
1488
+ const { rows } = await this.pool.query(`WITH updated AS (
1489
+ UPDATE "${this.schemaName}".workflow_status
1490
+ SET status = $1,
1491
+ -- As the workflow is complete, remove its deduplication ID; a finished workflow has no owner.
1492
+ deduplication_id = NULL,
1493
+ owner_xid = NULL,
1494
+ updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
1495
+ completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1496
+ WHERE ${target} AND status = $2 ${owned}
1497
+ RETURNING 1
1498
+ ), stored AS (
1499
+ INSERT INTO "${this.schemaName}".workflow_output (${[...keyColumns, 'workflow_uuid', 'output', 'error'].join(', ')})
1500
+ SELECT ${[...keyExprs, workflowParam, '$3::text', '$4::text'].join(', ')}
1501
+ WHERE EXISTS (SELECT 1 FROM updated)
1502
+ -- EXCLUDED is the row this INSERT proposed, so these are the values above.
1503
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error
1504
+ -- CockroachDB refuses a data-modifying CTE without RETURNING.
1505
+ RETURNING 1
1506
+ )
1507
+ -- Selecting from the CTEs rather than inserting at the top level lets a router see the shard key.
1508
+ SELECT count(*) AS count FROM updated`, params);
1509
+ return Number(rows[0]?.count ?? 0) > 0;
1510
+ }
1511
+ /** Whether a workflow's outcome is written in one statement, rather than a transaction of two. */
1512
+ outcomeInOneStatement() {
1513
+ return Promise.resolve(true);
1514
+ }
1515
+ /** The outcome write as a transaction of two statements, for a backend that cannot run it as one. */
1516
+ async #recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid) {
1517
+ const client = await this.connect();
1116
1518
  let committed = false;
1117
1519
  try {
1118
1520
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
@@ -1126,19 +1528,31 @@ class SystemDatabase {
1126
1528
  // The outcome was not ours to write, so leave no orphan payload.
1127
1529
  return false;
1128
1530
  }
1129
- await client.query(`INSERT INTO "${this.schemaName}".workflow_output (workflow_uuid, output, error)
1130
- VALUES ($1, $2, $3)
1131
- ON CONFLICT (workflow_uuid) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error`, [workflowID, outcome.output ?? null, outcome.error ?? null]);
1531
+ const values = {
1532
+ ...this.wfValues(workflowID),
1533
+ workflow_uuid: workflowID,
1534
+ output: outcome.output ?? null,
1535
+ error: outcome.error ?? null,
1536
+ };
1537
+ const columns = Object.keys(values);
1538
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_output (${columns.join(', ')})
1539
+ VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
1540
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error`, Object.values(values));
1132
1541
  committed = true;
1133
1542
  return true;
1134
1543
  }
1135
1544
  finally {
1136
- if (committed) {
1137
- await client.query('COMMIT');
1545
+ try {
1546
+ if (committed) {
1547
+ await client.query('COMMIT');
1548
+ }
1549
+ else {
1550
+ // Swallowed: a rollback failure must not mask why we are rolling back.
1551
+ await client.query('ROLLBACK').catch(() => undefined);
1552
+ }
1138
1553
  }
1139
- else {
1140
- // Swallowed: a rollback failure must not mask why we are rolling back.
1141
- await client.query('ROLLBACK').catch(() => undefined);
1554
+ finally {
1555
+ client.release();
1142
1556
  }
1143
1557
  }
1144
1558
  }
@@ -1147,6 +1561,7 @@ class SystemDatabase {
1147
1561
  const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
1148
1562
  // executor_id defaults to "local", so it collides across applications.
1149
1563
  const scope = this.appNameFilter('application_name', this.appName, params);
1564
+ const keyColumn = this.keyColumn(undefined);
1150
1565
  const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1151
1566
  SET started_at_epoch_ms = NULL,
1152
1567
  status = $1,
@@ -1157,27 +1572,31 @@ class SystemDatabase {
1157
1572
  AND executor_id = $4
1158
1573
  AND application_version = $5
1159
1574
  AND ${scope}
1160
- RETURNING workflow_uuid`, params);
1161
- return result.rows.map((row) => row.workflow_uuid);
1575
+ RETURNING workflow_uuid${keyColumn !== undefined ? `, ${keyColumn} AS shard_key` : ''}`, params);
1576
+ return result.rows.map((row) => ({ workflowID: row.workflow_uuid, shardKey: row.shard_key ?? undefined }));
1162
1577
  }
1163
1578
  async getWorkflowStatus(workflowID, callerID, callerFN) {
1579
+ // Resolved before binding the caller's own key, which a same-ID target would share.
1580
+ const shardKey = this.keyHint(workflowID);
1164
1581
  // Inside a workflow this runs on the transaction's client: taking a second pool connection while
1165
1582
  // holding the first deadlocks the pool once every connection is held by a waiting caller.
1166
1583
  const funcGetStatus = async (client) => {
1167
- const statuses = await this.#listWorkflows({ workflowIDs: [workflowID] }, client);
1584
+ const statuses = await this.#listWorkflows({ workflowIDs: [workflowID], shardKey }, {}, client);
1168
1585
  const status = statuses.find((s) => s.workflowUUID === workflowID);
1169
1586
  return status ? JSON.stringify(status) : null;
1170
1587
  };
1171
1588
  if (callerID && callerFN !== undefined) {
1172
- const client = await this.connect();
1173
- try {
1174
- // Check if the operation has been done before for OAOO (only do this inside a workflow).
1175
- const json = await this.#inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, () => funcGetStatus(client)));
1176
- return parseStatus(json);
1177
- }
1178
- finally {
1179
- client.release();
1180
- }
1589
+ const json = await this.ownKey(callerID, undefined, async () => {
1590
+ const client = await this.connect();
1591
+ try {
1592
+ // Check if the operation has been done before for OAOO (only do this inside a workflow).
1593
+ return await this.inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, () => funcGetStatus(client)));
1594
+ }
1595
+ finally {
1596
+ client.release();
1597
+ }
1598
+ });
1599
+ return parseStatus(json);
1181
1600
  }
1182
1601
  else {
1183
1602
  const json = await funcGetStatus();
@@ -1191,7 +1610,7 @@ class SystemDatabase {
1191
1610
  statusFetchChunkSize = 500;
1192
1611
  // Retried per chunk so a reconnect refetches one chunk, not every chunk before it.
1193
1612
  async fetchWorkflowStatusChunk(workflowIDs) {
1194
- return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false });
1613
+ return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false }, { keyedIDs: true });
1195
1614
  }
1196
1615
  /** Fetch many statuses in as few round trips as possible. IDs with no row are omitted. */
1197
1616
  async getWorkflowStatuses(workflowIDs) {
@@ -1222,23 +1641,26 @@ class SystemDatabase {
1222
1641
  }
1223
1642
  }
1224
1643
  // ==================== Step Results ====================
1225
- async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
1226
- const client = await this.connect();
1227
- try {
1228
- return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
1229
- }
1230
- finally {
1231
- client.release();
1232
- }
1644
+ async getOperationResultAndThrowIfCancelled(workflowID, functionID, ownerKey) {
1645
+ return await this.ownKey(workflowID, ownerKey, async () => {
1646
+ const client = await this.connect();
1647
+ try {
1648
+ return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
1649
+ }
1650
+ finally {
1651
+ client.release();
1652
+ }
1653
+ });
1233
1654
  }
1234
1655
  async getAllOperationResults(workflowID, limit, offset, loadOutput = true) {
1235
1656
  // Skip the output/error payload columns when the caller does not need them,
1236
1657
  // mirroring how listWorkflows skips its payload joins when loadOutput is false.
1658
+ const childKey = this.keyColumn(undefined, 'child_shard_key');
1237
1659
  const columns = loadOutput
1238
1660
  ? '*'
1239
- : 'workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp';
1240
- let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE workflow_uuid=$1 ORDER BY function_id`;
1241
- const params = [workflowID];
1661
+ : `workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? `, ${childKey}` : ''}`;
1662
+ const params = [];
1663
+ let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE ${this.wf(undefined, workflowID, params)} ORDER BY function_id`;
1242
1664
  if (limit !== undefined) {
1243
1665
  params.push(limit);
1244
1666
  query += ` LIMIT $${params.length}`;
@@ -1251,14 +1673,18 @@ class SystemDatabase {
1251
1673
  return rows;
1252
1674
  }
1253
1675
  async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
1254
- const client = await this.connect();
1255
- try {
1256
- await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options));
1257
- }
1258
- finally {
1259
- client.release();
1260
- await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
1261
- }
1676
+ // Resolved before binding the caller's own key, which a same-ID child would share.
1677
+ const childValues = this.childValues(options.childWorkflowID);
1678
+ await this.ownKey(workflowID, options.ownerKey, async () => {
1679
+ const client = await this.connect();
1680
+ try {
1681
+ await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, { ...options, childValues }));
1682
+ }
1683
+ finally {
1684
+ client.release();
1685
+ await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
1686
+ }
1687
+ });
1262
1688
  }
1263
1689
  /** Checkpoint an operation's failure so a replay re-throws it instead of re-running. */
1264
1690
  async recordOperationError(workflowID, functionID, functionName, startTimeEpochMs, error, childWorkflowID) {
@@ -1332,9 +1758,10 @@ class SystemDatabase {
1332
1758
  throw new TypeError('functionID must be defined');
1333
1759
  patchName = `DBOS.patch-${patchName}`;
1334
1760
  const readCheckpointName = async (queryable) => {
1761
+ const params = [functionID];
1335
1762
  const { rows } = await queryable.query(`SELECT function_name
1336
1763
  FROM "${this.schemaName}".operation_outputs
1337
- WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
1764
+ WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
1338
1765
  return rows.length === 0 ? undefined : rows[0].function_name;
1339
1766
  };
1340
1767
  if (deprecated) {
@@ -1349,7 +1776,7 @@ class SystemDatabase {
1349
1776
  // If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
1350
1777
  const client = await this.connect();
1351
1778
  try {
1352
- return await this.#inTransaction(client, async () => {
1779
+ return await this.inTransaction(client, async () => {
1353
1780
  const checkpointName = await readCheckpointName(client);
1354
1781
  if (checkpointName !== undefined) {
1355
1782
  const matches = checkpointName === patchName;
@@ -1375,7 +1802,7 @@ class SystemDatabase {
1375
1802
  let frontier = workflowIDs;
1376
1803
  while (frontier.length > 0) {
1377
1804
  await this.#cancelWorkflows(frontier);
1378
- const children = await this.#getDirectChildren(frontier);
1805
+ const children = await this.getDirectChildren(frontier);
1379
1806
  frontier = children.filter((id) => !visited.has(id));
1380
1807
  for (const id of frontier) {
1381
1808
  visited.add(id);
@@ -1383,41 +1810,44 @@ class SystemDatabase {
1383
1810
  }
1384
1811
  }
1385
1812
  async #cancelWorkflows(workflowIDs) {
1813
+ const params = [workflow_1.StatusString.CANCELLED, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR];
1386
1814
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1387
1815
  SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
1388
1816
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
1389
1817
  completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1390
- WHERE workflow_uuid = ANY($2)
1391
- AND status NOT IN ($3, $4)`, [workflow_1.StatusString.CANCELLED, workflowIDs, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR]);
1818
+ WHERE ${this.wfIn(undefined, workflowIDs, params)}
1819
+ AND status NOT IN ($2, $3)`, params);
1392
1820
  }
1393
1821
  async checkIfCanceled(workflowID) {
1394
- await this.#checkIfCanceled(this.pool, workflowID);
1822
+ // The caller's own workflow, even where a same-ID workflow is bound as a target.
1823
+ await this.ownKey(workflowID, undefined, () => this.#checkIfCanceled(this.pool, workflowID));
1395
1824
  }
1396
1825
  async resumeWorkflows(workflowIDs, queueName) {
1397
1826
  const client = await this.connect();
1398
1827
  try {
1399
1828
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1400
1829
  // Check existence separately: a zero-row update also means "already complete", a legal no-op.
1401
- const { rows } = await client.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [workflowIDs]);
1830
+ const existingParams = [];
1831
+ const { rows } = await client.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${this.wfIn(undefined, workflowIDs, existingParams)}`, existingParams);
1402
1832
  const existing = new Set(rows.map((r) => r.workflow_uuid));
1403
1833
  const missing = workflowIDs.filter((id) => !existing.has(id));
1404
1834
  if (missing.length > 0) {
1405
1835
  throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${missing.join(', ')} does not exist`);
1406
1836
  }
1837
+ const updateParams = [
1838
+ workflow_1.StatusString.ENQUEUED,
1839
+ queueName ?? utils_1.INTERNAL_QUEUE_NAME,
1840
+ workflow_1.StatusString.SUCCESS,
1841
+ workflow_1.StatusString.ERROR,
1842
+ ];
1407
1843
  await client.query(`UPDATE "${this.schemaName}".workflow_status
1408
1844
  SET status = $1, owner_xid = NULL, queue_name = $2, recovery_attempts = 0,
1409
1845
  workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
1410
1846
  started_at_epoch_ms = NULL,
1411
1847
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
1412
1848
  completed_at = NULL
1413
- WHERE workflow_uuid = ANY($3)
1414
- AND status NOT IN ($4, $5)`, [
1415
- workflow_1.StatusString.ENQUEUED,
1416
- queueName ?? utils_1.INTERNAL_QUEUE_NAME,
1417
- workflowIDs,
1418
- workflow_1.StatusString.SUCCESS,
1419
- workflow_1.StatusString.ERROR,
1420
- ]);
1849
+ WHERE ${this.wfIn(undefined, workflowIDs, updateParams)}
1850
+ AND status NOT IN ($3, $4)`, updateParams);
1421
1851
  await client.query('COMMIT');
1422
1852
  }
1423
1853
  catch (e) {
@@ -1430,16 +1860,18 @@ class SystemDatabase {
1430
1860
  }
1431
1861
  }
1432
1862
  async setWorkflowPriority(workflowID, priority) {
1863
+ const params = [priority, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED];
1433
1864
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1434
1865
  SET priority = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1435
- WHERE workflow_uuid = $2
1436
- AND status IN ($3, $4)`, [priority, workflowID, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED]);
1866
+ WHERE ${this.wf(undefined, workflowID, params)}
1867
+ AND status IN ($2, $3)`, params);
1437
1868
  }
1438
1869
  async setWorkflowDelay(workflowID, delayUntilEpochMS) {
1870
+ const params = [delayUntilEpochMS, workflow_1.StatusString.DELAYED];
1439
1871
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1440
1872
  SET delay_until_epoch_ms = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1441
- WHERE workflow_uuid = $2
1442
- AND status = $3`, [delayUntilEpochMS, workflowID, workflow_1.StatusString.DELAYED]);
1873
+ WHERE ${this.wf(undefined, workflowID, params)}
1874
+ AND status = $2`, params);
1443
1875
  }
1444
1876
  /**
1445
1877
  * Extend an existing debounced DELAYED workflow's delay and update its inputs, atomically.
@@ -1505,7 +1937,7 @@ class SystemDatabase {
1505
1937
  if (updated.rows.length > 0) {
1506
1938
  await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
1507
1939
  VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
1508
- ON CONFLICT (workflow_uuid) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
1940
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
1509
1941
  return {
1510
1942
  bouncedWorkflowID: updated.rows[0].workflow_uuid,
1511
1943
  holderWorkflowID: null,
@@ -1515,10 +1947,11 @@ class SystemDatabase {
1515
1947
  holderApplicationName: null,
1516
1948
  };
1517
1949
  }
1518
- // Unscoped, so a holder that blocked the update above is reportable.
1950
+ // No application filter, so a holder that blocked the update above is reportable.
1951
+ const holderParams = [params.queueName, params.deduplicationID];
1519
1952
  const holder = await client.query(`SELECT workflow_uuid, is_debounced, name, class_name, application_name
1520
1953
  FROM "${this.schemaName}".workflow_status
1521
- WHERE queue_name = $1 AND deduplication_id = $2`, [params.queueName, params.deduplicationID]);
1954
+ WHERE ${this.scoped(undefined, holderParams)} AND queue_name = $1 AND deduplication_id = $2`, holderParams);
1522
1955
  if (holder.rows.length === 0) {
1523
1956
  return {
1524
1957
  bouncedWorkflowID: null,
@@ -1539,7 +1972,7 @@ class SystemDatabase {
1539
1972
  };
1540
1973
  }
1541
1974
  // Get the immediate (one-level) child workflow IDs for a set of workflows.
1542
- async #getDirectChildren(workflowIDs) {
1975
+ async getDirectChildren(workflowIDs) {
1543
1976
  if (workflowIDs.length === 0) {
1544
1977
  return [];
1545
1978
  }
@@ -1553,7 +1986,7 @@ class SystemDatabase {
1553
1986
  const descendants = new Set();
1554
1987
  let frontier = [workflowID];
1555
1988
  while (frontier.length > 0) {
1556
- const children = await this.#getDirectChildren(frontier);
1989
+ const children = await this.getDirectChildren(frontier);
1557
1990
  frontier = children.filter((id) => !descendants.has(id));
1558
1991
  for (const id of frontier) {
1559
1992
  descendants.add(id);
@@ -1561,6 +1994,7 @@ class SystemDatabase {
1561
1994
  }
1562
1995
  return [...descendants];
1563
1996
  }
1997
+ /** Delete workflows and all associated data, and with deleteChildren every workflow they started, recursively. */
1564
1998
  async deleteWorkflows(workflowIDs, deleteChildren = false) {
1565
1999
  const allIds = [...workflowIDs];
1566
2000
  if (deleteChildren) {
@@ -1568,15 +2002,18 @@ class SystemDatabase {
1568
2002
  allIds.push(...(await this.getWorkflowChildren(wfid)));
1569
2003
  }
1570
2004
  }
2005
+ await this.deleteWorkflowsInternal(allIds);
2006
+ }
2007
+ async deleteWorkflowsInternal(allIds) {
1571
2008
  const client = await this.connect();
1572
2009
  try {
1573
2010
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1574
2011
  // The payload tables carry no foreign key, so the status delete does not cascade
1575
2012
  // into them; without this they survive as orphans.
1576
- for (const table of ['workflow_input', 'workflow_output', 'operation_outputs']) {
1577
- await client.query(`DELETE FROM "${this.schemaName}".${table} WHERE workflow_uuid = ANY($1)`, [allIds]);
2013
+ for (const table of ['workflow_input', 'workflow_output', 'operation_outputs', 'workflow_status']) {
2014
+ const params = [];
2015
+ await client.query(`DELETE FROM "${this.schemaName}".${table} WHERE ${this.wfIn(undefined, allIds, params)}`, params);
1578
2016
  }
1579
- await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [allIds]);
1580
2017
  await client.query('COMMIT');
1581
2018
  }
1582
2019
  catch (e) {
@@ -1617,7 +2054,8 @@ class SystemDatabase {
1617
2054
  const client = await this.connect();
1618
2055
  try {
1619
2056
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1620
- const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE workflow_uuid = $1`, [workflowID]);
2057
+ const statusParams = [];
2058
+ const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE ${this.wf(undefined, workflowID, statusParams)}`, statusParams);
1621
2059
  if (statusRows.length === 0) {
1622
2060
  throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
1623
2061
  }
@@ -1628,56 +2066,61 @@ class SystemDatabase {
1628
2066
  // Roll workflow_events back to the last value published before the cut, using
1629
2067
  // workflow_events_history as an undo log, then rewind that history too.
1630
2068
  // `publishedPastCut` is the correlated EXISTS both halves key off.
1631
- const publishedPastCut = `EXISTS (
2069
+ const publishedPastCut = (key, params) => `EXISTS (
1632
2070
  SELECT 1 FROM "${schema}".workflow_events_history discarded
1633
- WHERE discarded.workflow_uuid = $1 AND discarded.key = %KEY% AND discarded.function_id >= $2
2071
+ WHERE ${this.wf('discarded', workflowID, params)} AND discarded.key = ${key} AND discarded.function_id >= $1
1634
2072
  )`;
1635
2073
  // First unpublish everything the discarded run touched.
2074
+ const unpublishParams = [startStep];
1636
2075
  await client.query(`DELETE FROM "${schema}".workflow_events
1637
- WHERE workflow_uuid = $1 AND ${publishedPastCut.replace('%KEY%', '"' + schema + '".workflow_events.key')}`, [workflowID, startStep]);
2076
+ WHERE ${this.wf(undefined, workflowID, unpublishParams)}
2077
+ AND ${publishedPastCut('"' + schema + '".workflow_events.key', unpublishParams)}`, unpublishParams);
1638
2078
  // Then restore those keys, if any, to the value published below the cut.
1639
- await client.query(`INSERT INTO "${schema}".workflow_events (workflow_uuid, key, value, serialization)
1640
- SELECT surviving.workflow_uuid, surviving.key, surviving.value, surviving.serialization
2079
+ const restoreParams = [startStep];
2080
+ const keyColumns = this.wfColumns(workflowID, restoreParams);
2081
+ await client.query(`INSERT INTO "${schema}".workflow_events (${[...keyColumns.map(([c]) => c), 'workflow_uuid', 'key', 'value', 'serialization'].join(', ')})
2082
+ SELECT ${[...keyColumns.map(([, e]) => e), 'surviving.workflow_uuid', 'surviving.key', 'surviving.value', 'surviving.serialization'].join(', ')}
1641
2083
  FROM (
1642
2084
  SELECT weh.workflow_uuid, weh.key, weh.value, weh.serialization,
1643
2085
  ROW_NUMBER() OVER (PARTITION BY weh.key ORDER BY weh.function_id DESC) AS rn
1644
2086
  FROM "${schema}".workflow_events_history weh
1645
- WHERE weh.workflow_uuid = $1 AND weh.function_id < $2
1646
- AND ${publishedPastCut.replace('%KEY%', 'weh.key')}
2087
+ WHERE ${this.wf('weh', workflowID, restoreParams)} AND weh.function_id < $1
2088
+ AND ${publishedPastCut('weh.key', restoreParams)}
1647
2089
  ) surviving
1648
- WHERE surviving.rn = 1`, [workflowID, startStep]);
2090
+ WHERE surviving.rn = 1`, restoreParams);
1649
2091
  // Clear the streams' close sentinels so the replay can append again.
2092
+ const sentinelParams = [startStep, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name()];
1650
2093
  await client.query(`DELETE FROM "${schema}".streams
1651
- WHERE workflow_uuid = $1 AND function_id >= $2 AND value = $3 AND serialization = $4`, [workflowID, startStep, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name()]);
2094
+ WHERE ${this.wf(undefined, workflowID, sentinelParams)} AND function_id >= $1 AND value = $2 AND serialization = $3`, sentinelParams);
1652
2095
  // Discard the steps and the events history past the cut.
1653
2096
  for (const table of ['operation_outputs', 'workflow_events_history']) {
1654
- await client.query(`DELETE FROM "${schema}".${table} WHERE workflow_uuid = $1 AND function_id >= $2`, [
1655
- workflowID,
1656
- startStep,
1657
- ]);
2097
+ const params = [startStep];
2098
+ await client.query(`DELETE FROM "${schema}".${table} WHERE ${this.wf(undefined, workflowID, params)} AND function_id >= $1`, params);
1658
2099
  }
1659
2100
  // Delete messages consumed or received after the rewind point
1660
- await client.query(`DELETE FROM "${schema}".notifications WHERE destination_uuid = $1 AND (consumed_by_function_id >= $2 OR consumed = false)`, [workflowID, startStep]);
1661
- await client.query(`DELETE FROM "${schema}".workflow_output WHERE workflow_uuid = $1`, [workflowID]);
2101
+ const messageParams = [startStep];
2102
+ await client.query(`DELETE FROM "${schema}".notifications
2103
+ WHERE ${this.wf(undefined, workflowID, messageParams, 'destination_uuid')}
2104
+ AND (consumed_by_function_id >= $1 OR consumed = false)`, messageParams);
2105
+ const outputParams = [];
2106
+ await client.query(`DELETE FROM "${schema}".workflow_output WHERE ${this.wf(undefined, workflowID, outputParams)}`, outputParams);
1662
2107
  // Re-enqueue the workflow. Re-asserting the status we read keeps a workflow that
1663
2108
  // moved on underneath us from being resurrected.
1664
- const setVersion = options.applicationVersion !== undefined ? ', application_version = $6' : '';
1665
2109
  const params = [
1666
2110
  workflow_1.StatusString.ENQUEUED,
1667
2111
  options.queueName ?? utils_1.INTERNAL_QUEUE_NAME,
1668
2112
  options.queuePartitionKey ?? null,
1669
- workflowID,
1670
2113
  status,
1671
2114
  ];
1672
- if (options.applicationVersion !== undefined) {
1673
- params.push(options.applicationVersion);
1674
- }
2115
+ const setVersion = options.applicationVersion !== undefined
2116
+ ? `, application_version = $${params.push(options.applicationVersion)}`
2117
+ : '';
1675
2118
  const { rowCount } = await client.query(`UPDATE "${schema}".workflow_status
1676
2119
  SET status = $1, owner_xid = NULL, queue_name = $2, queue_partition_key = $3, recovery_attempts = 0,
1677
2120
  workflow_deadline_epoch_ms = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
1678
2121
  completed_at = NULL, output = NULL, error = NULL,
1679
2122
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint${setVersion}
1680
- WHERE workflow_uuid = $4 AND status = $5`, params);
2123
+ WHERE ${this.wf(undefined, workflowID, params)} AND status = $4`, params);
1681
2124
  if (rowCount !== 1) {
1682
2125
  throw new error_1.DBOSError(`Workflow ${workflowID} changed status while being rewound; retry the rewind`);
1683
2126
  }
@@ -1707,13 +2150,14 @@ class SystemDatabase {
1707
2150
  try {
1708
2151
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1709
2152
  // Fetch the status of all original workflows inside the transaction.
2153
+ const statusParams = [];
1710
2154
  const { rows: statusRows } = await client.query(`SELECT ws.workflow_uuid, ws.name, ws.class_name, ws.config_name, ws.application_id,
1711
2155
  ws.authenticated_user, ws.authenticated_roles, ws.assumed_role,
1712
2156
  COALESCE(wi.inputs, ws.inputs) AS inputs, ws.serialization,
1713
2157
  ws.application_version, ws.attributes, ws.application_name
1714
2158
  FROM "${this.schemaName}".workflow_status ws
1715
- LEFT JOIN "${this.schemaName}".workflow_input wi ON wi.workflow_uuid = ws.workflow_uuid
1716
- WHERE ws.workflow_uuid = ANY($1)`, [originalWorkflowIDs]);
2159
+ LEFT JOIN "${this.schemaName}".workflow_input wi ON ${this.wfJoin('ws', 'wi')}
2160
+ WHERE ${this.wfIn('ws', originalWorkflowIDs, statusParams)}`, statusParams);
1717
2161
  const statusByID = new Map(statusRows.map((r) => [r.workflow_uuid, r]));
1718
2162
  for (const wid of originalWorkflowIDs) {
1719
2163
  if (!statusByID.has(wid)) {
@@ -1749,35 +2193,42 @@ class SystemDatabase {
1749
2193
  forkedWorkflowIDs[i],
1750
2194
  statusByID.get(origID).application_name ?? this.appName ?? null,
1751
2195
  ]));
1752
- const valuesPlaceholders = [];
1753
- const params = [];
1754
- let paramIdx = 1;
1755
- for (let i = 0; i < originalWorkflowIDs.length; i++) {
1756
- const origID = originalWorkflowIDs[i];
2196
+ const forkKeys = forkedWorkflowIDs.map((forkID) => this.wfValues(forkID));
2197
+ await this.insertRows(client, 'workflow_status', [...Object.keys(forkKeys[0]), ...insertCols], originalWorkflowIDs.map((origID, i) => {
1757
2198
  const forkID = forkedWorkflowIDs[i];
1758
2199
  const ws = statusByID.get(origID);
1759
- const placeholders = insertCols.map(() => `$${paramIdx++}`).join(', ');
1760
- valuesPlaceholders.push(`(${placeholders})`);
1761
- params.push(forkID, workflow_1.StatusString.ENQUEUED, ws.name, ws.class_name ?? null, ws.config_name ?? null, queueName, ws.authenticated_user, ws.assumed_role, ws.authenticated_roles, options.applicationVersion ?? ws.application_version ?? null, ws.application_id,
1762
- // Legacy column: the payload lives in workflow_input.
1763
- null, options.queuePartitionKey ?? null, origID, ws.serialization, ws.attributes ? JSON.stringify(ws.attributes) : null, forkOwners.get(forkID) ?? null);
1764
- if (options.timeoutMS !== undefined) {
1765
- params.push(options.timeoutMS);
1766
- }
1767
- }
1768
- await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${insertCols.join(', ')})
1769
- VALUES ${valuesPlaceholders.join(', ')}`, params);
1770
- const inputPlaceholders = [];
1771
- const inputParams = [];
1772
- let inputIdx = 1;
1773
- for (let i = 0; i < originalWorkflowIDs.length; i++) {
1774
- inputPlaceholders.push(`($${inputIdx++}, $${inputIdx++})`);
1775
- inputParams.push(forkedWorkflowIDs[i], statusByID.get(originalWorkflowIDs[i]).inputs);
1776
- }
1777
- await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs)
1778
- VALUES ${inputPlaceholders.join(', ')}`, inputParams);
2200
+ return [
2201
+ ...Object.values(forkKeys[i]),
2202
+ forkID,
2203
+ workflow_1.StatusString.ENQUEUED,
2204
+ ws.name,
2205
+ ws.class_name ?? null,
2206
+ ws.config_name ?? null,
2207
+ queueName,
2208
+ ws.authenticated_user,
2209
+ ws.assumed_role,
2210
+ ws.authenticated_roles,
2211
+ options.applicationVersion ?? ws.application_version ?? null,
2212
+ ws.application_id,
2213
+ // Legacy column: the payload lives in workflow_input.
2214
+ null,
2215
+ options.queuePartitionKey ?? null,
2216
+ origID,
2217
+ ws.serialization,
2218
+ ws.attributes ? JSON.stringify(ws.attributes) : null,
2219
+ forkOwners.get(forkID) ?? null,
2220
+ ...(options.timeoutMS !== undefined ? [options.timeoutMS] : []),
2221
+ ];
2222
+ }));
2223
+ await this.insertRows(client, 'workflow_input', [...Object.keys(forkKeys[0]), 'workflow_uuid', 'inputs'], originalWorkflowIDs.map((origID, i) => [
2224
+ ...Object.values(forkKeys[i]),
2225
+ forkedWorkflowIDs[i],
2226
+ statusByID.get(origID).inputs,
2227
+ ]));
1779
2228
  // Mark all original workflows as having been forked from.
1780
- await client.query(`UPDATE "${this.schemaName}".workflow_status SET was_forked_from = TRUE WHERE workflow_uuid = ANY($1)`, [originalWorkflowIDs]);
2229
+ const forkedFromParams = [];
2230
+ await client.query(`UPDATE "${this.schemaName}".workflow_status SET was_forked_from = TRUE
2231
+ WHERE ${this.wfIn(undefined, originalWorkflowIDs, forkedFromParams)}`, forkedFromParams);
1781
2232
  // For workflows with start_step > 0, copy checkpoints/events/streams.
1782
2233
  // Build a mapping CTE so each copy is a single SQL statement.
1783
2234
  const forkMappings = originalWorkflowIDs
@@ -1794,55 +2245,85 @@ class SystemDatabase {
1794
2245
  mIdx += 4;
1795
2246
  }
1796
2247
  const mappingCTE = `WITH mapping(orig_id, fork_id, start_step, owner) AS (VALUES ${mappingValues.join(', ')})`;
2248
+ // The forks live under their originals' shard key, read from the scope; a copy's params extend the mapping's.
2249
+ const copyParams = () => [...mappingParams];
2250
+ const keyed = (params) => {
2251
+ const columns = this.scopedColumns(params);
2252
+ return { names: columns.map(([c]) => `${c}, `).join(''), exprs: columns.map(([, e]) => `${e}, `).join('') };
2253
+ };
2254
+ // A replacement child is a fork, so it shares the replaced child's key.
2255
+ const childKey = this.keyColumn('oo', 'child_shard_key');
2256
+ if (options.replacementChildren && childKey !== undefined) {
2257
+ // A replacement names a child by ID, which can't pick one of several keys.
2258
+ const ambiguityParams = copyParams();
2259
+ const replaced = `$${ambiguityParams.push(Object.keys(options.replacementChildren))}::text[]`;
2260
+ const { rows: ambiguous } = await client.query(`${mappingCTE}
2261
+ SELECT oo.child_workflow_id
2262
+ FROM mapping m
2263
+ JOIN "${this.schemaName}".operation_outputs oo
2264
+ ON ${this.scoped('oo', ambiguityParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step
2265
+ WHERE oo.child_workflow_id = ANY(${replaced})
2266
+ GROUP BY oo.child_workflow_id
2267
+ HAVING COUNT(DISTINCT ${childKey}) > 1
2268
+ LIMIT 1`, ambiguityParams);
2269
+ if (ambiguous.length > 0) {
2270
+ throw new error_1.DBOSError(`Can't replace child workflow ${ambiguous[0].child_workflow_id}: the forked steps refer to children with that ID under more than one shard key`);
2271
+ }
2272
+ }
1797
2273
  // Build the child_workflow_id expression, applying replacements if provided.
2274
+ const ooParams = copyParams();
1798
2275
  let childWfExpr = 'oo.child_workflow_id';
1799
- const ooParams = [...mappingParams];
1800
2276
  if (options.replacementChildren && Object.keys(options.replacementChildren).length > 0) {
1801
2277
  const whenClauses = [];
1802
2278
  for (const [oldId, newId] of Object.entries(options.replacementChildren)) {
1803
- whenClauses.push(`WHEN oo.child_workflow_id = $${mIdx} THEN $${mIdx + 1}::text`);
1804
- ooParams.push(oldId, newId);
1805
- mIdx += 2;
2279
+ whenClauses.push(`WHEN oo.child_workflow_id = $${ooParams.push(oldId)} THEN $${ooParams.push(newId)}::text`);
1806
2280
  }
1807
2281
  childWfExpr = `CASE ${whenClauses.join(' ')} ELSE oo.child_workflow_id END`;
1808
2282
  }
2283
+ const ooKeys = keyed(ooParams);
1809
2284
  // Copy operation outputs
1810
2285
  await client.query(`${mappingCTE}
1811
2286
  INSERT INTO "${this.schemaName}".operation_outputs
1812
- (workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp)
1813
- SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint
2287
+ (${ooKeys.names}workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? ', child_shard_key' : ''})
2288
+ SELECT ${ooKeys.exprs}m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint${childKey !== undefined ? `, ${childKey}` : ''}
1814
2289
  FROM mapping m
1815
2290
  JOIN "${this.schemaName}".operation_outputs oo
1816
- ON oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
2291
+ ON ${this.scoped('oo', ooParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
1817
2292
  // Copy streams
2293
+ const streamParams = copyParams();
2294
+ const streamKeys = keyed(streamParams);
1818
2295
  await client.query(`${mappingCTE}
1819
2296
  INSERT INTO "${this.schemaName}".streams
1820
- (workflow_uuid, key, value, serialization, "offset", function_id)
1821
- SELECT m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
2297
+ (${streamKeys.names}workflow_uuid, key, value, serialization, "offset", function_id)
2298
+ SELECT ${streamKeys.exprs}m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
1822
2299
  FROM mapping m
1823
2300
  JOIN "${this.schemaName}".streams s
1824
- ON s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`, mappingParams);
2301
+ ON ${this.scoped('s', streamParams)} AND s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`, streamParams);
1825
2302
  // Copy events history
2303
+ const historyParams = copyParams();
2304
+ const historyKeys = keyed(historyParams);
1826
2305
  await client.query(`${mappingCTE}
1827
2306
  INSERT INTO "${this.schemaName}".workflow_events_history
1828
- (workflow_uuid, function_id, key, value, serialization)
1829
- SELECT m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
2307
+ (${historyKeys.names}workflow_uuid, function_id, key, value, serialization)
2308
+ SELECT ${historyKeys.exprs}m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
1830
2309
  FROM mapping m
1831
2310
  JOIN "${this.schemaName}".workflow_events_history weh
1832
- ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`, mappingParams);
2311
+ ON ${this.scoped('weh', historyParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`, historyParams);
1833
2312
  // Copy only the latest version of each event using a window function
2313
+ const eventParams = copyParams();
2314
+ const eventKeys = keyed(eventParams);
1834
2315
  await client.query(`${mappingCTE}
1835
2316
  INSERT INTO "${this.schemaName}".workflow_events
1836
- (workflow_uuid, key, value, serialization)
1837
- SELECT ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
2317
+ (${eventKeys.names}workflow_uuid, key, value, serialization)
2318
+ SELECT ${eventKeys.exprs}ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
1838
2319
  FROM (
1839
2320
  SELECT m.fork_id AS workflow_uuid, weh.key, weh.value, weh.serialization,
1840
- ROW_NUMBER() OVER (PARTITION BY weh.workflow_uuid, weh.key ORDER BY weh.function_id DESC) AS rn
2321
+ ROW_NUMBER() OVER (PARTITION BY m.fork_id, weh.key ORDER BY weh.function_id DESC) AS rn
1841
2322
  FROM mapping m
1842
2323
  JOIN "${this.schemaName}".workflow_events_history weh
1843
- ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
2324
+ ON ${this.scoped('weh', eventParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
1844
2325
  ) ranked
1845
- WHERE ranked.rn = 1`, mappingParams);
2326
+ WHERE ranked.rn = 1`, eventParams);
1846
2327
  }
1847
2328
  await client.query('COMMIT');
1848
2329
  return forkedWorkflowIDs;
@@ -1965,27 +2446,36 @@ class SystemDatabase {
1965
2446
  }
1966
2447
  /** Cancellation check for polling waits: goes through the limiter so readers cannot starve the pool. */
1967
2448
  async checkIfCanceledLimited(workflowID) {
1968
- await this.#pollWithLimiter(() => this.#checkIfCanceled(this.pool, workflowID));
2449
+ // The caller's own workflow, even where a same-ID workflow is bound as a target.
2450
+ const params = [];
2451
+ const target = this.ownKey(workflowID, undefined, () => this.wf(undefined, workflowID, params));
2452
+ const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${target}`, params));
2453
+ if (rows[0]?.status === workflow_1.StatusString.CANCELLED) {
2454
+ throw new error_1.DBOSWorkflowCancelledError(workflowID);
2455
+ }
1969
2456
  }
1970
2457
  watchForCancellation(workflowID) {
1971
2458
  const controller = new AbortController();
1972
2459
  let state = 'idle';
2460
+ // The running workflow's own key, resolved now for the poller that reads it later outside this context.
2461
+ const shardKey = this.ownKey(workflowID, undefined, () => this.keyHint(workflowID));
2462
+ const waiterKey = this.ownKey(workflowID, undefined, () => this.waiter(workflowID, ''));
1973
2463
  const start = () => {
1974
2464
  state = 'watching';
1975
2465
  if (this.#destroyed)
1976
2466
  return;
1977
- let watchers = this.#cancelWatchers.get(workflowID);
2467
+ let watchers = this.#cancelWatchers.get(waiterKey);
1978
2468
  if (watchers === undefined) {
1979
- watchers = new Set();
1980
- this.#cancelWatchers.set(workflowID, watchers);
2469
+ watchers = { workflowID, shardKey, controllers: new Set() };
2470
+ this.#cancelWatchers.set(waiterKey, watchers);
1981
2471
  }
1982
- watchers.add(controller);
1983
- this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => this.#runCancelPoller());
2472
+ watchers.controllers.add(controller);
2473
+ this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => (0, shard_keys_1.withClearedShardKeyState)(() => this.#runCancelPoller()));
1984
2474
  };
1985
2475
  const stop = () => {
1986
- const watchers = this.#cancelWatchers.get(workflowID);
1987
- if (watchers?.delete(controller) && watchers.size === 0) {
1988
- this.#cancelWatchers.delete(workflowID);
2476
+ const watchers = this.#cancelWatchers.get(waiterKey);
2477
+ if (watchers?.controllers.delete(controller) && watchers.controllers.size === 0) {
2478
+ this.#cancelWatchers.delete(waiterKey);
1989
2479
  if (this.#cancelWatchers.size === 0)
1990
2480
  this.#cancelPollerWake?.();
1991
2481
  }
@@ -2024,15 +2514,36 @@ class SystemDatabase {
2024
2514
  this.#cancelPollerLoop = undefined;
2025
2515
  }
2026
2516
  async #abortCancelledWatchers() {
2027
- const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1) AND status = $2`, [Array.from(this.#cancelWatchers.keys()), workflow_1.StatusString.CANCELLED]));
2028
- for (const { workflow_uuid: workflowID } of rows) {
2029
- const watchers = this.#cancelWatchers.get(workflowID);
2030
- if (watchers === undefined)
2517
+ // One read per shard key, each bound to its own workflows.
2518
+ const byKey = new Map();
2519
+ for (const { workflowID, shardKey } of this.#cancelWatchers.values()) {
2520
+ const ids = byKey.get(shardKey) ?? [];
2521
+ ids.push(workflowID);
2522
+ byKey.set(shardKey, ids);
2523
+ }
2524
+ for (const [shardKey, workflowIDs] of byKey) {
2525
+ // Each key's read on its own, so one key's failure doesn't hide another key's cancellations.
2526
+ let rows;
2527
+ try {
2528
+ ({ rows } = await (0, shard_keys_1.bindShardKeys)(workflowIDs.map((id) => [id, shardKey]), () => {
2529
+ const params = [workflow_1.StatusString.CANCELLED];
2530
+ const ids = this.wfIn(undefined, workflowIDs, params);
2531
+ return this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${ids} AND status = $1`, params));
2532
+ }));
2533
+ }
2534
+ catch (e) {
2535
+ this.logger.warn(`Error polling status for cancellable steps: ${String(e)}`);
2031
2536
  continue;
2032
- this.#cancelWatchers.delete(workflowID);
2033
- const reason = new error_1.DBOSWorkflowCancelledError(workflowID);
2034
- for (const controller of watchers)
2035
- controller.abort(reason);
2537
+ }
2538
+ const cancelled = new Set(rows.map((row) => row.workflow_uuid));
2539
+ for (const [waiterKey, watchers] of this.#cancelWatchers) {
2540
+ if (watchers.shardKey !== shardKey || !cancelled.has(watchers.workflowID))
2541
+ continue;
2542
+ this.#cancelWatchers.delete(waiterKey);
2543
+ const reason = new error_1.DBOSWorkflowCancelledError(watchers.workflowID);
2544
+ for (const controller of watchers.controllers)
2545
+ controller.abort(reason);
2546
+ }
2036
2547
  }
2037
2548
  }
2038
2549
  // A missing row normally means the workflow has not been inserted yet, so
@@ -2044,21 +2555,23 @@ class SystemDatabase {
2044
2555
  const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
2045
2556
  let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
2046
2557
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
2047
- // Record the durable timeout deadline once before polling. #durableSleep persists it on the
2558
+ // Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
2048
2559
  // first call and reads back the same value on recovery, so it never changes across iterations.
2049
2560
  if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
2050
- finishTime = await this.#durableSleep(callerID, timerFuncID, timeoutms);
2561
+ finishTime = await this.durableSleepInternal(callerID, timerFuncID, timeoutms);
2051
2562
  }
2052
2563
  while (true) {
2053
2564
  if (callerID)
2054
2565
  await this.checkIfCanceledLimited(callerID);
2055
2566
  let rows;
2056
2567
  try {
2568
+ const params = [];
2569
+ const target = this.wf('ws', workflowID, params);
2057
2570
  ({ rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT ws.status, COALESCE(wo.output, ws.output) AS output,
2058
2571
  COALESCE(wo.error, ws.error) AS error, ws.serialization
2059
2572
  FROM "${this.schemaName}".workflow_status ws
2060
- LEFT JOIN "${this.schemaName}".workflow_output wo ON wo.workflow_uuid = ws.workflow_uuid
2061
- WHERE ws.workflow_uuid=$1`, [workflowID])));
2573
+ LEFT JOIN "${this.schemaName}".workflow_output wo ON ${this.wfJoin('ws', 'wo')}
2574
+ WHERE ${target}`, params)));
2062
2575
  }
2063
2576
  catch (e) {
2064
2577
  const err = e;
@@ -2095,21 +2608,27 @@ class SystemDatabase {
2095
2608
  }
2096
2609
  }
2097
2610
  async awaitFirstWorkflowId(workflowIds, callerID, pollingIntervalMs) {
2098
- const placeholders = workflowIds.map((_, i) => `$${i + 1}`).join(', ');
2099
2611
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
2100
2612
  while (true) {
2101
2613
  if (callerID)
2102
2614
  await this.checkIfCanceledLimited(callerID);
2103
- const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2104
- WHERE workflow_uuid IN (${placeholders})
2105
- AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
2106
- LIMIT 1`, workflowIds));
2107
- if (rows.length > 0) {
2108
- return rows[0].workflow_uuid;
2615
+ const first = await this.checkFirstWorkflowId(workflowIds);
2616
+ if (first !== undefined) {
2617
+ return first;
2109
2618
  }
2110
2619
  await (0, utils_1.sleepms)(pollIntervalMs);
2111
2620
  }
2112
2621
  }
2622
+ /** The first of these workflows found complete, if any. */
2623
+ async checkFirstWorkflowId(workflowIds) {
2624
+ const params = [];
2625
+ const ids = this.wfIn(undefined, workflowIds, params);
2626
+ const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2627
+ WHERE ${ids}
2628
+ AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
2629
+ LIMIT 1`, params));
2630
+ return rows[0]?.workflow_uuid;
2631
+ }
2113
2632
  async awaitWorkflowIds(workflowIds, callerID, pollingIntervalMs) {
2114
2633
  const remainingWorkflowIds = new Set(workflowIds);
2115
2634
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
@@ -2117,11 +2636,8 @@ class SystemDatabase {
2117
2636
  const currentWorkflowIds = [...remainingWorkflowIds];
2118
2637
  if (callerID)
2119
2638
  await this.checkIfCanceledLimited(callerID);
2120
- const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2121
- WHERE workflow_uuid = ANY($1::text[])
2122
- AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, [currentWorkflowIds]));
2123
- for (const row of rows) {
2124
- remainingWorkflowIds.delete(row.workflow_uuid);
2639
+ for (const workflowID of await this.checkCompletedWorkflowIds(currentWorkflowIds)) {
2640
+ remainingWorkflowIds.delete(workflowID);
2125
2641
  }
2126
2642
  if (remainingWorkflowIds.size === 0) {
2127
2643
  return;
@@ -2129,9 +2645,18 @@ class SystemDatabase {
2129
2645
  await (0, utils_1.sleepms)(pollIntervalMs);
2130
2646
  }
2131
2647
  }
2648
+ /** Those of these workflows found complete. */
2649
+ async checkCompletedWorkflowIds(workflowIds) {
2650
+ const params = [];
2651
+ const ids = this.wfIn(undefined, workflowIds, params);
2652
+ const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2653
+ WHERE ${ids}
2654
+ AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, params));
2655
+ return rows.map((row) => row.workflow_uuid);
2656
+ }
2132
2657
  // ==================== Sleep ====================
2133
2658
  async durableSleepms(workflowID, functionID, durationMS) {
2134
- const endTime = await this.#durableSleep(workflowID, functionID, durationMS, true);
2659
+ const endTime = await this.durableSleepInternal(workflowID, functionID, durationMS, true);
2135
2660
  while (Date.now() < endTime) {
2136
2661
  await (0, utils_1.sleepms)(Math.min(endTime - Date.now(), utils_1.sleepConfig.maxTimeoutMS));
2137
2662
  }
@@ -2139,31 +2664,28 @@ class SystemDatabase {
2139
2664
  }
2140
2665
  // ==================== Messaging ====================
2141
2666
  nullTopic = '__null__topic__';
2142
- async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey) {
2143
- topic = topic ?? this.nullTopic;
2144
- const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
2667
+ async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey,
2668
+ // The destination's shard key; required on a sharded system database.
2669
+ shardKey) {
2145
2670
  const startTime = Date.now();
2146
2671
  let missing;
2147
2672
  const client = await this.connect();
2148
2673
  try {
2149
2674
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
2150
2675
  await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
2151
- await client.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
2152
- VALUES ($1, $2, $3, $4, $5)
2153
- ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
2676
+ await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2154
2677
  return undefined;
2155
2678
  });
2156
2679
  await client.query('COMMIT');
2157
2680
  }
2158
2681
  catch (error) {
2159
2682
  await client.query('ROLLBACK');
2160
- const err = error;
2161
- if (err.code === '23503') {
2683
+ if (error instanceof error_1.DBOSNonExistentWorkflowError) {
2162
2684
  // Foreign key constraint violation (only expected for the INSERT query)
2163
- missing = new error_1.DBOSNonExistentWorkflowError(`Sent to non-existent destination workflow UUID: ${destinationID}`);
2685
+ missing = error;
2164
2686
  }
2165
2687
  else {
2166
- throw err;
2688
+ throw error;
2167
2689
  }
2168
2690
  }
2169
2691
  finally {
@@ -2176,22 +2698,22 @@ class SystemDatabase {
2176
2698
  }
2177
2699
  }
2178
2700
  /** Runs on `client` if given, joining its transaction; otherwise on the pool with retries. */
2179
- async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client) {
2701
+ async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client, shardKey) {
2180
2702
  if (client !== undefined) {
2181
- return await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
2703
+ return await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2182
2704
  }
2183
- return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey);
2705
+ return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey);
2184
2706
  }
2185
2707
  /** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
2186
- async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
2708
+ async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
2187
2709
  const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
2188
2710
  const client = await this.connect();
2189
2711
  try {
2190
- await this.#inTransaction(client, async () => {
2191
- await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
2712
+ await this.inTransaction(client, async () => {
2713
+ await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2192
2714
  // After the insert, in the order the recorded send locks, so they cannot deadlock.
2193
2715
  if (ownerXid !== undefined) {
2194
- await this.#checkOwner(client, workflowID, ownerXid);
2716
+ await this.checkOwner(client, workflowID, ownerXid);
2195
2717
  }
2196
2718
  });
2197
2719
  }
@@ -2199,17 +2721,23 @@ class SystemDatabase {
2199
2721
  client.release();
2200
2722
  }
2201
2723
  }
2202
- async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey) {
2203
- return await this.#sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey);
2724
+ async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey) {
2725
+ return await this.sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2204
2726
  }
2205
- async #sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey) {
2727
+ async sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
2206
2728
  topic = topic ?? this.nullTopic;
2207
- // Same per-destination scoping as send() above.
2729
+ // An idempotency key is scoped per destination: suffix it with the recipient's workflow ID.
2208
2730
  const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
2731
+ const values = {
2732
+ ...this.wfValues(destinationID, shardKey),
2733
+ destination_uuid: destinationID,
2734
+ topic,
2735
+ message,
2736
+ serialization,
2737
+ message_uuid: messageUUID,
2738
+ };
2209
2739
  try {
2210
- await db.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
2211
- VALUES ($1, $2, $3, $4, $5)
2212
- ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
2740
+ await this.insertRows(db, 'notifications', Object.keys(values), [Object.values(values)], `ON CONFLICT (${this.keyConflict('message_uuid')}) DO NOTHING`);
2213
2741
  }
2214
2742
  catch (error) {
2215
2743
  const err = error;
@@ -2233,10 +2761,10 @@ class SystemDatabase {
2233
2761
  const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
2234
2762
  let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
2235
2763
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
2236
- // Record the durable timeout deadline once before polling. #durableSleep persists it on the
2764
+ // Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
2237
2765
  // first call and reads back the same value on recovery, so it never changes across iterations.
2238
2766
  if (timeoutms) {
2239
- finishTime = await this.#durableSleep(workflowID, timeoutFunctionID, timeoutms);
2767
+ finishTime = await this.durableSleepInternal(workflowID, timeoutFunctionID, timeoutms);
2240
2768
  }
2241
2769
  while (true) {
2242
2770
  // register the key with the global notifications listener.
@@ -2244,12 +2772,14 @@ class SystemDatabase {
2244
2772
  const messagePromise = new Promise((resolve) => {
2245
2773
  resolveNotification = resolve;
2246
2774
  });
2247
- const payload = `${workflowID}::${topic}`;
2775
+ const payload = this.waiter(workflowID, topic);
2248
2776
  const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
2249
2777
  try {
2250
2778
  await this.checkIfCanceledLimited(workflowID);
2251
2779
  // Check if the key is already in the DB, then wait for the notification if it isn't.
2252
- const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE destination_uuid=$1 AND topic=$2 AND consumed = false;`, [workflowID, topic]))).rows;
2780
+ const params = [topic];
2781
+ const target = this.wf(undefined, workflowID, params, 'destination_uuid');
2782
+ const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE ${target} AND topic=$1 AND consumed = false;`, params))).rows;
2253
2783
  if (initRecvRows.length !== 0)
2254
2784
  break;
2255
2785
  const ct = Date.now();
@@ -2276,21 +2806,22 @@ class SystemDatabase {
2276
2806
  const client = await this.connect();
2277
2807
  try {
2278
2808
  await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
2809
+ const consumeParams = [topic, functionID];
2279
2810
  const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
2280
- SET consumed = true, consumed_by_function_id = $3
2281
- WHERE destination_uuid = $1
2282
- AND topic = $2
2811
+ SET consumed = true, consumed_by_function_id = $2
2812
+ WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
2813
+ AND topic = $1
2283
2814
  AND consumed = false
2284
2815
  AND message_uuid = (
2285
2816
  SELECT message_uuid
2286
2817
  FROM "${this.schemaName}".notifications
2287
- WHERE destination_uuid = $1
2288
- AND topic = $2
2818
+ WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
2819
+ AND topic = $1
2289
2820
  AND consumed = false
2290
2821
  ORDER BY created_at_epoch_ms ASC
2291
2822
  LIMIT 1
2292
2823
  )
2293
- RETURNING notifications.message, notifications.serialization;`, [workflowID, topic, functionID])).rows;
2824
+ RETURNING notifications.message, notifications.serialization;`, consumeParams)).rows;
2294
2825
  if (finalRecvRows.length > 0) {
2295
2826
  message = finalRecvRows[0].message;
2296
2827
  serialization = finalRecvRows[0].serialization;
@@ -2319,16 +2850,19 @@ class SystemDatabase {
2319
2850
  // Only a real write (not a replay) should wake readers.
2320
2851
  let didWrite = false;
2321
2852
  await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SETEVENT, workflowID, functionID, async () => {
2322
- await client.query(`INSERT INTO "${this.schemaName}".workflow_events (workflow_uuid, key, value, serialization)
2323
- VALUES ($1, $2, $3, $4)
2324
- ON CONFLICT (workflow_uuid, key)
2853
+ const keys = this.wfValues(workflowID);
2854
+ const keyColumns = Object.keys(keys).map((c) => `${c}, `);
2855
+ const keyPlaceholders = (from) => Object.keys(keys).map((_, i) => `$${from + i}, `);
2856
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_events (${keyColumns.join('')}workflow_uuid, key, value, serialization)
2857
+ VALUES (${keyPlaceholders(5).join('')}$1, $2, $3, $4)
2858
+ ON CONFLICT (${this.keyConflict('workflow_uuid', 'key')})
2325
2859
  DO UPDATE SET value = $3, serialization = $4
2326
- RETURNING workflow_uuid;`, [workflowID, key, message, serialization]);
2860
+ RETURNING workflow_uuid;`, [workflowID, key, message, serialization, ...Object.values(keys)]);
2327
2861
  // Also write to the immutable history table for fork support
2328
- await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (workflow_uuid, function_id, key, value, serialization)
2329
- VALUES ($1, $2, $3, $4, $5)
2330
- ON CONFLICT (workflow_uuid, function_id, key)
2331
- DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization]);
2862
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (${keyColumns.join('')}workflow_uuid, function_id, key, value, serialization)
2863
+ VALUES (${keyPlaceholders(6).join('')}$1, $2, $3, $4, $5)
2864
+ ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id', 'key')})
2865
+ DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization, ...Object.values(keys)]);
2332
2866
  didWrite = true;
2333
2867
  return undefined;
2334
2868
  });
@@ -2362,15 +2896,15 @@ class SystemDatabase {
2362
2896
  // Get the return the value. if it's in the DB, otherwise return null.
2363
2897
  let value = null;
2364
2898
  let valueSer = null;
2365
- const payloadKey = `${workflowID}::${key}`;
2899
+ const payloadKey = this.waiter(workflowID, key);
2366
2900
  const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
2367
2901
  let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
2368
2902
  const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
2369
2903
  // If we have a callerWorkflow, we want a durable sleep, otherwise, not. Record the durable
2370
- // timeout deadline once before polling. #durableSleep persists it on the first call and reads
2904
+ // timeout deadline once before polling. durableSleepInternal persists it on the first call and reads
2371
2905
  // back the same value on recovery, so it never changes across iterations.
2372
2906
  if (callerWorkflow && timeoutms) {
2373
- finishTime = await this.#durableSleep(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms);
2907
+ finishTime = await this.durableSleepInternal(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms);
2374
2908
  }
2375
2909
  // Register the key with the global notifications listener first... we do not want to look in the DB first
2376
2910
  // or that would cause a timing hole.
@@ -2384,9 +2918,11 @@ class SystemDatabase {
2384
2918
  if (callerWorkflow?.workflowID)
2385
2919
  await this.checkIfCanceledLimited(callerWorkflow?.workflowID);
2386
2920
  // Check if the key is already in the DB, then wait for the notification if it isn't.
2921
+ const params = [key];
2922
+ const target = this.wf(undefined, workflowID, params);
2387
2923
  const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT key, value, serialization
2388
2924
  FROM "${this.schemaName}".workflow_events
2389
- WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key]))).rows;
2925
+ WHERE ${target} AND key=$1;`, params))).rows;
2390
2926
  if (initRecvRows.length > 0) {
2391
2927
  value = initRecvRows[0].value;
2392
2928
  valueSer = initRecvRows[0].serialization;
@@ -2425,15 +2961,12 @@ class SystemDatabase {
2425
2961
  try {
2426
2962
  while (true) {
2427
2963
  try {
2428
- await this.#inTransaction(client, async () => {
2964
+ await this.inTransaction(client, async () => {
2429
2965
  // Derives the first unused offset inside the insert; two writers can still pick the same one.
2430
- await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
2431
- SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
2432
- FROM "${this.schemaName}".streams s
2433
- WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
2966
+ await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
2434
2967
  // After the insert, in the order the workflow-level writes lock, so they cannot deadlock.
2435
2968
  if (ownerXid !== undefined) {
2436
- await this.#checkOwner(client, workflowID, ownerXid);
2969
+ await this.checkOwner(client, workflowID, ownerXid);
2437
2970
  }
2438
2971
  });
2439
2972
  }
@@ -2467,10 +3000,7 @@ class SystemDatabase {
2467
3000
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
2468
3001
  await this.#runAndRecordResult(client, functionName, workflowID, functionID, async () => {
2469
3002
  // Derives the first unused offset inside the insert; two writers can still pick the same one.
2470
- await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
2471
- SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
2472
- FROM "${this.schemaName}".streams s
2473
- WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
3003
+ await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
2474
3004
  didWrite = true;
2475
3005
  return undefined;
2476
3006
  });
@@ -2502,6 +3032,15 @@ class SystemDatabase {
2502
3032
  client.release();
2503
3033
  }
2504
3034
  }
3035
+ /** Append a value to a stream, at the first unused offset, derived inside the insert. */
3036
+ async #insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization) {
3037
+ const params = [key, serializedValue, functionID, serialization, workflowID];
3038
+ const keyColumns = this.wfColumns(workflowID, params);
3039
+ await client.query(`INSERT INTO "${this.schemaName}".streams (${keyColumns.map(([c]) => `${c}, `).join('')}workflow_uuid, key, value, "offset", function_id, serialization)
3040
+ SELECT ${keyColumns.map(([, e]) => `${e}, `).join('')}$5::text, $1::text, $2::text, COALESCE(MAX(s."offset"), -1) + 1, $3::int, $4::text
3041
+ FROM "${this.schemaName}".streams s
3042
+ WHERE ${this.wf('s', workflowID, params)} AND s.key = $1`, params);
3043
+ }
2505
3044
  async closeStreamFromWorkflow(workflowID, functionID, key) {
2506
3045
  await this.writeStreamFromWorkflow(workflowID, functionID, key, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name(), exports.DBOS_FUNCNAME_CLOSESTREAM);
2507
3046
  }
@@ -2511,13 +3050,15 @@ class SystemDatabase {
2511
3050
  // Read the value at `offset` and the workflow's status in one query: status null = no such workflow, value undefined = nothing at that offset.
2512
3051
  async readStreamValue(workflowID, key, offset) {
2513
3052
  // LEFT JOIN so a workflow with nothing at offset still reports its status (single PK lookup); under the poll limiter, inside @dbRetry so the permit frees across backoff.
3053
+ const params = [key, offset];
3054
+ const target = this.wf('ws', workflowID, params);
2514
3055
  const result = await this.#pollWithLimiter(() => this.pool.query(
2515
3056
  // "offset" is a reserved word, so alias it (stream_offset) to read it back plainly.
2516
3057
  `SELECT ws.status AS status, s.value AS value, s.serialization AS serialization, s."offset" AS stream_offset
2517
3058
  FROM "${this.schemaName}".workflow_status ws
2518
3059
  LEFT OUTER JOIN "${this.schemaName}".streams s
2519
- ON s.workflow_uuid = ws.workflow_uuid AND s.key = $2 AND s."offset" = $3
2520
- WHERE ws.workflow_uuid = $1`, [workflowID, key, offset]));
3060
+ ON ${this.wfJoin('s', 'ws')} AND s.key = $1 AND s."offset" = $2
3061
+ WHERE ${target}`, params));
2521
3062
  if (result.rows.length === 0) {
2522
3063
  return { status: null, value: undefined };
2523
3064
  }
@@ -2654,7 +3195,7 @@ class SystemDatabase {
2654
3195
  // Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v3, so cost scales with the number of partitions rather than the backlog depth.
2655
3196
  const params = [queueName, workflow_1.StatusString.ENQUEUED];
2656
3197
  // Only partitions this application can actually dequeue from.
2657
- const scope = this.appNameFilter('application_name', this.appName, params);
3198
+ const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
2658
3199
  const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
2659
3200
  (SELECT MIN(queue_partition_key) AS pk
2660
3201
  FROM "${this.schemaName}".workflow_status
@@ -2671,7 +3212,14 @@ class SystemDatabase {
2671
3212
  }
2672
3213
  async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0,
2673
3214
  // Written to every claimed row: the dispatches of this claim own their executions.
2674
- ownerXid = null) {
3215
+ ownerXid = null, isLatestVersion) {
3216
+ const { claimedIDs } = await this.claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount, partitionLocalRunningCount, ownerXid, isLatestVersion);
3217
+ return claimedIDs;
3218
+ }
3219
+ /** Claim this queue's next workflows, plus the limit that left no room to claim any (undefined if every limit had room). */
3220
+ async claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0, ownerXid = null, isLatestVersion) {
3221
+ // Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
3222
+ const startTimeMs = Date.now();
2675
3223
  const claimedIDs = [];
2676
3224
  const partitionParams = queuePartitionKey !== undefined ? [queuePartitionKey] : [];
2677
3225
  // Shares a concurrency or rate limit budget with other executors.
@@ -2679,13 +3227,49 @@ class SystemDatabase {
2679
3227
  queue.partitionConcurrency !== undefined ||
2680
3228
  queue.rateLimit !== undefined ||
2681
3229
  queue.partitionRateLimit !== undefined;
2682
- // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
3230
+ // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it: lock the whole queue.
2683
3231
  const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
3232
+ // Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
3233
+ let maxTasks = Infinity;
3234
+ // The first limit found with no room, which the caller reads as why nothing was claimed.
3235
+ let stopReason = undefined;
3236
+ const limit = (room, reason) => {
3237
+ maxTasks = Math.min(maxTasks, room);
3238
+ if (maxTasks <= 0)
3239
+ stopReason ??= reason;
3240
+ };
3241
+ // This worker's own limits need no database, so a full worker returns before taking the claim lock a peer could use.
3242
+ if (queue.workerConcurrency !== undefined) {
3243
+ // Use the in-memory registry for this worker's running count — avoids a DB round trip.
3244
+ limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
3245
+ }
3246
+ if (queue.partitionWorkerConcurrency !== undefined) {
3247
+ limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
3248
+ }
3249
+ if (maxTasks <= 0) {
3250
+ return { claimedIDs, stopReason };
3251
+ }
3252
+ // A sharded database keeps the version row under another key, which the claim's transaction cannot span.
3253
+ if (isLatestVersion === undefined && this.sharded) {
3254
+ isLatestVersion = await this.isLatestVersion(appVersion);
3255
+ }
3256
+ // Row locks only collide when claimers see the same queue head, so serialize claims on the budget's scope; CockroachDB has no advisory locks before 26.3.
3257
+ const useClaimLock = hasSharedBudget && (this.sharded || !(await this.#isCockroach()));
2684
3258
  const client = await this.connect();
2685
3259
  try {
2686
- // Default to READ COMMITTED except with a budget shared across executors
2687
- if (hasSharedBudget) {
2688
- await client.query(`BEGIN ISOLATION LEVEL ${hasWriteSkew ? 'SERIALIZABLE' : 'REPEATABLE READ'}`);
3260
+ if (useClaimLock) {
3261
+ await client.query('BEGIN');
3262
+ const lockParams = [];
3263
+ const { rows } = await client.query(this.claimLockQuery(queue.name, hasWriteSkew ? undefined : queuePartitionKey, lockParams), lockParams);
3264
+ if (rows[0]?.acquired !== true) {
3265
+ await client.query('COMMIT');
3266
+ // Another worker is claiming from this scope, so it still has work; the next poll retries.
3267
+ return rows.length > 0 ? { claimedIDs, stopReason: 'claim_lock' } : { claimedIDs };
3268
+ }
3269
+ }
3270
+ else if (hasSharedBudget) {
3271
+ // CockroachDB, without advisory locks before 26.3: SERIALIZABLE aborts claimers that spend the same budget.
3272
+ await client.query('BEGIN ISOLATION LEVEL SERIALIZABLE');
2689
3273
  }
2690
3274
  else {
2691
3275
  await client.query('BEGIN');
@@ -2694,7 +3278,7 @@ class SystemDatabase {
2694
3278
  const rateLimitRemaining = async (rateLimit, partitionScoped) => {
2695
3279
  const params = [queue.name, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED, rateLimit.periodSec * 1000];
2696
3280
  // Count only what this application would dequeue, matching the select below.
2697
- const scope = this.appNameFilter('application_name', this.appName, params);
3281
+ const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
2698
3282
  const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
2699
3283
  const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
2700
3284
  WHERE queue_name = $1
@@ -2713,35 +3297,26 @@ class SystemDatabase {
2713
3297
  */
2714
3298
  const pendingCount = async (partitionScoped) => {
2715
3299
  const params = [queue.name, workflow_1.StatusString.PENDING];
2716
- const scope = this.appNameFilter('application_name', this.appName, params);
3300
+ const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
2717
3301
  const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
2718
3302
  const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
2719
3303
  WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
2720
3304
  return Number(rows[0]?.count ?? 0);
2721
3305
  };
2722
- // Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
2723
- let maxTasks = Infinity;
2724
- if (queue.workerConcurrency !== undefined) {
2725
- // Use the in-memory registry for this worker's running count — avoids a DB round trip.
2726
- maxTasks = Math.min(maxTasks, Math.max(0, queue.workerConcurrency - localRunningCount));
2727
- }
2728
- if (queue.partitionWorkerConcurrency !== undefined) {
2729
- maxTasks = Math.min(maxTasks, Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount));
2730
- }
2731
- if (maxTasks <= 0) {
2732
- await client.query('COMMIT');
2733
- return claimedIDs;
2734
- }
2735
3306
  if (queue.rateLimit !== undefined) {
2736
3307
  // Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
2737
- maxTasks = Math.min(maxTasks, await rateLimitRemaining(queue.rateLimit, false));
3308
+ limit(await rateLimitRemaining(queue.rateLimit, false), 'limiter');
3309
+ if (maxTasks <= 0) {
3310
+ await client.query('COMMIT');
3311
+ return { claimedIDs, stopReason };
3312
+ }
2738
3313
  }
2739
3314
  if (queue.partitionRateLimit !== undefined) {
2740
- maxTasks = Math.min(maxTasks, await rateLimitRemaining(queue.partitionRateLimit, true));
3315
+ limit(await rateLimitRemaining(queue.partitionRateLimit, true), 'partition_limiter');
2741
3316
  }
2742
3317
  if (maxTasks <= 0) {
2743
3318
  await client.query('COMMIT');
2744
- return claimedIDs;
3319
+ return { claimedIDs, stopReason };
2745
3320
  }
2746
3321
  if (queue.concurrency !== undefined) {
2747
3322
  // Global concurrency still requires a DB query since other workers may be running workflows too.
@@ -2749,23 +3324,26 @@ class SystemDatabase {
2749
3324
  if (totalRunningTasks > queue.concurrency) {
2750
3325
  this.logger.warn(`Total running tasks (${totalRunningTasks}) exceeds the global concurrency limit (${queue.concurrency})`);
2751
3326
  }
2752
- maxTasks = Math.min(maxTasks, Math.max(0, queue.concurrency - totalRunningTasks));
3327
+ limit(Math.max(0, queue.concurrency - totalRunningTasks), 'concurrency');
3328
+ if (maxTasks <= 0) {
3329
+ await client.query('COMMIT');
3330
+ return { claimedIDs, stopReason };
3331
+ }
2753
3332
  }
2754
3333
  if (queue.partitionConcurrency !== undefined) {
2755
3334
  const partitionRunningTasks = await pendingCount(true);
2756
3335
  if (partitionRunningTasks > queue.partitionConcurrency) {
2757
3336
  this.logger.warn(`Total running tasks (${partitionRunningTasks}) on partition ${queuePartitionKey} of queue ${queue.name} exceeds the partition concurrency limit (${queue.partitionConcurrency})`);
2758
3337
  }
2759
- maxTasks = Math.min(maxTasks, Math.max(0, queue.partitionConcurrency - partitionRunningTasks));
3338
+ limit(Math.max(0, queue.partitionConcurrency - partitionRunningTasks), 'partition_concurrency');
2760
3339
  }
2761
3340
  // Return immediately if there are no available tasks due to flow control limits
2762
3341
  if (maxTasks <= 0) {
2763
3342
  await client.query('COMMIT');
2764
- return claimedIDs;
3343
+ return { claimedIDs, stopReason };
2765
3344
  }
2766
3345
  // Retrieve the first max_tasks workflows in the queue.
2767
- const latestVersion = await this.#latestApplicationVersionName(client);
2768
- const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
3346
+ isLatestVersion ??= await this.isLatestVersion(appVersion, client);
2769
3347
  const versionClause = isLatestVersion
2770
3348
  ? '(application_version = $3 OR application_version IS NULL)'
2771
3349
  : 'application_version = $3';
@@ -2775,13 +3353,17 @@ class SystemDatabase {
2775
3353
  const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
2776
3354
  const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
2777
3355
  const selectScope = this.appNameFilter('application_name', this.appName, selectParams);
3356
+ const keyScope = this.scoped(undefined, selectParams);
3357
+ const claimGate = this.claimGate(queue.name, queuePartitionKey, selectParams);
2778
3358
  const selectQuery = `
2779
3359
  SELECT workflow_uuid
2780
3360
  FROM "${this.schemaName}".workflow_status
2781
- WHERE status = $1
3361
+ WHERE ${keyScope}
3362
+ AND status = $1
2782
3363
  AND queue_name = $2
2783
3364
  AND ${versionClause}
2784
3365
  AND ${selectScope}
3366
+ AND ${claimGate}
2785
3367
  ${queuePartitionKey !== undefined ? 'AND queue_partition_key = $4' : ''}
2786
3368
  ORDER BY priority ASC, created_at ASC
2787
3369
  ${limitClause}
@@ -2800,35 +3382,36 @@ class SystemDatabase {
2800
3382
  executorID,
2801
3383
  appVersion,
2802
3384
  queue.rateLimit !== undefined || queue.partitionRateLimit !== undefined,
2803
- workflowIDs,
2804
3385
  workflow_1.StatusString.ENQUEUED,
2805
3386
  // Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
2806
3387
  this.appName ?? null,
2807
3388
  ownerXid,
3389
+ startTimeMs,
2808
3390
  ];
3391
+ const claimedIDsFilter = this.wfIn(undefined, workflowIDs, updateParams);
2809
3392
  // Re-check ownership alongside status, as the partitioned claim guard does.
2810
3393
  const claimScope = this.appNameFilter('application_name', this.appName, updateParams);
2811
3394
  // RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
2812
3395
  const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
2813
3396
  SET status = $1,
2814
- owner_xid = $8,
3397
+ owner_xid = $7,
2815
3398
  executor_id = $2,
2816
3399
  application_version = $3,
2817
3400
  started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2818
3401
  rate_limited = $4,
2819
- application_name = COALESCE(application_name, $7),
3402
+ application_name = COALESCE(application_name, $6),
2820
3403
  recovery_attempts = recovery_attempts + 1,
2821
3404
  updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2822
3405
  workflow_deadline_epoch_ms = CASE
2823
3406
  WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
2824
3407
  CASE
2825
- WHEN workflow_timeout_ms > 9223372036854775807 - (EXTRACT(epoch FROM now()) * 1000)::bigint
3408
+ WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
2826
3409
  THEN 9223372036854775807
2827
- ELSE (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
3410
+ ELSE $8::bigint + workflow_timeout_ms
2828
3411
  END
2829
3412
  ELSE workflow_deadline_epoch_ms
2830
3413
  END
2831
- WHERE workflow_uuid = ANY($5::text[]) AND status = $6 AND ${claimScope}
3414
+ WHERE ${claimedIDsFilter} AND status = $5 AND ${claimScope}
2832
3415
  RETURNING workflow_uuid`, updateParams);
2833
3416
  const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
2834
3417
  for (const id of workflowIDs) {
@@ -2846,14 +3429,18 @@ class SystemDatabase {
2846
3429
  client.release();
2847
3430
  }
2848
3431
  // Return the IDs of all functions we marked started
2849
- return claimedIDs;
3432
+ return { claimedIDs };
2850
3433
  }
2851
3434
  /** Max heads admitted per sweep: bounds dispatch, not the partition walk; an unconstrained sweep takes the lowest keys, so higher keys can wait under sustained load. */
2852
3435
  partitionedDequeueSweepCap = 8192;
2853
- /** Dequeue each partition's head-of-line workflow in one transaction, at most {@link partitionedDequeueSweepCap} per sweep; only valid for partition-concurrency-1 queues with no queue-wide limit. */
3436
+ /** Max partitions a sweep claims per transaction, which bounds the locks it holds at once and for how long. */
3437
+ partitionedDequeueBatchSize = 64;
3438
+ /** Dequeue each partition's head-of-line workflow, at most {@link partitionedDequeueSweepCap} per sweep in transactions of {@link partitionedDequeueBatchSize}; only valid for partition-concurrency-1 queues with no queue-wide limit. */
2854
3439
  async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity,
2855
3440
  // Written to every claimed row: the dispatches of this claim own their executions.
2856
- ownerXid = null) {
3441
+ ownerXid = null, isLatestVersion) {
3442
+ // Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
3443
+ const startTimeMs = Date.now();
2857
3444
  if (queue.partitionConcurrency !== 1 ||
2858
3445
  queue.concurrency !== undefined ||
2859
3446
  queue.rateLimit !== undefined ||
@@ -2861,14 +3448,35 @@ class SystemDatabase {
2861
3448
  throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
2862
3449
  }
2863
3450
  // partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
3451
+ // A sharded database keeps the version row under another key, which the claim's transaction cannot span.
3452
+ if (isLatestVersion === undefined && this.sharded) {
3453
+ isLatestVersion = await this.isLatestVersion(appVersion);
3454
+ }
3455
+ // Sweeps whose heads differ lock disjoint rows, so a claim also takes its partition's lock; CockroachDB, without advisory locks before 26.3, relies on its SERIALIZABLE default.
3456
+ const usePartitionLocks = this.sharded || !(await this.#isCockroach());
2864
3457
  const client = await this.connect();
2865
3458
  try {
2866
- await client.query('BEGIN');
2867
- const latestVersion = await this.#latestApplicationVersionName(client);
2868
- const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
3459
+ isLatestVersion ??= await this.isLatestVersion(appVersion, client);
2869
3460
  const versionClause = (n) => isLatestVersion
2870
3461
  ? `(application_version = $${n} OR application_version IS NULL)`
2871
3462
  : `application_version = $${n}`;
3463
+ // The partition's head: $1 queue, $2 ENQUEUED, `version` the app version, `pk` the partition key expression.
3464
+ const headOf = (pk, version, scope) => `SELECT workflow_uuid
3465
+ FROM "${this.schemaName}".workflow_status
3466
+ WHERE queue_name = $1 AND status = $2
3467
+ AND queue_partition_key = ${pk}
3468
+ AND ${versionClause(version)}
3469
+ AND ${scope}
3470
+ -- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
3471
+ ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
3472
+ LIMIT 1`;
3473
+ // No application filter by design: a mutual-exclusion probe must block on any owner's row.
3474
+ const noPending = (pk, pending, scope) => `NOT EXISTS (
3475
+ SELECT 1
3476
+ FROM "${this.schemaName}".workflow_status
3477
+ WHERE ${scope} AND queue_name = $1 AND status = $${pending}
3478
+ AND queue_partition_key IS NOT NULL AND queue_partition_key = ${pk}
3479
+ )`;
2872
3480
  // This worker's own budget bounds the sweep alongside the cap.
2873
3481
  const sweepLimit = Math.min(this.partitionedDequeueSweepCap, maxTasks);
2874
3482
  // When the worker's budget is the binding constraint, probe partitions in random order to prevent starvation.
@@ -2880,8 +3488,10 @@ class SystemDatabase {
2880
3488
  workflow_1.StatusString.PENDING,
2881
3489
  sweepLimit,
2882
3490
  ];
2883
- const candidateScope = this.appNameFilter('application_name', this.appName, candidateParams);
3491
+ const candidateScope = `${this.scoped(undefined, candidateParams)} AND ${this.appNameFilter('application_name', this.appName, candidateParams)}`;
3492
+ const probeScope = this.scoped(undefined, candidateParams);
2884
3493
  // Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
3494
+ // The walk takes no lock: each batch below picks its heads again under its partitions' locks.
2885
3495
  const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
2886
3496
  (SELECT MIN(queue_partition_key) AS pk
2887
3497
  FROM "${this.schemaName}".workflow_status
@@ -2897,118 +3507,140 @@ class SystemDatabase {
2897
3507
  SELECT partitions.pk
2898
3508
  FROM partitions
2899
3509
  WHERE partitions.pk IS NOT NULL
2900
- -- Unscoped by design: a mutual-exclusion probe must block on any owner's row.
2901
- AND NOT EXISTS (
2902
- SELECT 1
2903
- FROM "${this.schemaName}".workflow_status
2904
- WHERE queue_name = $1 AND status = $4
2905
- AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
2906
- )
3510
+ AND ${noPending('partitions.pk', 4, probeScope)}
2907
3511
  ORDER BY ${sweepOrder}
2908
3512
  LIMIT $5
2909
3513
  )
2910
- SELECT head.workflow_uuid
3514
+ SELECT chosen.pk
2911
3515
  FROM chosen
2912
- -- LATERAL plans as a tight nested loop; a correlated scalar subquery runs as a slower per-row SubPlan.
2913
- JOIN LATERAL (
2914
- SELECT workflow_uuid
2915
- FROM "${this.schemaName}".workflow_status
2916
- WHERE queue_name = $1 AND status = $2
2917
- AND queue_partition_key = chosen.pk
2918
- AND ${versionClause(3)}
2919
- AND ${candidateScope}
2920
- -- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
2921
- ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
2922
- LIMIT 1
2923
- ) head ON TRUE
2924
- -- Which partitions were chosen is settled above; order the result so the claim, and the dispatch it feeds, are deterministic.
3516
+ -- Only partitions with a head this worker may run; LATERAL plans as a tight nested loop.
3517
+ JOIN LATERAL (${headOf('chosen.pk', 3, candidateScope)}) head ON TRUE
3518
+ -- Which partitions were chosen is settled above; order them so the claim, and the dispatch it feeds, are deterministic.
2925
3519
  ORDER BY chosen.pk ASC`, candidateParams);
2926
- const candidateIDs = candidateResult.rows.map((row) => row.workflow_uuid);
2927
- if (candidateIDs.length === 0) {
2928
- await client.query('COMMIT');
2929
- return [];
2930
- }
2931
- await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
3520
+ const partitionKeys = candidateResult.rows.map((row) => row.pk);
2932
3521
  // Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
2933
- const claimGuard = (ids, status, name, version, scope) => `workflow_uuid = ANY($${ids}::text[])
3522
+ const claimGuard = (ids, status, name, version, scope) => `${ids}
2934
3523
  AND status = $${status}
2935
3524
  AND queue_name = $${name}
2936
3525
  AND queue_partition_key IS NOT NULL
2937
3526
  AND ${versionClause(version)}
2938
3527
  AND ${scope}`;
2939
- const lockedParams = [candidateIDs, workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
2940
- const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
2941
- // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
2942
- const lockedResult = await client.query(`SELECT workflow_uuid
2943
- FROM "${this.schemaName}".workflow_status
2944
- WHERE ${claimGuard(1, 2, 3, 4, lockedScope)}
2945
- FOR UPDATE SKIP LOCKED`, lockedParams);
2946
- const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
2947
- // Preserve partition order for submission.
2948
- const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
2949
- if (claimIDs.length === 0) {
2950
- await client.query('COMMIT');
2951
- return [];
3528
+ /** Claim the heads of these partitions inside the open transaction, returning the IDs it marked PENDING. */
3529
+ const claimBatch = async (batch) => {
3530
+ let heldPks = batch;
3531
+ if (usePartitionLocks) {
3532
+ await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_BEFORE_LOCKS);
3533
+ const lockParams = [];
3534
+ const { rows } = await client.query(this.partitionLockQuery(queue.name, batch, lockParams), lockParams);
3535
+ // Partitions another sweep holds are left to it, or to the next poll.
3536
+ heldPks = rows[0]?.held ?? [];
3537
+ if (heldPks.length === 0)
3538
+ return [];
3539
+ }
3540
+ // Pick the heads again under the locks, so a previous holder's claims and any newly visible head are seen.
3541
+ const headParams = [queue.name, workflow_1.StatusString.ENQUEUED, appVersion, workflow_1.StatusString.PENDING];
3542
+ const headScope = `${this.scoped(undefined, headParams)} AND ${this.appNameFilter('application_name', this.appName, headParams)}`;
3543
+ const headProbeScope = this.scoped(undefined, headParams);
3544
+ headParams.push(heldPks);
3545
+ const headResult = await client.query(`SELECT head.workflow_uuid
3546
+ FROM unnest($${headParams.length}::text[]) AS chosen(pk)
3547
+ JOIN LATERAL (${headOf('chosen.pk', 3, headScope)}) head ON TRUE
3548
+ WHERE ${noPending('chosen.pk', 4, headProbeScope)}
3549
+ ORDER BY chosen.pk ASC`, headParams);
3550
+ const candidateIDs = headResult.rows.map((row) => row.workflow_uuid);
3551
+ if (candidateIDs.length === 0)
3552
+ return [];
3553
+ await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
3554
+ const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
3555
+ const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
3556
+ const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
3557
+ // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
3558
+ const lockedResult = await client.query(`SELECT workflow_uuid
3559
+ FROM "${this.schemaName}".workflow_status
3560
+ WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
3561
+ FOR UPDATE SKIP LOCKED`, lockedParams);
3562
+ const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
3563
+ // Preserve partition order for submission.
3564
+ const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
3565
+ if (claimIDs.length === 0)
3566
+ return [];
3567
+ const flipParams = [
3568
+ workflow_1.StatusString.PENDING,
3569
+ executorID,
3570
+ workflow_1.StatusString.ENQUEUED,
3571
+ queue.name,
3572
+ appVersion,
3573
+ // Claim the row, as the unpartitioned dequeue does.
3574
+ this.appName ?? null,
3575
+ ownerXid,
3576
+ startTimeMs,
3577
+ ];
3578
+ const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
3579
+ const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
3580
+ // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
3581
+ const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
3582
+ SET status = $1,
3583
+ owner_xid = $7,
3584
+ executor_id = $2,
3585
+ application_version = $5,
3586
+ started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3587
+ rate_limited = FALSE,
3588
+ application_name = COALESCE(application_name, $6),
3589
+ recovery_attempts = recovery_attempts + 1,
3590
+ updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3591
+ workflow_deadline_epoch_ms = CASE
3592
+ WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
3593
+ CASE
3594
+ WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
3595
+ THEN 9223372036854775807
3596
+ ELSE $8::bigint + workflow_timeout_ms
3597
+ END
3598
+ ELSE workflow_deadline_epoch_ms
3599
+ END
3600
+ WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
3601
+ RETURNING workflow_uuid`, flipParams);
3602
+ const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
3603
+ return claimIDs.filter((id) => flippedIDs.has(id));
3604
+ };
3605
+ // One short transaction per batch, so a sweep holds at most a batch's locks, and only while it claims that batch.
3606
+ const claimedIDs = [];
3607
+ for (let start = 0; start < partitionKeys.length; start += this.partitionedDequeueBatchSize) {
3608
+ const batch = partitionKeys.slice(start, start + this.partitionedDequeueBatchSize);
3609
+ try {
3610
+ await client.query('BEGIN');
3611
+ const batchIDs = await claimBatch(batch);
3612
+ await client.query('COMMIT');
3613
+ claimedIDs.push(...batchIDs);
3614
+ }
3615
+ catch (error) {
3616
+ // Swallowed: a rollback failure must not mask why we are rolling back.
3617
+ await client.query('ROLLBACK').catch(() => undefined);
3618
+ // Earlier batches committed their claims, which must still reach dispatch; the next poll retries the rest.
3619
+ if (claimedIDs.length === 0)
3620
+ throw error;
3621
+ this.logger.warn(`[${queue.name}] stopping a partitioned sweep after ${claimedIDs.length} claim(s): ${String(error)}`);
3622
+ break;
3623
+ }
2952
3624
  }
2953
- const flipParams = [
2954
- workflow_1.StatusString.PENDING,
2955
- executorID,
2956
- claimIDs,
2957
- workflow_1.StatusString.ENQUEUED,
2958
- queue.name,
2959
- appVersion,
2960
- // Claim the row, as the unpartitioned dequeue does.
2961
- this.appName ?? null,
2962
- ownerXid,
2963
- ];
2964
- const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
2965
- // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
2966
- const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
2967
- SET status = $1,
2968
- owner_xid = $8,
2969
- executor_id = $2,
2970
- application_version = $6,
2971
- started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2972
- rate_limited = FALSE,
2973
- application_name = COALESCE(application_name, $7),
2974
- recovery_attempts = recovery_attempts + 1,
2975
- updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
2976
- workflow_deadline_epoch_ms = CASE
2977
- WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
2978
- CASE
2979
- WHEN workflow_timeout_ms > 9223372036854775807 - (EXTRACT(epoch FROM now()) * 1000)::bigint
2980
- THEN 9223372036854775807
2981
- ELSE (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
2982
- END
2983
- ELSE workflow_deadline_epoch_ms
2984
- END
2985
- WHERE ${claimGuard(3, 4, 5, 6, flipScope)}
2986
- RETURNING workflow_uuid`, flipParams);
2987
- await client.query('COMMIT');
2988
- const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
2989
- const claimedIDs = claimIDs.filter((id) => flippedIDs.has(id));
2990
3625
  if (claimedIDs.length > 0) {
2991
3626
  this.logger.debug(`[${queue.name}] dequeueing ${claimedIDs.length} task(s)`);
2992
3627
  }
2993
3628
  return claimedIDs;
2994
3629
  }
2995
- catch (error) {
2996
- await client.query('ROLLBACK');
2997
- throw error;
2998
- }
2999
3630
  finally {
3000
3631
  client.release();
3001
3632
  }
3002
3633
  }
3003
3634
  // ==================== Queries & Maintenance ====================
3004
- async listWorkflows(input) {
3005
- return await this.#listWorkflows(input);
3635
+ /** With `keyedIDs`, the listed workflow IDs are addressed by their shard keys, as a batch read of known workflows. */
3636
+ async listWorkflows(input, options = {}) {
3637
+ return await this.#listWorkflows(input, options);
3006
3638
  }
3007
3639
  /**
3008
3640
  * Unretried body of {@link listWorkflows}. An ID-keyed read runs on `client` when given, so a caller
3009
3641
  * holding a transaction never waits on a second pool connection; the caller owns the retry.
3010
3642
  */
3011
- async #listWorkflows(input, client) {
3643
+ async #listWorkflows(input, options = {}, client) {
3012
3644
  const schemaName = this.schemaName;
3013
3645
  const selectColumns = [
3014
3646
  'workflow_uuid',
@@ -3050,16 +3682,23 @@ class SystemDatabase {
3050
3682
  const payloadJoins = [];
3051
3683
  if (input.loadInput) {
3052
3684
  payloadColumns.push('COALESCE(wi.inputs, workflow_status.inputs) AS inputs');
3053
- payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON wi.workflow_uuid = workflow_status.workflow_uuid`);
3685
+ payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON ${this.wfJoin('wi', 'workflow_status')}`);
3054
3686
  }
3055
3687
  if (input.loadOutput) {
3056
3688
  payloadColumns.push('COALESCE(wo.output, workflow_status.output) AS output');
3057
3689
  payloadColumns.push('COALESCE(wo.error, workflow_status.error) AS error');
3058
- payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON wo.workflow_uuid = workflow_status.workflow_uuid`);
3690
+ payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON ${this.wfJoin('wo', 'workflow_status')}`);
3059
3691
  }
3060
3692
  if (input.loadInput || input.loadOutput) {
3061
3693
  selectColumns.push('serialization');
3062
3694
  }
3695
+ const keyColumns = [];
3696
+ const keyColumn = this.keyColumn('workflow_status');
3697
+ if (keyColumn !== undefined)
3698
+ keyColumns.push(`${keyColumn} AS shard_key`);
3699
+ const parentKeyColumn = this.keyColumn('workflow_status', 'parent_shard_key');
3700
+ if (parentKeyColumn !== undefined)
3701
+ keyColumns.push(`${parentKeyColumn} AS parent_shard_key`);
3063
3702
  input.sortDesc = input.sortDesc ?? false; // By default, sort in ascending order
3064
3703
  // Build WHERE clauses
3065
3704
  const whereClauses = [];
@@ -3102,6 +3741,18 @@ class SystemDatabase {
3102
3741
  whereClauses.push(idKeyed
3103
3742
  ? this.appNameFilter('application_name', input.applicationName, params)
3104
3743
  : this.observabilityFilter('application_name', input.applicationName, params));
3744
+ // An empty key list matches nothing, rather than every key.
3745
+ if (input.shardKey !== undefined && input.shardKey !== null) {
3746
+ const keys = Array.isArray(input.shardKey) ? input.shardKey : [input.shardKey];
3747
+ whereClauses.push(this.keyIn('workflow_status', keys, params));
3748
+ }
3749
+ if (input.parentShardKey !== undefined && input.parentShardKey !== null) {
3750
+ const keys = Array.isArray(input.parentShardKey) ? input.parentShardKey : [input.parentShardKey];
3751
+ whereClauses.push(this.keyIn('workflow_status', keys, params, 'parent_shard_key'));
3752
+ }
3753
+ if (options.keyedIDs && input.workflowIDs) {
3754
+ whereClauses.push(this.wfIn('workflow_status', input.workflowIDs, params));
3755
+ }
3105
3756
  paramCounter = params.length + 1;
3106
3757
  if (input.workflow_id_prefix) {
3107
3758
  if (Array.isArray(input.workflow_id_prefix)) {
@@ -3116,7 +3767,7 @@ class SystemDatabase {
3116
3767
  paramCounter++;
3117
3768
  }
3118
3769
  }
3119
- if (input.workflowIDs) {
3770
+ if (input.workflowIDs && !options.keyedIDs) {
3120
3771
  const placeholders = input.workflowIDs.map((_, i) => `$${paramCounter + i}`).join(', ');
3121
3772
  whereClauses.push(`workflow_status.workflow_uuid IN (${placeholders})`);
3122
3773
  params.push(...input.workflowIDs);
@@ -3198,7 +3849,7 @@ class SystemDatabase {
3198
3849
  params.push(input.offset);
3199
3850
  paramCounter++;
3200
3851
  }
3201
- const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns].join(', ');
3852
+ const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns, ...keyColumns].join(', ');
3202
3853
  const query = `
3203
3854
  SELECT ${projection}
3204
3855
  FROM "${schemaName}".workflow_status
@@ -3218,23 +3869,16 @@ class SystemDatabase {
3218
3869
  async createSchedule(schedule, client) {
3219
3870
  const q = client ?? this.pool;
3220
3871
  const owner = await this.#resolveRowOwner(q, 'workflow_schedules', 'schedule_name', schedule.scheduleName, schedule.applicationName, 'Schedule');
3872
+ const values = {
3873
+ ...this.globalValues(),
3874
+ ...this.scheduleValues(schedule),
3875
+ ...scheduleRow(schedule, owner),
3876
+ };
3221
3877
  try {
3222
- await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules
3223
- (schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name)
3224
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, [
3225
- schedule.scheduleId,
3226
- schedule.scheduleName,
3227
- schedule.workflowName,
3228
- schedule.workflowClassName,
3229
- schedule.schedule,
3230
- schedule.status,
3231
- schedule.context,
3232
- schedule.lastFiredAt,
3233
- schedule.automaticBackfill,
3234
- schedule.cronTimezone,
3235
- schedule.queueName,
3236
- owner ?? null,
3237
- ]);
3878
+ await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
3879
+ VALUES (${Object.keys(values)
3880
+ .map((_, i) => `$${i + 1}`)
3881
+ .join(', ')})`, Object.values(values));
3238
3882
  }
3239
3883
  catch (e) {
3240
3884
  if (isPgDatabaseError(e) && e.code === '23505') {
@@ -3281,31 +3925,37 @@ class SystemDatabase {
3281
3925
  // Unset scopes to this application, as on every other observability query.
3282
3926
  conditions.push(this.observabilityFilter('application_name', filters?.applicationName, params));
3283
3927
  paramIdx = params.length + 1;
3928
+ conditions.push(this.global(undefined, params));
3284
3929
  const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
3285
- const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
3930
+ const result = await q.query(`SELECT ${this.#scheduleColumns()}
3286
3931
  FROM "${this.schemaName}".workflow_schedules${where}
3287
3932
  ORDER BY schedule_name`, params);
3288
3933
  return result.rows.map((row) => mapWorkflowSchedule(row));
3289
3934
  }
3290
3935
  async getSchedule(name, client) {
3291
3936
  const q = client ?? this.pool;
3292
- const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
3937
+ const params = [name];
3938
+ const result = await q.query(`SELECT ${this.#scheduleColumns()}
3293
3939
  FROM "${this.schemaName}".workflow_schedules
3294
- WHERE schedule_name = $1`, [name]);
3940
+ WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
3295
3941
  if (result.rows.length === 0)
3296
3942
  return null;
3297
3943
  return mapWorkflowSchedule(result.rows[0]);
3298
3944
  }
3945
+ /** The columns a schedule is read back with, its workflows' shard key among them on a sharded database. */
3946
+ #scheduleColumns() {
3947
+ const keyColumn = this.scheduleKeyColumn();
3948
+ return keyColumn !== undefined ? `${SCHEDULE_COLUMNS}, ${keyColumn}` : SCHEDULE_COLUMNS;
3949
+ }
3299
3950
  async deleteSchedule(name, client) {
3300
3951
  const q = client ?? this.pool;
3301
- await q.query(`DELETE FROM "${this.schemaName}".workflow_schedules WHERE schedule_name = $1`, [name]);
3952
+ const params = [name];
3953
+ await q.query(`DELETE FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
3302
3954
  }
3303
3955
  async setScheduleStatus(name, status, client) {
3304
3956
  const q = client ?? this.pool;
3305
- await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET status = $1 WHERE schedule_name = $2`, [
3306
- status,
3307
- name,
3308
- ]);
3957
+ const params = [status, name];
3958
+ await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET status = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
3309
3959
  }
3310
3960
  async updateSchedule(name, updates, client) {
3311
3961
  const q = client ?? this.pool;
@@ -3328,22 +3978,22 @@ class SystemDatabase {
3328
3978
  }
3329
3979
  if (setClauses.length === 0) {
3330
3980
  // Nothing to change, but still surface a missing schedule as an error.
3331
- const existing = await q.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules WHERE schedule_name = $1`, [
3332
- name,
3333
- ]);
3981
+ const existingParams = [name];
3982
+ const existing = await q.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, existingParams)} AND schedule_name = $1`, existingParams);
3334
3983
  if (existing.rows.length === 0) {
3335
3984
  throw new error_1.DBOSError(`Schedule '${name}' not found`);
3336
3985
  }
3337
3986
  return;
3338
3987
  }
3339
3988
  params.push(name);
3340
- const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx}`, params);
3989
+ const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx} AND ${this.global(undefined, params)}`, params);
3341
3990
  if (result.rowCount === 0) {
3342
3991
  throw new error_1.DBOSError(`Schedule '${name}' not found`);
3343
3992
  }
3344
3993
  }
3345
3994
  async updateLastFiredAt(name, lastFiredAt) {
3346
- await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE schedule_name = $2`, [lastFiredAt, name]);
3995
+ const params = [lastFiredAt, name];
3996
+ await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
3347
3997
  }
3348
3998
  async applySchedules(schedules) {
3349
3999
  const client = await this.connect();
@@ -3351,12 +4001,21 @@ class SystemDatabase {
3351
4001
  await client.query('BEGIN');
3352
4002
  for (const sched of schedules) {
3353
4003
  const owner = await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
4004
+ const scheduleKeys = this.scheduleValues(sched);
4005
+ const values = {
4006
+ ...this.globalValues(),
4007
+ ...scheduleKeys,
4008
+ ...scheduleRow(sched, owner),
4009
+ };
3354
4010
  // Upsert on schedule_name; on conflict, preserve schedule_id and runtime state (status, last_fired_at) and update only the declared definition fields, so an unchanged re-apply is a no-op.
3355
- await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules
3356
- (schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name)
3357
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
3358
- ON CONFLICT (schedule_name) DO UPDATE SET
3359
- workflow_name = EXCLUDED.workflow_name,
4011
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
4012
+ VALUES (${Object.keys(values)
4013
+ .map((_, i) => `$${i + 1}`)
4014
+ .join(', ')})
4015
+ ON CONFLICT (${this.keyConflict('schedule_name')}) DO UPDATE SET
4016
+ ${Object.keys(scheduleKeys)
4017
+ .map((c) => `${c} = EXCLUDED.${c}, `)
4018
+ .join('')}workflow_name = EXCLUDED.workflow_name,
3360
4019
  workflow_class_name = EXCLUDED.workflow_class_name,
3361
4020
  schedule = EXCLUDED.schedule,
3362
4021
  context = EXCLUDED.context,
@@ -3364,20 +4023,7 @@ class SystemDatabase {
3364
4023
  cron_timezone = EXCLUDED.cron_timezone,
3365
4024
  queue_name = EXCLUDED.queue_name,
3366
4025
  -- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it took.
3367
- application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, [
3368
- sched.scheduleId,
3369
- sched.scheduleName,
3370
- sched.workflowName,
3371
- sched.workflowClassName,
3372
- sched.schedule,
3373
- sched.status,
3374
- sched.context,
3375
- sched.lastFiredAt,
3376
- sched.automaticBackfill,
3377
- sched.cronTimezone,
3378
- sched.queueName,
3379
- owner ?? null,
3380
- ]);
4026
+ application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, Object.values(values));
3381
4027
  // Read back, since the guard above is silent about why it declined to claim.
3382
4028
  await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
3383
4029
  }
@@ -3402,14 +4048,23 @@ class SystemDatabase {
3402
4048
  try {
3403
4049
  await client.query('BEGIN');
3404
4050
  // Claim a pre-upgrade row in place, so the version is not recreated or retimed.
4051
+ const claimParams = [owner ?? null, versionName];
3405
4052
  const claimed = await client.query(`UPDATE "${this.schemaName}".application_versions
3406
4053
  SET application_name = $1
3407
- WHERE version_name = $2 AND application_name IS NULL`, [owner ?? null, versionName]);
4054
+ WHERE ${this.global(undefined, claimParams)} AND version_name = $2 AND application_name IS NULL`, claimParams);
3408
4055
  if ((claimed.rowCount ?? 0) === 0) {
4056
+ const values = {
4057
+ ...this.globalValues(),
4058
+ version_id: (0, crypto_1.randomUUID)(),
4059
+ version_name: versionName,
4060
+ application_name: owner ?? null,
4061
+ };
3409
4062
  // Targetless DO NOTHING: names no arbiter, so it survives version_name's uniqueness being dropped while still absorbing a concurrent registrar.
3410
- await client.query(`INSERT INTO "${this.schemaName}".application_versions (version_id, version_name, application_name)
3411
- VALUES ($1, $2, $3)
3412
- ON CONFLICT DO NOTHING`, [(0, crypto_1.randomUUID)(), versionName, owner ?? null]);
4063
+ await client.query(`INSERT INTO "${this.schemaName}".application_versions (${Object.keys(values).join(', ')})
4064
+ VALUES (${Object.keys(values)
4065
+ .map((_, i) => `$${i + 1}`)
4066
+ .join(', ')})
4067
+ ON CONFLICT DO NOTHING`, Object.values(values));
3413
4068
  }
3414
4069
  // Read back, since the writes above are silent about why they declined to claim.
3415
4070
  await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
@@ -3440,7 +4095,7 @@ class SystemDatabase {
3440
4095
  params.push(resolved);
3441
4096
  await client.query(`UPDATE "${this.schemaName}".application_versions
3442
4097
  SET version_timestamp = $1, application_name = ${resolved === undefined ? 'application_name' : '$3'}
3443
- WHERE version_name = $2 AND ${scope}`, params);
4098
+ WHERE ${this.global(undefined, params)} AND version_name = $2 AND ${scope}`, params);
3444
4099
  await client.query('COMMIT');
3445
4100
  }
3446
4101
  catch (e) {
@@ -3453,7 +4108,7 @@ class SystemDatabase {
3453
4108
  }
3454
4109
  async listApplicationVersions() {
3455
4110
  const params = [];
3456
- const scope = this.appNameFilter('application_name', this.appName, params);
4111
+ const scope = `${this.global(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
3457
4112
  const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
3458
4113
  FROM "${this.schemaName}".application_versions
3459
4114
  WHERE ${scope}
@@ -3473,9 +4128,10 @@ class SystemDatabase {
3473
4128
  }
3474
4129
  // ==================== Queues ====================
3475
4130
  async getQueue(name) {
4131
+ const params = [name];
3476
4132
  const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
3477
4133
  FROM "${this.schemaName}".queues
3478
- WHERE name = $1`, [name]);
4134
+ WHERE ${this.global(undefined, params)} AND name = $1`, params);
3479
4135
  return rows.length === 0 ? null : queueRecordFromRow(rows[0]);
3480
4136
  }
3481
4137
  /**
@@ -3484,14 +4140,15 @@ class SystemDatabase {
3484
4140
  */
3485
4141
  async listQueues(applicationName) {
3486
4142
  const params = [];
3487
- const scope = this.observabilityFilter('application_name', applicationName, params);
4143
+ const scope = `${this.global(undefined, params)} AND ${this.observabilityFilter('application_name', applicationName, params)}`;
3488
4144
  const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
3489
4145
  FROM "${this.schemaName}".queues
3490
4146
  WHERE ${scope}`, params);
3491
4147
  return rows.map(queueRecordFromRow);
3492
4148
  }
3493
4149
  async deleteQueue(name) {
3494
- await this.pool.query(`DELETE FROM "${this.schemaName}".queues WHERE name = $1`, [name]);
4150
+ const params = [name];
4151
+ await this.pool.query(`DELETE FROM "${this.schemaName}".queues WHERE ${this.global(undefined, params)} AND name = $1`, params);
3495
4152
  }
3496
4153
  async updateQueue(name, fields) {
3497
4154
  const setClauses = [];
@@ -3507,7 +4164,7 @@ class SystemDatabase {
3507
4164
  setClauses.push(`"updated_at" = $${idx++}`);
3508
4165
  params.push(Date.now());
3509
4166
  params.push(name);
3510
- await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx}`, params);
4167
+ await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx} AND ${this.global(undefined, params)}`, params);
3511
4168
  }
3512
4169
  /** Returns true iff this call inserted a new row (i.e. the queue did not
3513
4170
  * previously exist). False if the row already existed, regardless of
@@ -3515,7 +4172,7 @@ class SystemDatabase {
3515
4172
  async upsertQueue(record, updateExisting) {
3516
4173
  const now = Date.now();
3517
4174
  const onConflict = updateExisting
3518
- ? `ON CONFLICT (name) DO UPDATE SET
4175
+ ? `ON CONFLICT (${this.keyConflict('name')}) DO UPDATE SET
3519
4176
  concurrency = EXCLUDED.concurrency,
3520
4177
  worker_concurrency = EXCLUDED.worker_concurrency,
3521
4178
  rate_limit_max = EXCLUDED.rate_limit_max,
@@ -3530,21 +4187,27 @@ class SystemDatabase {
3530
4187
  updated_at = EXCLUDED.updated_at,
3531
4188
  -- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it just took.
3532
4189
  application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
3533
- : `ON CONFLICT (name) DO NOTHING`;
4190
+ : `ON CONFLICT (${this.keyConflict('name')}) DO NOTHING`;
3534
4191
  const owner = record.applicationName ?? this.appName;
3535
4192
  const client = await this.connect();
3536
4193
  try {
3537
4194
  await client.query('BEGIN');
3538
- const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE name = $1`, [record.name]);
4195
+ const existedParams = [record.name];
4196
+ const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE ${this.global(undefined, existedParams)} AND name = $1`, existedParams);
3539
4197
  // A name collision is a conflict in every mode: the name is the queue's address.
3540
4198
  const resolvedOwner = await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
4199
+ const keys = this.globalValues();
3541
4200
  await client.query(`INSERT INTO "${this.schemaName}".queues
3542
- (name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
4201
+ (${Object.keys(keys)
4202
+ .map((c) => `${c}, `)
4203
+ .join('')}name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
3543
4204
  priority_enabled, partition_queue, partition_concurrency, partition_worker_concurrency,
3544
4205
  partition_rate_limit_max, partition_rate_limit_period_sec,
3545
4206
  polling_interval_sec, updated_at, application_name)
3546
4207
  -- priority_enabled is vestigial: every queue dispatches in priority order.
3547
- VALUES ($1, $2, $3, $4, $5, TRUE, $6, $7, $8, $9, $10, $11, $12, $13)
4208
+ VALUES (${Object.keys(keys)
4209
+ .map((_, i) => `$${14 + i}, `)
4210
+ .join('')}$1, $2, $3, $4, $5, TRUE, $6, $7, $8, $9, $10, $11, $12, $13)
3548
4211
  ${onConflict}`, [
3549
4212
  record.name,
3550
4213
  record.concurrency,
@@ -3559,6 +4222,7 @@ class SystemDatabase {
3559
4222
  record.pollingIntervalSec,
3560
4223
  now,
3561
4224
  resolvedOwner ?? null,
4225
+ ...Object.values(keys),
3562
4226
  ]);
3563
4227
  // Read back, since the guard above is silent about why it declined to claim.
3564
4228
  await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
@@ -3668,6 +4332,9 @@ class SystemDatabase {
3668
4332
  params.push(statuses);
3669
4333
  where = `${where} AND status = ANY($${params.length})`;
3670
4334
  }
4335
+ else {
4336
+ where = `${where} AND ${this.global(undefined, params)}`;
4337
+ }
3671
4338
  const res = await client.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${where}`, params);
3672
4339
  return res.rowCount ?? 0;
3673
4340
  };
@@ -3692,84 +4359,64 @@ class SystemDatabase {
3692
4359
  }
3693
4360
  // ==================== Internal ====================
3694
4361
  async insertWorkflowStatus(client, initStatus, creatorXid) {
4362
+ const values = {
4363
+ ...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
4364
+ workflow_uuid: initStatus.workflowUUID,
4365
+ status: initStatus.status,
4366
+ name: initStatus.workflowName,
4367
+ // For cross-language compatibility, these variables MUST be NULL in the database when not set
4368
+ class_name: initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
4369
+ config_name: initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
4370
+ queue_name: initStatus.queueName ?? null,
4371
+ authenticated_user: initStatus.authenticatedUser,
4372
+ assumed_role: initStatus.assumedRole,
4373
+ authenticated_roles: JSON.stringify(initStatus.authenticatedRoles),
4374
+ executor_id: initStatus.executorId,
4375
+ application_version: initStatus.applicationVersion ?? null,
4376
+ application_id: initStatus.applicationID,
4377
+ recovery_attempts: initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
4378
+ workflow_timeout_ms: initStatus.timeoutMS ?? null,
4379
+ workflow_deadline_epoch_ms: initStatus.deadlineEpochMS ?? null,
4380
+ // Legacy column: the payload lives in workflow_input.
4381
+ inputs: null,
4382
+ deduplication_id: initStatus.deduplicationID ?? null,
4383
+ priority: initStatus.priority,
4384
+ queue_partition_key: initStatus.queuePartitionKey ?? null,
4385
+ forked_from: initStatus.forkedFrom ?? null,
4386
+ parent_workflow_id: initStatus.parentWorkflowID ?? null,
4387
+ ...this.parentValues(initStatus.parentWorkflowID),
4388
+ serialization: initStatus.serialization,
4389
+ creator_xid: creatorXid,
4390
+ delay_until_epoch_ms: initStatus.delayUntilEpochMS ?? null,
4391
+ attributes: initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
4392
+ schedule_name: initStatus.scheduleName ?? null,
4393
+ debounce_deadline_epoch_ms: initStatus.debounceDeadlineEpochMS ?? null,
4394
+ is_debounced: initStatus.isDebounced ?? false,
4395
+ application_name: initStatus.applicationName ?? null,
4396
+ // A direct start runs at once, so its creator token also owns the execution.
4397
+ owner_xid: initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
4398
+ };
4399
+ const columns = Object.keys(values);
4400
+ const inputValues = {
4401
+ ...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
4402
+ workflow_uuid: initStatus.workflowUUID,
4403
+ inputs: initStatus.input ?? null,
4404
+ };
4405
+ const inputColumns = Object.keys(inputValues);
3695
4406
  try {
3696
- const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
3697
- workflow_uuid,
3698
- status,
3699
- name,
3700
- class_name,
3701
- config_name,
3702
- queue_name,
3703
- authenticated_user,
3704
- assumed_role,
3705
- authenticated_roles,
3706
- executor_id,
3707
- application_version,
3708
- application_id,
3709
- recovery_attempts,
3710
- workflow_timeout_ms,
3711
- workflow_deadline_epoch_ms,
3712
- inputs,
3713
- deduplication_id,
3714
- priority,
3715
- queue_partition_key,
3716
- forked_from,
3717
- parent_workflow_id,
3718
- serialization,
3719
- creator_xid,
3720
- delay_until_epoch_ms,
3721
- attributes,
3722
- schedule_name,
3723
- debounce_deadline_epoch_ms,
3724
- is_debounced,
3725
- application_name,
3726
- owner_xid
3727
- ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)
3728
- ON CONFLICT (workflow_uuid)
4407
+ const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
4408
+ VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
4409
+ ON CONFLICT (${this.keyConflict('workflow_uuid')})
3729
4410
  -- A no-op update, so an existing row comes back unchanged for the caller to inspect.
3730
4411
  DO UPDATE SET creator_xid = workflow_status.creator_xid
3731
- RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`, [
3732
- initStatus.workflowUUID,
3733
- initStatus.status,
3734
- initStatus.workflowName,
3735
- // For cross-language compatibility, these variables MUST be NULL in the database when not set
3736
- initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
3737
- initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
3738
- initStatus.queueName ?? null,
3739
- initStatus.authenticatedUser,
3740
- initStatus.assumedRole,
3741
- JSON.stringify(initStatus.authenticatedRoles),
3742
- initStatus.executorId,
3743
- initStatus.applicationVersion ?? null,
3744
- initStatus.applicationID,
3745
- initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
3746
- initStatus.timeoutMS ?? null,
3747
- initStatus.deadlineEpochMS ?? null,
3748
- // Legacy column: the payload lives in workflow_input.
3749
- null,
3750
- initStatus.deduplicationID ?? null,
3751
- initStatus.priority,
3752
- initStatus.queuePartitionKey ?? null,
3753
- initStatus.forkedFrom ?? null,
3754
- initStatus.parentWorkflowID ?? null,
3755
- initStatus.serialization,
3756
- creatorXid,
3757
- initStatus.delayUntilEpochMS ?? null,
3758
- initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
3759
- initStatus.scheduleName ?? null,
3760
- initStatus.debounceDeadlineEpochMS ?? null,
3761
- initStatus.isDebounced ?? false,
3762
- initStatus.applicationName ?? null,
3763
- // A direct start runs at once, so its creator token also owns the execution.
3764
- initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
3765
- ]);
4412
+ RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`, Object.values(values));
3766
4413
  if (rows.length === 0) {
3767
4414
  throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
3768
4415
  }
3769
4416
  // Two statements, not a data-modifying CTE: at scale the CTE costs more than the round trip it saves.
3770
- await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs)
3771
- VALUES ($1, $2)
3772
- ON CONFLICT (workflow_uuid) DO NOTHING`, [initStatus.workflowUUID, initStatus.input ?? null]);
4417
+ await client.query(`INSERT INTO "${this.schemaName}".workflow_input (${inputColumns.join(', ')})
4418
+ VALUES (${inputColumns.map((_, i) => `$${i + 1}`).join(', ')})
4419
+ ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`, Object.values(inputValues));
3773
4420
  const ret = rows[0];
3774
4421
  ret.class_name = ret.class_name ?? '';
3775
4422
  ret.config_name = ret.config_name ?? '';
@@ -3785,16 +4432,17 @@ class SystemDatabase {
3785
4432
  }
3786
4433
  }
3787
4434
  async getWorkflowStatusValue(client, workflowID) {
3788
- const { rows } = await client.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE workflow_uuid=$1`, [workflowID]);
4435
+ const params = [];
4436
+ const { rows } = await client.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
3789
4437
  return rows.length === 0 ? undefined : rows[0].status;
3790
4438
  }
3791
4439
  async updateWorkflowStatus(client, workflowID, status, options = {}) {
3792
4440
  // Use SQL now() so updated_at and completed_at (when set together) are
3793
4441
  // computed in the same statement against the same clock.
3794
4442
  const nowMsExpr = `(EXTRACT(EPOCH FROM now()) * 1000)::bigint`;
3795
- let setClause = `SET status=$2, updated_at=${nowMsExpr}`;
3796
- let whereClause = `WHERE workflow_uuid=$1`;
3797
- const args = [workflowID, status];
4443
+ let setClause = `SET status=$1, updated_at=${nowMsExpr}`;
4444
+ const args = [status];
4445
+ let whereClause = `WHERE ${this.wf(undefined, workflowID, args)}`;
3798
4446
  const update = options.update ?? {};
3799
4447
  if (update.resetRecoveryAttempts) {
3800
4448
  setClause += `, recovery_attempts = 0`;
@@ -3853,27 +4501,31 @@ class SystemDatabase {
3853
4501
  async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
3854
4502
  const ownerXid = options.ownerChecked ? undefined : (0, context_1.currentOwnerXid)(workflowID);
3855
4503
  if (ownerXid !== undefined) {
3856
- await this.#checkOwner(client, workflowID, ownerXid);
3857
- }
4504
+ await this.checkOwner(client, workflowID, ownerXid);
4505
+ }
4506
+ const values = {
4507
+ ...this.wfValues(workflowID),
4508
+ workflow_uuid: workflowID,
4509
+ function_id: functionID,
4510
+ output: options.output ?? null,
4511
+ error: options.error ?? null,
4512
+ function_name: functionName,
4513
+ child_workflow_id: options.childWorkflowID ?? null,
4514
+ ...(options.childValues ?? this.childValues(options.childWorkflowID)),
4515
+ started_at_epoch_ms: startTimeEpochMs,
4516
+ completed_at_epoch_ms: endTimeEpochMs,
4517
+ serialization: options.serialization ?? null,
4518
+ // Mirrors the parent: only the running application records its steps.
4519
+ application_name: this.appName ?? null,
4520
+ };
4521
+ const columns = Object.keys(values);
3858
4522
  try {
3859
4523
  const out = await client.query(`INSERT INTO "${this.schemaName}".operation_outputs
3860
- (workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization, application_name, retention_timestamp)
3861
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
3862
- ON CONFLICT (workflow_uuid, function_id) DO UPDATE
4524
+ (${columns.join(', ')}, retention_timestamp)
4525
+ VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')}, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
4526
+ ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id')}) DO UPDATE
3863
4527
  SET completed_at_epoch_ms = operation_outputs.completed_at_epoch_ms
3864
- RETURNING completed_at_epoch_ms;`, [
3865
- workflowID,
3866
- functionID,
3867
- options.output ?? null,
3868
- options.error ?? null,
3869
- functionName,
3870
- options.childWorkflowID ?? null,
3871
- startTimeEpochMs,
3872
- endTimeEpochMs,
3873
- options.serialization ?? null,
3874
- // Mirrors the parent: only the running application records its steps.
3875
- this.appName ?? null,
3876
- ]);
4528
+ RETURNING completed_at_epoch_ms;`, Object.values(values));
3877
4529
  // A row already there with another completion time is this execution's own doing
3878
4530
  // (a retry carries the same time), so it is nondeterminism, not a duplicate.
3879
4531
  if (checkConflict &&
@@ -3890,7 +4542,7 @@ class SystemDatabase {
3890
4542
  }
3891
4543
  }
3892
4544
  /** Run `fn` in a READ COMMITTED transaction on `client`, so an ownership check holds its row lock until the write commits. */
3893
- async #inTransaction(client, fn) {
4545
+ async inTransaction(client, fn) {
3894
4546
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
3895
4547
  let result;
3896
4548
  try {
@@ -3905,22 +4557,26 @@ class SystemDatabase {
3905
4557
  return result;
3906
4558
  }
3907
4559
  /** Throw DBOSWorkflowConflictError unless ownerXid still owns the workflow; the row lock holds until commit, so no hand-off lands in between. */
3908
- async #checkOwner(client, workflowID, ownerXid) {
3909
- const { rows } = await client.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = $1 FOR NO KEY UPDATE`, [workflowID]);
4560
+ async checkOwner(client, workflowID, ownerXid) {
4561
+ const params = [];
4562
+ const { rows } = await client.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)} FOR NO KEY UPDATE`, params);
3910
4563
  if (rows[0]?.owner_xid !== ownerXid) {
3911
4564
  throw new error_1.DBOSWorkflowConflictError(workflowID);
3912
4565
  }
3913
4566
  }
3914
4567
  /** The workflow's current ownership token; null if unowned or missing. */
3915
4568
  async getWorkflowOwner(workflowID) {
3916
- const { rows } = await this.pool.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = $1`, [workflowID]);
4569
+ const params = [];
4570
+ const { rows } = await this.pool.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
3917
4571
  return rows[0]?.owner_xid ?? null;
3918
4572
  }
3919
4573
  async #getOperationResultAndThrowIfCancelled(client, workflowID, functionID) {
3920
4574
  await this.#checkIfCanceled(client, workflowID);
3921
- const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name, serialization
4575
+ const childKey = this.keyColumn(undefined, 'child_shard_key');
4576
+ const params = [functionID];
4577
+ const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name, serialization${childKey !== undefined ? `, ${childKey}` : ''}
3922
4578
  FROM "${this.schemaName}".operation_outputs
3923
- WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
4579
+ WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
3924
4580
  if (rows.length === 0) {
3925
4581
  return undefined;
3926
4582
  }
@@ -3929,6 +4585,7 @@ class SystemDatabase {
3929
4585
  output: rows[0].output,
3930
4586
  error: rows[0].error,
3931
4587
  childWorkflowID: rows[0].child_workflow_id,
4588
+ childShardKey: rows[0].child_shard_key ?? undefined,
3932
4589
  functionName: rows[0].function_name,
3933
4590
  // Return serialization so recv/getEvent replay deserializes with the stored format, not the default.
3934
4591
  serialization: rows[0].serialization,
@@ -3966,7 +4623,11 @@ class SystemDatabase {
3966
4623
  // been cancelled.
3967
4624
  // For an actual sleep, completed_at is the wake deadline so the step's duration reflects the
3968
4625
  // sleep; a timeout marker records zero duration since its deadline may never be reached.
3969
- async #durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline = false) {
4626
+ async durableSleepInternal(workflowID, functionID, durationMS, recordCompletionAtDeadline = false) {
4627
+ // The caller's own workflow, even where a same-ID workflow is bound as a target.
4628
+ return await this.ownKey(workflowID, undefined, () => this.#durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline));
4629
+ }
4630
+ async #durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline) {
3970
4631
  const startTimeMs = Date.now();
3971
4632
  // Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
3972
4633
  const endTimeMs = startTimeMs + Math.ceil(durationMS);
@@ -3979,7 +4640,7 @@ class SystemDatabase {
3979
4640
  }
3980
4641
  return JSON.parse(res.output);
3981
4642
  }
3982
- await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
4643
+ await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
3983
4644
  output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
3984
4645
  serialization: serialization_1.DBOSPortableJSON.name(),
3985
4646
  }));
@@ -4159,7 +4820,7 @@ __decorate([
4159
4820
  __decorate([
4160
4821
  dbRetry(),
4161
4822
  __metadata("design:type", Function),
4162
- __metadata("design:paramtypes", [String, Number]),
4823
+ __metadata("design:paramtypes", [String, Number, String]),
4163
4824
  __metadata("design:returntype", Promise)
4164
4825
  ], SystemDatabase.prototype, "getOperationResultAndThrowIfCancelled", null);
4165
4826
  __decorate([
@@ -4219,9 +4880,9 @@ __decorate([
4219
4880
  __decorate([
4220
4881
  dbRetry(),
4221
4882
  __metadata("design:type", Function),
4222
- __metadata("design:paramtypes", [Array, Boolean]),
4883
+ __metadata("design:paramtypes", [Array]),
4223
4884
  __metadata("design:returntype", Promise)
4224
- ], SystemDatabase.prototype, "deleteWorkflows", null);
4885
+ ], SystemDatabase.prototype, "deleteWorkflowsInternal", null);
4225
4886
  __decorate([
4226
4887
  dbRetry()
4227
4888
  // A missing row normally means the workflow has not been inserted yet, so
@@ -4255,19 +4916,19 @@ __decorate([
4255
4916
  __decorate([
4256
4917
  dbRetry(),
4257
4918
  __metadata("design:type", Function),
4258
- __metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String]),
4919
+ __metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String, String]),
4259
4920
  __metadata("design:returntype", Promise)
4260
4921
  ], SystemDatabase.prototype, "send", null);
4261
4922
  __decorate([
4262
4923
  dbRetry(),
4263
4924
  __metadata("design:type", Function),
4264
- __metadata("design:paramtypes", [String, String, Object, Object, Object, String]),
4925
+ __metadata("design:paramtypes", [String, String, Object, Object, Object, String, String]),
4265
4926
  __metadata("design:returntype", Promise)
4266
4927
  ], SystemDatabase.prototype, "sendFromStep", null);
4267
4928
  __decorate([
4268
4929
  dbRetry(),
4269
4930
  __metadata("design:type", Function),
4270
- __metadata("design:paramtypes", [String, Object, Object, Object, String]),
4931
+ __metadata("design:paramtypes", [String, Object, Object, Object, String, String]),
4271
4932
  __metadata("design:returntype", Promise)
4272
4933
  ], SystemDatabase.prototype, "sendDirectStandalone", null);
4273
4934
  __decorate([
@@ -4321,7 +4982,7 @@ __decorate([
4321
4982
  __decorate([
4322
4983
  dbRetry(),
4323
4984
  __metadata("design:type", Function),
4324
- __metadata("design:paramtypes", [Object]),
4985
+ __metadata("design:paramtypes", [Object, Object]),
4325
4986
  __metadata("design:returntype", Promise)
4326
4987
  ], SystemDatabase.prototype, "listWorkflows", null);
4327
4988
  __decorate([