@dbos-inc/dbos-sdk 5.3.6-preview → 5.3.7-preview

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
9
9
  if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
10
  };
11
11
  Object.defineProperty(exports, "__esModule", { value: true });
12
- exports.SystemDatabase = exports.retriablePostgresException = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.advisoryLockKey = exports.retentionLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_GC_BATCH_SIZE = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
12
+ exports.SystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
13
13
  const dbos_executor_1 = require("./dbos-executor");
14
14
  const pg_1 = require("pg");
15
15
  const error_1 = require("./error");
@@ -88,8 +88,6 @@ function setsIdleTransactionTimeout(databaseUrl) {
88
88
  }
89
89
  // Workflows re-owned per transaction by a rename.
90
90
  exports.DEFAULT_RENAME_BATCH_SIZE = 10_000;
91
- // Rows deleted per transaction by garbage collection.
92
- exports.DEFAULT_GC_BATCH_SIZE = 50_000;
93
91
  const QUEUE_COLUMN_BY_FIELD = {
94
92
  concurrency: 'concurrency',
95
93
  workerConcurrency: 'worker_concurrency',
@@ -204,16 +202,6 @@ async function releaseSystemDatabaseClient(client, customPool) {
204
202
  }
205
203
  catch (e) { }
206
204
  }
207
- /**
208
- * The pg_try_advisory_lock argument for one schema's retention round: the leading 8 bytes of
209
- * SHA-256 over a fixed string, read as a signed big-endian 64-bit integer. Separate locks for
210
- * separate schemas. Every DBOS SDK derives the key this way, so rounds in different languages
211
- * against one system database contend for the same lock; changing it here changes it everywhere.
212
- */
213
- function retentionLockKey(schemaName) {
214
- return advisoryLockKey(`dbos.retention.${schemaName}`);
215
- }
216
- exports.retentionLockKey = retentionLockKey;
217
205
  /** A Postgres advisory lock key: the leading 8 bytes of SHA-256 over `lockName`, as a signed big-endian integer. */
218
206
  function advisoryLockKey(lockName) {
219
207
  return (0, crypto_1.createHash)('sha256').update(lockName).digest().readBigInt64BE(0);
@@ -223,6 +211,7 @@ async function isCockroachDB(client) {
223
211
  const versionRes = await client.query('SELECT version() AS version');
224
212
  return /cockroachdb/i.test(versionRes.rows[0]?.version ?? '');
225
213
  }
214
+ exports.isCockroachDB = isCockroachDB;
226
215
  async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
227
216
  if (!customPool) {
228
217
  // A custom pool means the database already exists; otherwise, create it if it does not.
@@ -378,14 +367,6 @@ const RETRY_SQLSTATE_CODES = new Set([
378
367
  '25P03', // idle_in_transaction_session_timeout, when the kill lands on an in-flight query; an idle kill surfaces as a message match below
379
368
  '40P01', // deadlock_detected: the victim's transaction rolled back, so rerunning it is safe
380
369
  ]);
381
- /**
382
- * Bulk maintenance retries these a bounded number of times. 40001 is kept out of the sets above,
383
- * which feed `dbRetry` and so retry forever; every other path runs READ COMMITTED and never sees it.
384
- */
385
- const SERIALIZATION_SQLSTATE_CODES = new Set([
386
- '40001', // serialization_failure (MVCC conflict)
387
- '40P01', // deadlock_detected
388
- ]);
389
370
  // Node.js transient network error codes (system call level)
390
371
  const RETRY_NODE_ERRNOS = new Set([
391
372
  'ECONNRESET',
@@ -400,6 +381,7 @@ function isPgDatabaseError(e) {
400
381
  // Matched by shape, not instanceof: a user-supplied pool may throw another pg copy's DatabaseError.
401
382
  return !!e && typeof e === 'object' && typeof e.code === 'string' && e.code.length === 5;
402
383
  }
384
+ exports.isPgDatabaseError = isPgDatabaseError;
403
385
  function sqlStateLooksRetryable(sqlstate) {
404
386
  if (!sqlstate)
405
387
  return false;
@@ -449,6 +431,7 @@ function* unwrapErrors(e) {
449
431
  yield cur;
450
432
  }
451
433
  }
434
+ exports.unwrapErrors = unwrapErrors;
452
435
  // "What could possibly go wrong?"
453
436
  function retriablePostgresException(err) {
454
437
  // Dig into AggregateErrors of various types
@@ -481,45 +464,45 @@ exports.retriablePostgresException = retriablePostgresException;
481
464
  function isStatementTimeout(err) {
482
465
  return !!err && typeof err === 'object' && err.code === '57014';
483
466
  }
484
- function isSerializationError(err) {
485
- for (const e of unwrapErrors(err)) {
486
- const anyErr = e;
487
- if (isPgDatabaseError(anyErr) && !!anyErr.code && SERIALIZATION_SQLSTATE_CODES.has(anyErr.code)) {
488
- return true;
489
- }
490
- }
491
- return false;
492
- }
493
- /**
494
- * Re-run a batch that lost a deadlock or serialization race. The database already rolled it
495
- * back, so replaying it is safe.
496
- */
497
- async function retryOnSerializationError(operation) {
498
- const maxAttempts = 10;
499
- const maxBackoff = 2.0;
500
- let backoff = 0.05;
501
- for (let attempt = 1;; attempt++) {
467
+ /** Run `operation` under the {@link dbRetry} policy, for code that cannot take the decorator. */
468
+ async function withDbRetry(operation, options = {}) {
469
+ // Read the defaults per call so the backoff stays tunable after the decorator is applied.
470
+ const maxBackoff = options.maxBackoff ?? utils_1.dbRetryConfig.maxBackoffSec;
471
+ let retries = 0;
472
+ let backoff = options.initialBackoff ?? utils_1.dbRetryConfig.initialBackoffSec;
473
+ const abortSignal = (0, context_1.currentDbRetryAbortSignal)();
474
+ while (true) {
502
475
  try {
503
476
  return await operation();
504
477
  }
505
478
  catch (e) {
506
- if (!isSerializationError(e)) {
507
- throw e;
479
+ if (retriablePostgresException(e)) {
480
+ if (abortSignal?.aborted)
481
+ throw e;
482
+ retries++;
483
+ // Calculate backoff with jitter
484
+ const actualBackoff = backoff * (0.5 + Math.random());
485
+ dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Database connection failed: ${e instanceof Error ? e.message : String(e)}. ` +
486
+ `Retrying in ${actualBackoff.toFixed(2)}s (attempt ${retries})`);
487
+ // Sleep with backoff
488
+ if (abortSignal) {
489
+ await (0, utils_1.interruptibleSleep)(actualBackoff * 1000, abortSignal);
490
+ if (abortSignal.aborted)
491
+ throw e;
492
+ }
493
+ else {
494
+ await (0, utils_1.sleepms)(actualBackoff * 1000); // Convert to milliseconds
495
+ }
496
+ // Increase backoff for next attempt (exponential)
497
+ backoff = Math.min(backoff * 2, maxBackoff);
508
498
  }
509
- const message = e instanceof Error ? e.message : String(e);
510
- if (attempt === maxAttempts) {
511
- dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Garbage collection failed after ${maxAttempts} attempts: ${message}`);
499
+ else {
512
500
  throw e;
513
501
  }
514
- // Jittered backoff, so peers that collided do not collide again
515
- const actualBackoff = backoff * (0.5 + Math.random());
516
- dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Contention or deadlock detected in workflow garbage collection: ${message}. ` +
517
- `Retrying in ${actualBackoff.toFixed(2)}s (attempt ${attempt})`);
518
- await (0, utils_1.sleepms)(actualBackoff * 1000);
519
- backoff = Math.min(backoff * 2, maxBackoff);
520
502
  }
521
503
  }
522
504
  }
505
+ exports.withDbRetry = withDbRetry;
523
506
  /**
524
507
  * If a workflow encounters a database connection issue while performing an operation,
525
508
  * block the workflow and retry the operation until it reconnects and succeeds.
@@ -533,41 +516,7 @@ function dbRetry(options = {}) {
533
516
  return function (target, propertyName, descriptor) {
534
517
  const method = descriptor.value;
535
518
  descriptor.value = async function (...args) {
536
- // Read the defaults per call so the backoff stays tunable after the decorator is applied.
537
- const maxBackoff = options.maxBackoff ?? utils_1.dbRetryConfig.maxBackoffSec;
538
- let retries = 0;
539
- let backoff = options.initialBackoff ?? utils_1.dbRetryConfig.initialBackoffSec;
540
- const abortSignal = (0, context_1.currentDbRetryAbortSignal)();
541
- while (true) {
542
- try {
543
- return await method.apply(this, args);
544
- }
545
- catch (e) {
546
- if (retriablePostgresException(e)) {
547
- if (abortSignal?.aborted)
548
- throw e;
549
- retries++;
550
- // Calculate backoff with jitter
551
- const actualBackoff = backoff * (0.5 + Math.random());
552
- dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Database connection failed: ${e instanceof Error ? e.message : String(e)}. ` +
553
- `Retrying in ${actualBackoff.toFixed(2)}s (attempt ${retries})`);
554
- // Sleep with backoff
555
- if (abortSignal) {
556
- await (0, utils_1.interruptibleSleep)(actualBackoff * 1000, abortSignal);
557
- if (abortSignal.aborted)
558
- throw e;
559
- }
560
- else {
561
- await (0, utils_1.sleepms)(actualBackoff * 1000); // Convert to milliseconds
562
- }
563
- // Increase backoff for next attempt (exponential)
564
- backoff = Math.min(backoff * 2, maxBackoff);
565
- }
566
- else {
567
- throw e;
568
- }
569
- }
570
- }
519
+ return withDbRetry(() => method.apply(this, args), options);
571
520
  };
572
521
  return descriptor;
573
522
  };
@@ -649,11 +598,12 @@ class SystemDatabase {
649
598
  #batchCreatedAtCursors = new Map();
650
599
  // Set by destroy(), so polling waits end instead of running on against a pool that outlives this handle.
651
600
  #destroyed = false;
652
- // Resolved on first use by #cockroach(), since detecting it costs a query.
653
- #isCockroach = undefined;
601
+ get destroyed() {
602
+ return this.#destroyed;
603
+ }
654
604
  // Connections a retention round holds right now. destroy() cuts them, since closing the
655
605
  // pool would otherwise wait on the lock session and on any statement in flight.
656
- #retentionClients = new Set();
606
+ retentionClients = new Set();
657
607
  constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS,
658
608
  // The application this handle acts for; undefined writes unclaimed rows.
659
609
  appName, observabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS, idleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS) {
@@ -713,25 +663,9 @@ class SystemDatabase {
713
663
  this.logger.warn(`Unexpected error on a system database connection: ${err}`);
714
664
  };
715
665
  /** Check out a pool connection guarded for as long as we hold it. See {@link borrowClient}. */
716
- #connect() {
666
+ connect() {
717
667
  return borrowClient(this.pool, this.#onClientError);
718
668
  }
719
- /** Borrow a connection for a retention round, so destroy() can cut it. */
720
- async #borrowRetentionClient() {
721
- if (this.#destroyed) {
722
- throw new Error('System database shutting down');
723
- }
724
- const client = await this.#connect();
725
- this.#retentionClients.add(client);
726
- return client;
727
- }
728
- /** Return a retention connection, unless destroy() already cut it: a second release would throw. */
729
- #releaseRetentionClient(client) {
730
- if (this.#retentionClients.delete(client)) {
731
- // No error argument: a genuinely dead connection is still evicted by the pool's own check.
732
- client.release();
733
- }
734
- }
735
669
  /**
736
670
  * Cap an introspection read with a statement timeout, so one scanning a huge table cannot hold a
737
671
  * snapshot for minutes and stall autovacuum database-wide. Soft-private so tests can assert the cap
@@ -742,7 +676,7 @@ class SystemDatabase {
742
676
  */
743
677
  async observabilityQuery(fn) {
744
678
  const timeoutMs = this.observabilityQueryTimeoutMs;
745
- const client = await this.#connect();
679
+ const client = await this.connect();
746
680
  try {
747
681
  if (timeoutMs === undefined) {
748
682
  return await fn(client);
@@ -771,7 +705,7 @@ class SystemDatabase {
771
705
  * A predicate matching rows owned by these applications plus unclaimed ones, which belong to
772
706
  * every application; unset or empty matches everything. Appends its bind parameter to `params`.
773
707
  */
774
- #appNameFilter(column, value, params) {
708
+ appNameFilter(column, value, params) {
775
709
  // An empty name is no name: it is not a value any application could be configured with.
776
710
  const names = !value ? [] : Array.isArray(value) ? value : [value];
777
711
  if (names.length === 0)
@@ -784,8 +718,8 @@ class SystemDatabase {
784
718
  * this application owns, not to every application's rows. A handle with no application of its
785
719
  * own still matches every one.
786
720
  */
787
- #observabilityFilter(column, value, params) {
788
- return this.#appNameFilter(column, value ?? this.appName, params);
721
+ observabilityFilter(column, value, params) {
722
+ return this.appNameFilter(column, value ?? this.appName, params);
789
723
  }
790
724
  /**
791
725
  * The version name a dequeue treats as latest: this application's own plus unclaimed
@@ -793,7 +727,7 @@ class SystemDatabase {
793
727
  */
794
728
  async #latestApplicationVersionName(client) {
795
729
  const params = [];
796
- const scope = this.#appNameFilter('application_name', this.appName, params);
730
+ const scope = this.appNameFilter('application_name', this.appName, params);
797
731
  const { rows } = await client.query(`SELECT version_name
798
732
  FROM "${this.schemaName}".application_versions
799
733
  WHERE ${scope}
@@ -856,7 +790,7 @@ class SystemDatabase {
856
790
  // A retention round still running is cut here rather than waited for: returned with an
857
791
  // error, each connection is destroyed at once, the idle lock session lets go of the
858
792
  // advisory lock, and the round fails on its next statement instead of holding shutdown.
859
- for (const client of this.#retentionClients) {
793
+ for (const client of this.retentionClients) {
860
794
  // Cover the release() call itself, which tears the connection down and can surface a socket error.
861
795
  client.on('error', () => { });
862
796
  try {
@@ -866,7 +800,7 @@ class SystemDatabase {
866
800
  this.logger.warn(`Error releasing a retention connection: ${String(e)}`);
867
801
  }
868
802
  }
869
- this.#retentionClients.clear();
803
+ this.retentionClients.clear();
870
804
  // We attached nothing to the pool object itself, so there is nothing to unpick; only close one we own.
871
805
  if (!this.customPool) {
872
806
  await this.pool.end();
@@ -881,7 +815,7 @@ class SystemDatabase {
881
815
  return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
882
816
  }
883
817
  async initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy) {
884
- const client = await this.#connect();
818
+ const client = await this.connect();
885
819
  let shouldCommit = false;
886
820
  try {
887
821
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
@@ -907,7 +841,7 @@ class SystemDatabase {
907
841
  }
908
842
  /** Insert a child workflow's row and record it as the parent's step in one transaction, so a crash leaves both or neither. */
909
843
  async initChildWorkflowStatus(initStatus, creatorXid, parentWorkflowID, parentFunctionID, startTimeEpochMs, endTimeEpochMs, reusePolicy = 'return-existing') {
910
- const client = await this.#connect();
844
+ const client = await this.connect();
911
845
  let shouldCommit = false;
912
846
  try {
913
847
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
@@ -1084,7 +1018,7 @@ class SystemDatabase {
1084
1018
  'schedule_name',
1085
1019
  'application_name',
1086
1020
  ];
1087
- const client = await this.#connect();
1021
+ const client = await this.connect();
1088
1022
  try {
1089
1023
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1090
1024
  // Chunk to stay well under the bind-parameter limit.
@@ -1137,7 +1071,7 @@ class SystemDatabase {
1137
1071
  return inserted;
1138
1072
  }
1139
1073
  async recordWorkflowOutput(workflowID, status, ownerXid) {
1140
- const client = await this.#connect();
1074
+ const client = await this.connect();
1141
1075
  try {
1142
1076
  return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
1143
1077
  }
@@ -1146,7 +1080,7 @@ class SystemDatabase {
1146
1080
  }
1147
1081
  }
1148
1082
  async recordWorkflowError(workflowID, status, ownerXid) {
1149
- const client = await this.#connect();
1083
+ const client = await this.connect();
1150
1084
  try {
1151
1085
  return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
1152
1086
  }
@@ -1194,22 +1128,11 @@ class SystemDatabase {
1194
1128
  }
1195
1129
  }
1196
1130
  }
1197
- async getPendingWorkflows(executorID, appVersion) {
1198
- const params = [workflow_1.StatusString.PENDING, executorID, appVersion];
1199
- // executor_id defaults to "local", so it collides across applications.
1200
- const scope = this.#appNameFilter('application_name', this.appName, params);
1201
- const getWorkflows = await this.pool.query(`SELECT workflow_uuid
1202
- FROM "${this.schemaName}".workflow_status
1203
- WHERE status=$1 AND executor_id=$2 AND application_version=$3 AND ${scope}`, params);
1204
- return getWorkflows.rows.map((i) => ({
1205
- workflowUUID: i.workflow_uuid,
1206
- }));
1207
- }
1208
1131
  // Recovery re-enqueues rather than executing directly so the queue's atomic dequeue admits exactly one runner, and the executor ID predicate rejects sweeps for rows a live executor has already claimed.
1209
1132
  async reenqueueWorkflowsForRecovery(executorID, appVersion, recoveryQueueName) {
1210
1133
  const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
1211
1134
  // executor_id defaults to "local", so it collides across applications.
1212
- const scope = this.#appNameFilter('application_name', this.appName, params);
1135
+ const scope = this.appNameFilter('application_name', this.appName, params);
1213
1136
  const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1214
1137
  SET started_at_epoch_ms = NULL,
1215
1138
  status = $1,
@@ -1230,7 +1153,7 @@ class SystemDatabase {
1230
1153
  return status ? JSON.stringify(status) : null;
1231
1154
  };
1232
1155
  if (callerID && callerFN) {
1233
- const client = await this.#connect();
1156
+ const client = await this.connect();
1234
1157
  try {
1235
1158
  // Check if the operation has been done before for OAOO (only do this inside a workflow).
1236
1159
  const json = await this.#inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, funcGetStatus));
@@ -1266,7 +1189,7 @@ class SystemDatabase {
1266
1189
  }
1267
1190
  // Only used in tests
1268
1191
  async setWorkflowStatus(workflowID, status, resetRecoveryAttempts, internalOptions) {
1269
- const client = await this.#connect();
1192
+ const client = await this.connect();
1270
1193
  try {
1271
1194
  await this.updateWorkflowStatus(client, workflowID, status, {
1272
1195
  update: {
@@ -1284,7 +1207,7 @@ class SystemDatabase {
1284
1207
  }
1285
1208
  // ==================== Step Results ====================
1286
1209
  async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
1287
- const client = await this.#connect();
1210
+ const client = await this.connect();
1288
1211
  try {
1289
1212
  return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
1290
1213
  }
@@ -1307,7 +1230,7 @@ class SystemDatabase {
1307
1230
  return rows;
1308
1231
  }
1309
1232
  async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
1310
- const client = await this.#connect();
1233
+ const client = await this.connect();
1311
1234
  try {
1312
1235
  await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options));
1313
1236
  }
@@ -1317,7 +1240,7 @@ class SystemDatabase {
1317
1240
  }
1318
1241
  }
1319
1242
  async runTransactionalStep(workflowID, functionID, functionName, callback) {
1320
- const client = await this.#connect();
1243
+ const client = await this.connect();
1321
1244
  try {
1322
1245
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1323
1246
  const existing = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
@@ -1363,7 +1286,7 @@ class SystemDatabase {
1363
1286
  }
1364
1287
  // Nondeprecated - skip matching entry, unpatched if nonmatching entry,
1365
1288
  // If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
1366
- const client = await this.#connect();
1289
+ const client = await this.connect();
1367
1290
  try {
1368
1291
  return await this.#inTransaction(client, async () => {
1369
1292
  const checkpointName = await readCheckpointName(client);
@@ -1410,7 +1333,7 @@ class SystemDatabase {
1410
1333
  await this.#checkIfCanceled(this.pool, workflowID);
1411
1334
  }
1412
1335
  async resumeWorkflows(workflowIDs, queueName) {
1413
- const client = await this.#connect();
1336
+ const client = await this.connect();
1414
1337
  try {
1415
1338
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1416
1339
  // Check existence separately: a zero-row update also means "already complete", a legal no-op.
@@ -1474,7 +1397,7 @@ class SystemDatabase {
1474
1397
  return await this.debounceDelayedWorkflowStandalone(params);
1475
1398
  }
1476
1399
  async debounceDelayedWorkflowStandalone(params) {
1477
- const client = await this.#connect();
1400
+ const client = await this.connect();
1478
1401
  try {
1479
1402
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1480
1403
  const result = await this.#debounceDelayedWorkflowInternal(client, params);
@@ -1502,7 +1425,7 @@ class SystemDatabase {
1502
1425
  params.applicationName ?? null,
1503
1426
  ];
1504
1427
  // Never extend a workflow the target application doesn't own; falls through to the holder below.
1505
- const ownScope = this.#appNameFilter('application_name', params.applicationName, updateParams);
1428
+ const ownScope = this.appNameFilter('application_name', params.applicationName, updateParams);
1506
1429
  const updated = await client.query(`UPDATE "${this.schemaName}".workflow_status
1507
1430
  SET delay_until_epoch_ms = CASE
1508
1431
  WHEN debounce_deadline_epoch_ms IS NOT NULL AND debounce_deadline_epoch_ms < $1
@@ -1584,7 +1507,7 @@ class SystemDatabase {
1584
1507
  allIds.push(...(await this.getWorkflowChildren(wfid)));
1585
1508
  }
1586
1509
  }
1587
- const client = await this.#connect();
1510
+ const client = await this.connect();
1588
1511
  try {
1589
1512
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1590
1513
  // The payload tables carry no foreign key, so the status delete does not cascade
@@ -1630,7 +1553,7 @@ class SystemDatabase {
1630
1553
  throw new error_1.DBOSError(`startStep must be >= 0, got ${startStep}`);
1631
1554
  }
1632
1555
  const schema = this.schemaName;
1633
- const client = await this.#connect();
1556
+ const client = await this.connect();
1634
1557
  try {
1635
1558
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1636
1559
  const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE workflow_uuid = $1`, [workflowID]);
@@ -1712,62 +1635,6 @@ class SystemDatabase {
1712
1635
  const result = await this.bulkForkWorkflows([workflowID], [newWorkflowID], [startStep], options);
1713
1636
  return result[0];
1714
1637
  }
1715
- async forkFromFailure(workflowIDs, options = {}) {
1716
- const modes = [
1717
- options.fromLastFailure ?? false,
1718
- options.fromLastStep ?? false,
1719
- options.fromStep !== undefined,
1720
- options.fromStepName !== undefined,
1721
- ].filter(Boolean).length;
1722
- if (modes !== 1) {
1723
- throw new Error('Exactly one of fromLastFailure, fromLastStep, fromStep, or fromStepName must be specified');
1724
- }
1725
- let startSteps;
1726
- if (options.fromStep !== undefined) {
1727
- startSteps = Array(workflowIDs.length).fill(options.fromStep);
1728
- }
1729
- else {
1730
- let query;
1731
- const params = [workflowIDs];
1732
- if (options.fromLastFailure) {
1733
- query = `SELECT workflow_uuid,
1734
- COALESCE(
1735
- MAX(function_id) FILTER (WHERE error IS NOT NULL),
1736
- MAX(function_id)
1737
- ) AS start_step
1738
- FROM "${this.schemaName}".operation_outputs
1739
- WHERE workflow_uuid = ANY($1)
1740
- GROUP BY workflow_uuid`;
1741
- }
1742
- else if (options.fromLastStep) {
1743
- query = `SELECT workflow_uuid, MAX(function_id) AS start_step
1744
- FROM "${this.schemaName}".operation_outputs
1745
- WHERE workflow_uuid = ANY($1)
1746
- GROUP BY workflow_uuid`;
1747
- }
1748
- else {
1749
- // fromStepName
1750
- query = `SELECT workflow_uuid, MAX(function_id) AS start_step
1751
- FROM "${this.schemaName}".operation_outputs
1752
- WHERE workflow_uuid = ANY($1) AND function_name = $2
1753
- GROUP BY workflow_uuid`;
1754
- params.push(options.fromStepName);
1755
- }
1756
- const result = await this.pool.query(query, params);
1757
- const startStepByID = new Map(result.rows.map((r) => [r.workflow_uuid, Number(r.start_step)]));
1758
- if (options.fromStepName !== undefined) {
1759
- for (const wid of workflowIDs) {
1760
- if (!startStepByID.has(wid)) {
1761
- throw new Error(`Workflow ${wid} has no step named '${options.fromStepName}'`);
1762
- }
1763
- }
1764
- }
1765
- // A workflow with no recorded steps has nothing to resume from, so restart it from the beginning.
1766
- startSteps = workflowIDs.map((wid) => startStepByID.get(wid) ?? 0);
1767
- }
1768
- const forkedIDs = workflowIDs.map(() => (0, crypto_1.randomUUID)());
1769
- return this.bulkForkWorkflows(workflowIDs, forkedIDs, startSteps, options);
1770
- }
1771
1638
  async bulkForkWorkflows(originalWorkflowIDs, forkedWorkflowIDs, startSteps, options = {}) {
1772
1639
  if (originalWorkflowIDs.length === 0) {
1773
1640
  return [];
@@ -1775,7 +1642,7 @@ class SystemDatabase {
1775
1642
  if (originalWorkflowIDs.length !== forkedWorkflowIDs.length || originalWorkflowIDs.length !== startSteps.length) {
1776
1643
  throw new Error('originalWorkflowIDs, forkedWorkflowIDs, and startSteps must have the same length');
1777
1644
  }
1778
- const client = await this.#connect();
1645
+ const client = await this.connect();
1779
1646
  try {
1780
1647
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1781
1648
  // Fetch the status of all original workflows inside the transaction.
@@ -1927,188 +1794,6 @@ class SystemDatabase {
1927
1794
  client.release();
1928
1795
  }
1929
1796
  }
1930
- async exportWorkflow(workflowID, exportChildren = false) {
1931
- const workflowIDs = [workflowID];
1932
- if (exportChildren) {
1933
- workflowIDs.push(...(await this.getWorkflowChildren(workflowID)));
1934
- }
1935
- const exportedWorkflows = [];
1936
- const client = await this.#connect();
1937
- try {
1938
- for (const wfID of workflowIDs) {
1939
- // Export workflow_status
1940
- const statusResult = await client.query(
1941
- // creator_xid and owner_xid are intentionally omitted: they are transient
1942
- // tokens, not logical workflow state, and a source database's
1943
- // tokens are meaningless in the target.
1944
- `SELECT
1945
- ws.workflow_uuid, ws.status, ws.name, ws.authenticated_user, ws.assumed_role,
1946
- ws.authenticated_roles,
1947
- COALESCE(wo.output, ws.output) AS output, COALESCE(wo.error, ws.error) AS error,
1948
- ws.executor_id,
1949
- ws.created_at, ws.updated_at, ws.application_version, ws.application_id,
1950
- ws.class_name, ws.config_name, ws.recovery_attempts, ws.queue_name,
1951
- ws.workflow_timeout_ms, ws.workflow_deadline_epoch_ms, ws.started_at_epoch_ms,
1952
- ws.deduplication_id, COALESCE(wi.inputs, ws.inputs) AS inputs,
1953
- ws.priority, ws.queue_partition_key, ws.forked_from,
1954
- ws.parent_workflow_id, ws.serialization, ws.delay_until_epoch_ms,
1955
- ws.was_forked_from, ws.rate_limited, ws.completed_at, ws.attributes, ws.schedule_name,
1956
- ws.debounce_deadline_epoch_ms, ws.is_debounced, ws.application_name
1957
- FROM "${this.schemaName}".workflow_status ws
1958
- LEFT JOIN "${this.schemaName}".workflow_input wi ON wi.workflow_uuid = ws.workflow_uuid
1959
- LEFT JOIN "${this.schemaName}".workflow_output wo ON wo.workflow_uuid = ws.workflow_uuid
1960
- WHERE ws.workflow_uuid = $1`, [wfID]);
1961
- if (statusResult.rows.length === 0) {
1962
- throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${wfID} does not exist`);
1963
- }
1964
- const workflowStatus = statusResult.rows[0];
1965
- // Export operation_outputs
1966
- const outputsResult = await client.query(`SELECT
1967
- workflow_uuid, function_id, function_name, output, error,
1968
- child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
1969
- serialization, application_name
1970
- FROM "${this.schemaName}".operation_outputs
1971
- WHERE workflow_uuid = $1`, [wfID]);
1972
- // Export workflow_events
1973
- const eventsResult = await client.query(`SELECT workflow_uuid, key, value, serialization
1974
- FROM "${this.schemaName}".workflow_events
1975
- WHERE workflow_uuid = $1`, [wfID]);
1976
- // Export workflow_events_history
1977
- const historyResult = await client.query(`SELECT workflow_uuid, function_id, key, value, serialization
1978
- FROM "${this.schemaName}".workflow_events_history
1979
- WHERE workflow_uuid = $1`, [wfID]);
1980
- // Export streams
1981
- const streamsResult = await client.query(`SELECT workflow_uuid, key, value, "offset", function_id, serialization
1982
- FROM "${this.schemaName}".streams
1983
- WHERE workflow_uuid = $1`, [wfID]);
1984
- exportedWorkflows.push({
1985
- workflow_status: workflowStatus,
1986
- operation_outputs: outputsResult.rows,
1987
- workflow_events: eventsResult.rows,
1988
- workflow_events_history: historyResult.rows,
1989
- streams: streamsResult.rows,
1990
- });
1991
- }
1992
- }
1993
- finally {
1994
- client.release();
1995
- }
1996
- return exportedWorkflows;
1997
- }
1998
- async importWorkflow(workflows) {
1999
- const client = await this.#connect();
2000
- try {
2001
- await client.query('BEGIN');
2002
- for (const workflow of workflows) {
2003
- const status = workflow.workflow_status;
2004
- // Import workflow_status
2005
- await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
2006
- workflow_uuid, status, name, authenticated_user, assumed_role,
2007
- authenticated_roles, output, error, executor_id,
2008
- created_at, updated_at, application_version, application_id,
2009
- class_name, config_name, recovery_attempts, queue_name,
2010
- workflow_timeout_ms, workflow_deadline_epoch_ms, started_at_epoch_ms,
2011
- deduplication_id, inputs, priority, queue_partition_key, forked_from,
2012
- parent_workflow_id, serialization, delay_until_epoch_ms,
2013
- was_forked_from, rate_limited, completed_at, attributes, schedule_name,
2014
- debounce_deadline_epoch_ms, is_debounced, application_name
2015
- ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)`, [
2016
- status.workflow_uuid,
2017
- status.status,
2018
- status.name,
2019
- status.authenticated_user,
2020
- status.assumed_role,
2021
- status.authenticated_roles,
2022
- // Legacy columns: the payloads live in their own tables.
2023
- null,
2024
- null,
2025
- status.executor_id,
2026
- status.created_at,
2027
- status.updated_at,
2028
- status.application_version,
2029
- status.application_id,
2030
- status.class_name,
2031
- status.config_name,
2032
- status.recovery_attempts,
2033
- status.queue_name,
2034
- status.workflow_timeout_ms,
2035
- status.workflow_deadline_epoch_ms,
2036
- status.started_at_epoch_ms,
2037
- status.deduplication_id,
2038
- null,
2039
- status.priority,
2040
- status.queue_partition_key,
2041
- status.forked_from,
2042
- status.parent_workflow_id,
2043
- status.serialization,
2044
- status.delay_until_epoch_ms ?? null,
2045
- // NOT NULL columns: fall back to FALSE for payloads exported before
2046
- // these fields were included.
2047
- status.was_forked_from ?? false,
2048
- status.rate_limited ?? false,
2049
- status.completed_at ?? null,
2050
- status.attributes ? JSON.stringify(status.attributes) : null,
2051
- status.schedule_name ?? null,
2052
- status.debounce_deadline_epoch_ms ?? null,
2053
- status.is_debounced ?? false,
2054
- status.application_name ?? null,
2055
- ]);
2056
- // Retention starts at import: the original timestamps are long past the cutoff
2057
- // and would be collected immediately.
2058
- await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
2059
- VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [status.workflow_uuid, status.inputs]);
2060
- if (status.output !== null || status.error !== null) {
2061
- await client.query(`INSERT INTO "${this.schemaName}".workflow_output (workflow_uuid, output, error, retention_timestamp)
2062
- VALUES ($1, $2, $3, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [status.workflow_uuid, status.output, status.error]);
2063
- }
2064
- // Import operation_outputs
2065
- for (const output of workflow.operation_outputs) {
2066
- await client.query(`INSERT INTO "${this.schemaName}".operation_outputs (
2067
- workflow_uuid, function_id, function_name, output, error,
2068
- child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
2069
- serialization, application_name, retention_timestamp
2070
- ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [
2071
- output.workflow_uuid,
2072
- output.function_id,
2073
- output.function_name,
2074
- output.output,
2075
- output.error,
2076
- output.child_workflow_id,
2077
- output.started_at_epoch_ms,
2078
- output.completed_at_epoch_ms,
2079
- output.serialization,
2080
- output.application_name ?? null,
2081
- ]);
2082
- }
2083
- // Import workflow_events
2084
- for (const event of workflow.workflow_events) {
2085
- await client.query(`INSERT INTO "${this.schemaName}".workflow_events (
2086
- workflow_uuid, key, value, serialization
2087
- ) VALUES ($1, $2, $3, $4)`, [event.workflow_uuid, event.key, event.value, event.serialization]);
2088
- }
2089
- // Import workflow_events_history
2090
- for (const history of workflow.workflow_events_history) {
2091
- await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (
2092
- workflow_uuid, function_id, key, value, serialization
2093
- ) VALUES ($1, $2, $3, $4, $5)`, [history.workflow_uuid, history.function_id, history.key, history.value, history.serialization]);
2094
- }
2095
- // Import streams
2096
- for (const stream of workflow.streams) {
2097
- await client.query(`INSERT INTO "${this.schemaName}".streams (
2098
- workflow_uuid, key, value, "offset", function_id, serialization
2099
- ) VALUES ($1, $2, $3, $4, $5, $6)`, [stream.workflow_uuid, stream.key, stream.value, stream.offset, stream.function_id, stream.serialization]);
2100
- }
2101
- }
2102
- await client.query('COMMIT');
2103
- }
2104
- catch (error) {
2105
- await client.query('ROLLBACK');
2106
- throw error;
2107
- }
2108
- finally {
2109
- client.release();
2110
- }
2111
- }
2112
1797
  // ==================== Awaiting Workflows ====================
2113
1798
  /** Register one execution; the returned release is idempotent, so it can run both before a park and once the run settles. */
2114
1799
  registerRunningWorkflow(workflowID, queueName, queuePartitionKey) {
@@ -2396,7 +2081,7 @@ class SystemDatabase {
2396
2081
  async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey) {
2397
2082
  topic = topic ?? this.nullTopic;
2398
2083
  const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
2399
- const client = await this.#connect();
2084
+ const client = await this.connect();
2400
2085
  try {
2401
2086
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
2402
2087
  await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
@@ -2432,7 +2117,7 @@ class SystemDatabase {
2432
2117
  /** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
2433
2118
  async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
2434
2119
  const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
2435
- const client = await this.#connect();
2120
+ const client = await this.connect();
2436
2121
  try {
2437
2122
  await this.#inTransaction(client, async () => {
2438
2123
  await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
@@ -2520,7 +2205,7 @@ class SystemDatabase {
2520
2205
  // Transactionally consume and return the message if it's in the DB, otherwise return null.
2521
2206
  let message = null;
2522
2207
  let serialization = null;
2523
- const client = await this.#connect();
2208
+ const client = await this.connect();
2524
2209
  try {
2525
2210
  await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
2526
2211
  const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
@@ -2560,7 +2245,7 @@ class SystemDatabase {
2560
2245
  }
2561
2246
  // ==================== Events ====================
2562
2247
  async setEvent(workflowID, functionID, key, message, serialization) {
2563
- const client = await this.#connect();
2248
+ const client = await this.connect();
2564
2249
  try {
2565
2250
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
2566
2251
  // Only a real write (not a replay) should wake readers.
@@ -2668,7 +2353,7 @@ class SystemDatabase {
2668
2353
  // ==================== Streams ====================
2669
2354
  async writeStreamFromStep(workflowID, functionID, key, serializedValue, serialization) {
2670
2355
  const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
2671
- const client = await this.#connect();
2356
+ const client = await this.connect();
2672
2357
  try {
2673
2358
  while (true) {
2674
2359
  try {
@@ -2705,7 +2390,7 @@ class SystemDatabase {
2705
2390
  }
2706
2391
  }
2707
2392
  async writeStreamFromWorkflow(workflowID, functionID, key, serializedValue, serialization, functionName) {
2708
- const client = await this.#connect();
2393
+ const client = await this.connect();
2709
2394
  try {
2710
2395
  while (true) {
2711
2396
  // Only a real insert (not a replay) should wake readers.
@@ -2839,7 +2524,7 @@ class SystemDatabase {
2839
2524
  }
2840
2525
  try {
2841
2526
  // One statement: one round trip, one async-notify queue-lock acquisition; unnest emits one notification per payload.
2842
- const client = await this.#connect();
2527
+ const client = await this.connect();
2843
2528
  try {
2844
2529
  await client.query(`SELECT pg_notify($1, p) FROM unnest($2::text[]) AS p`, [channel, Array.from(batch)]);
2845
2530
  }
@@ -2854,49 +2539,6 @@ class SystemDatabase {
2854
2539
  }
2855
2540
  }
2856
2541
  // ==================== Observability: Workflow Communications ====================
2857
- async getAllEvents(workflowID) {
2858
- const { rows } = await this.observabilityQuery((client) => client.query(`SELECT key, value, serialization FROM "${this.schemaName}".workflow_events
2859
- WHERE workflow_uuid = $1`, [workflowID]));
2860
- const events = {};
2861
- for (const row of rows) {
2862
- events[row.key] = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
2863
- }
2864
- return events;
2865
- }
2866
- async getAllNotifications(workflowID) {
2867
- const { rows } = await this.observabilityQuery((client) => client.query(`SELECT topic, message, serialization, created_at_epoch_ms, consumed
2868
- FROM "${this.schemaName}".notifications
2869
- WHERE destination_uuid = $1
2870
- ORDER BY created_at_epoch_ms`, [workflowID]));
2871
- return await Promise.all(rows.map(async (row) => ({
2872
- topic: row.topic === this.nullTopic ? null : row.topic,
2873
- message: await (0, serialization_1.safeParse)(this.serializer, row.message, row.serialization),
2874
- createdAtEpochMs: Number(row.created_at_epoch_ms),
2875
- consumed: row.consumed,
2876
- })));
2877
- }
2878
- async getAllStreamEntries(workflowID) {
2879
- const { rows } = await this.observabilityQuery((client) => client.query(`SELECT key, value, serialization FROM "${this.schemaName}".streams
2880
- WHERE workflow_uuid = $1
2881
- ORDER BY key, "offset"`, [workflowID]));
2882
- const streams = {};
2883
- const closed = new Set();
2884
- for (const row of rows) {
2885
- if (closed.has(row.key)) {
2886
- continue;
2887
- }
2888
- // safeParse yields the raw string for the legacy unserialized marker, which does not parse.
2889
- const value = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
2890
- if (isStreamClosedSentinel(value)) {
2891
- // End the stream where readStream does, so the two never disagree.
2892
- closed.add(row.key);
2893
- streams[row.key] ??= [];
2894
- continue;
2895
- }
2896
- (streams[row.key] ??= []).push(value);
2897
- }
2898
- return streams;
2899
- }
2900
2542
  // ==================== Queues ====================
2901
2543
  async transitionDelayedWorkflows() {
2902
2544
  // Transition workflows from DELAYED to ENQUEUED when their delay has expired.
@@ -2904,7 +2546,7 @@ class SystemDatabase {
2904
2546
  // debounce key held only while DELAYED, so a later same-key debounce starts a fresh workflow.
2905
2547
  const params = [workflow_1.StatusString.ENQUEUED, Date.now(), workflow_1.StatusString.DELAYED];
2906
2548
  // Only what this application would dequeue: a peer's debounce key is not ours to clear.
2907
- const scope = this.#appNameFilter('application_name', this.appName, params);
2549
+ const scope = this.appNameFilter('application_name', this.appName, params);
2908
2550
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
2909
2551
  SET status = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
2910
2552
  deduplication_id = CASE WHEN is_debounced THEN NULL ELSE deduplication_id END
@@ -2913,7 +2555,7 @@ class SystemDatabase {
2913
2555
  /** Cancel up to `limit` of this application's active workflows whose deadline has passed, returning their IDs. */
2914
2556
  async cancelTimedOutWorkflows(limit) {
2915
2557
  const params = [workflow_1.StatusString.CANCELLED, Date.now(), limit];
2916
- const scope = this.#appNameFilter('application_name', this.appName, params);
2558
+ const scope = this.appNameFilter('application_name', this.appName, params);
2917
2559
  // Literal statuses let the planner match idx_workflow_status_deadline under any plan mode.
2918
2560
  // SKIP LOCKED leaves a row a dequeue or a peer's sweep holds for the next sweep.
2919
2561
  const { rows } = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
@@ -2944,7 +2586,7 @@ class SystemDatabase {
2944
2586
  // Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v3, so cost scales with the number of partitions rather than the backlog depth.
2945
2587
  const params = [queueName, workflow_1.StatusString.ENQUEUED];
2946
2588
  // Only partitions this application can actually dequeue from.
2947
- const scope = this.#appNameFilter('application_name', this.appName, params);
2589
+ const scope = this.appNameFilter('application_name', this.appName, params);
2948
2590
  const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
2949
2591
  (SELECT MIN(queue_partition_key) AS pk
2950
2592
  FROM "${this.schemaName}".workflow_status
@@ -2971,7 +2613,7 @@ class SystemDatabase {
2971
2613
  queue.partitionRateLimit !== undefined;
2972
2614
  // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
2973
2615
  const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
2974
- const client = await this.#connect();
2616
+ const client = await this.connect();
2975
2617
  try {
2976
2618
  // Default to READ COMMITTED except with a budget shared across executors
2977
2619
  if (hasSharedBudget) {
@@ -2984,7 +2626,7 @@ class SystemDatabase {
2984
2626
  const rateLimitRemaining = async (rateLimit, partitionScoped) => {
2985
2627
  const params = [queue.name, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED, rateLimit.periodSec * 1000];
2986
2628
  // Count only what this application would dequeue, matching the select below.
2987
- const scope = this.#appNameFilter('application_name', this.appName, params);
2629
+ const scope = this.appNameFilter('application_name', this.appName, params);
2988
2630
  const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
2989
2631
  const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
2990
2632
  WHERE queue_name = $1
@@ -3003,7 +2645,7 @@ class SystemDatabase {
3003
2645
  */
3004
2646
  const pendingCount = async (partitionScoped) => {
3005
2647
  const params = [queue.name, workflow_1.StatusString.PENDING];
3006
- const scope = this.#appNameFilter('application_name', this.appName, params);
2648
+ const scope = this.appNameFilter('application_name', this.appName, params);
3007
2649
  const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
3008
2650
  const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
3009
2651
  WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
@@ -3064,7 +2706,7 @@ class SystemDatabase {
3064
2706
  const lockMode = hasSharedBudget ? 'FOR UPDATE NOWAIT' : 'FOR UPDATE SKIP LOCKED';
3065
2707
  const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
3066
2708
  const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
3067
- const selectScope = this.#appNameFilter('application_name', this.appName, selectParams);
2709
+ const selectScope = this.appNameFilter('application_name', this.appName, selectParams);
3068
2710
  const selectQuery = `
3069
2711
  SELECT workflow_uuid
3070
2712
  FROM "${this.schemaName}".workflow_status
@@ -3097,7 +2739,7 @@ class SystemDatabase {
3097
2739
  ownerXid,
3098
2740
  ];
3099
2741
  // Re-check ownership alongside status, as the partitioned claim guard does.
3100
- const claimScope = this.#appNameFilter('application_name', this.appName, updateParams);
2742
+ const claimScope = this.appNameFilter('application_name', this.appName, updateParams);
3101
2743
  // RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
3102
2744
  const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
3103
2745
  SET status = $1,
@@ -3147,7 +2789,7 @@ class SystemDatabase {
3147
2789
  throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
3148
2790
  }
3149
2791
  // partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
3150
- const client = await this.#connect();
2792
+ const client = await this.connect();
3151
2793
  try {
3152
2794
  await client.query('BEGIN');
3153
2795
  const latestVersion = await this.#latestApplicationVersionName(client);
@@ -3166,7 +2808,7 @@ class SystemDatabase {
3166
2808
  workflow_1.StatusString.PENDING,
3167
2809
  sweepLimit,
3168
2810
  ];
3169
- const candidateScope = this.#appNameFilter('application_name', this.appName, candidateParams);
2811
+ const candidateScope = this.appNameFilter('application_name', this.appName, candidateParams);
3170
2812
  // Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
3171
2813
  const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
3172
2814
  (SELECT MIN(queue_partition_key) AS pk
@@ -3223,7 +2865,7 @@ class SystemDatabase {
3223
2865
  AND ${versionClause(version)}
3224
2866
  AND ${scope}`;
3225
2867
  const lockedParams = [candidateIDs, workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
3226
- const lockedScope = this.#appNameFilter('application_name', this.appName, lockedParams);
2868
+ const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
3227
2869
  // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
3228
2870
  const lockedResult = await client.query(`SELECT workflow_uuid
3229
2871
  FROM "${this.schemaName}".workflow_status
@@ -3247,7 +2889,7 @@ class SystemDatabase {
3247
2889
  this.appName ?? null,
3248
2890
  ownerXid,
3249
2891
  ];
3250
- const flipScope = this.#appNameFilter('application_name', this.appName, flipParams);
2892
+ const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
3251
2893
  // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
3252
2894
  const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
3253
2895
  SET status = $1,
@@ -3373,8 +3015,8 @@ class SystemDatabase {
3373
3015
  // Falsy, not just undefined: a Conductor request carries an omitted list as JSON null.
3374
3016
  const idKeyed = (input.workflowIDs?.length ?? 0) > 0;
3375
3017
  whereClauses.push(idKeyed
3376
- ? this.#appNameFilter('application_name', input.applicationName, params)
3377
- : this.#observabilityFilter('application_name', input.applicationName, params));
3018
+ ? this.appNameFilter('application_name', input.applicationName, params)
3019
+ : this.observabilityFilter('application_name', input.applicationName, params));
3378
3020
  paramCounter = params.length + 1;
3379
3021
  if (input.workflow_id_prefix) {
3380
3022
  if (Array.isArray(input.workflow_id_prefix)) {
@@ -3487,634 +3129,6 @@ class SystemDatabase {
3487
3129
  : await this.observabilityQuery((client) => client.query(query, params));
3488
3130
  return result.rows.map(mapWorkflowStatus);
3489
3131
  }
3490
- async getWorkflowAggregates(input) {
3491
- if (input.timeBucketSizeMs !== undefined && input.timeBucketSizeMs <= 0) {
3492
- throw new Error('time_bucket_size_ms must be > 0');
3493
- }
3494
- const groupByFlags = [
3495
- ['status', input.groupByStatus ?? false, 'status'],
3496
- ['name', input.groupByName ?? false, 'name'],
3497
- ['queue_name', input.groupByQueueName ?? false, 'queue_name'],
3498
- ['executor_id', input.groupByExecutorId ?? false, 'executor_id'],
3499
- ['application_version', input.groupByApplicationVersion ?? false, 'application_version'],
3500
- ['application_name', input.groupByApplicationName ?? false, 'application_name'],
3501
- ];
3502
- const groupNames = [];
3503
- const groupColumns = [];
3504
- const groupSelectColumns = [];
3505
- for (const [colName, enabled, col] of groupByFlags) {
3506
- if (enabled) {
3507
- groupNames.push(colName);
3508
- groupColumns.push(col);
3509
- groupSelectColumns.push(col);
3510
- }
3511
- }
3512
- const params = [];
3513
- if (input.timeBucketSizeMs !== undefined) {
3514
- // Bucket on created_at — the indexed wall-clock timestamp on workflow_status.
3515
- // One placeholder shared by SELECT and GROUP BY, so Postgres sees the two expressions as identical.
3516
- params.push(input.timeBucketSizeMs);
3517
- const bucket = `$${params.length}::bigint`;
3518
- const bucketExpr = `(CAST(FLOOR(created_at / ${bucket}) AS BIGINT) * ${bucket})`;
3519
- groupNames.push('time_bucket');
3520
- groupColumns.push(bucketExpr);
3521
- groupSelectColumns.push(`${bucketExpr} AS time_bucket`);
3522
- }
3523
- if (groupColumns.length === 0) {
3524
- throw new Error('At least one group_by flag must be set to True');
3525
- }
3526
- // Build select columns from boolean flags. MAX ignores NULLs, so rows
3527
- // missing started_at_epoch_ms or completed_at naturally drop out of the
3528
- // latency maxes.
3529
- const selectFlags = [
3530
- ['count', input.selectCount ?? false, 'COUNT(*)'],
3531
- ['min_created_at', input.selectMinCreatedAt ?? false, 'MIN(created_at)'],
3532
- ['max_queue_wait_ms', input.selectMaxQueueWaitMs ?? false, 'MAX(started_at_epoch_ms - created_at)'],
3533
- ['max_total_latency_ms', input.selectMaxTotalLatencyMs ?? false, 'MAX(completed_at - created_at)'],
3534
- ];
3535
- const selectNames = [];
3536
- const selectColumns = [];
3537
- for (const [name, enabled, expr] of selectFlags) {
3538
- if (enabled) {
3539
- selectNames.push(name);
3540
- selectColumns.push(`${expr} AS ${name}`);
3541
- }
3542
- }
3543
- if (selectColumns.length === 0) {
3544
- throw new Error('At least one select_ flag must be set to True');
3545
- }
3546
- const whereClauses = [];
3547
- let paramIdx = params.length + 1;
3548
- const addFilter = (column, values) => {
3549
- if (!values || values.length === 0)
3550
- return;
3551
- const placeholders = values.map((_, i) => `$${paramIdx + i}`).join(', ');
3552
- whereClauses.push(`${column} IN (${placeholders})`);
3553
- params.push(...values);
3554
- paramIdx += values.length;
3555
- };
3556
- addFilter('status', input.status);
3557
- addFilter('name', input.name);
3558
- addFilter('application_version', input.appVersion);
3559
- addFilter('executor_id', input.executorId);
3560
- addFilter('queue_name', input.queueName);
3561
- if (input.workflowIdPrefix && input.workflowIdPrefix.length > 0) {
3562
- const likeClauses = input.workflowIdPrefix.map((p) => {
3563
- params.push(`${p}%`);
3564
- return `workflow_uuid LIKE $${paramIdx++}`;
3565
- });
3566
- whereClauses.push(`(${likeClauses.join(' OR ')})`);
3567
- }
3568
- addFilter('workflow_uuid', input.workflowIDs);
3569
- addFilter('authenticated_user', input.authenticatedUser);
3570
- addFilter('forked_from', input.forkedFrom);
3571
- addFilter('parent_workflow_id', input.parentWorkflowID);
3572
- addFilter('schedule_name', input.scheduleName);
3573
- // Unset scopes to this application, as on every other observability query.
3574
- whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
3575
- paramIdx = params.length + 1;
3576
- if (input.wasForkedFrom !== undefined) {
3577
- whereClauses.push(`was_forked_from = $${paramIdx}`);
3578
- params.push(input.wasForkedFrom);
3579
- paramIdx++;
3580
- }
3581
- // Matches the forks themselves, as opposed to wasForkedFrom, which matches the workflows they were forked from.
3582
- if (input.isFork !== undefined) {
3583
- whereClauses.push(input.isFork ? `forked_from IS NOT NULL` : `forked_from IS NULL`);
3584
- }
3585
- if (input.hasParent !== undefined) {
3586
- whereClauses.push(input.hasParent ? `parent_workflow_id IS NOT NULL` : `parent_workflow_id IS NULL`);
3587
- }
3588
- // Match workflows whose attributes JSONB contains all the given key-value pairs.
3589
- if (input.attributes && Object.keys(input.attributes).length > 0) {
3590
- whereClauses.push(`attributes @> $${paramIdx}::jsonb`);
3591
- params.push(JSON.stringify(input.attributes));
3592
- paramIdx++;
3593
- }
3594
- if (input.startTime) {
3595
- whereClauses.push(`created_at >= $${paramIdx}`);
3596
- params.push(new Date(input.startTime).getTime());
3597
- paramIdx++;
3598
- }
3599
- if (input.endTime) {
3600
- whereClauses.push(`created_at <= $${paramIdx}`);
3601
- params.push(new Date(input.endTime).getTime());
3602
- paramIdx++;
3603
- }
3604
- if (input.completedAfter) {
3605
- whereClauses.push(`completed_at >= $${paramIdx}`);
3606
- params.push(new Date(input.completedAfter).getTime());
3607
- paramIdx++;
3608
- }
3609
- if (input.completedBefore) {
3610
- whereClauses.push(`completed_at <= $${paramIdx}`);
3611
- params.push(new Date(input.completedBefore).getTime());
3612
- paramIdx++;
3613
- }
3614
- // dequeuedAfter/Before filter on started_at_epoch_ms: that column is
3615
- // populated on dequeue and surfaced as WorkflowStatus.dequeuedAt.
3616
- if (input.dequeuedAfter) {
3617
- whereClauses.push(`started_at_epoch_ms >= $${paramIdx}`);
3618
- params.push(new Date(input.dequeuedAfter).getTime());
3619
- paramIdx++;
3620
- }
3621
- if (input.dequeuedBefore) {
3622
- whereClauses.push(`started_at_epoch_ms <= $${paramIdx}`);
3623
- params.push(new Date(input.dequeuedBefore).getTime());
3624
- paramIdx++;
3625
- }
3626
- const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
3627
- const groupByClause = groupColumns.join(', ');
3628
- const selectClause = [...groupSelectColumns, ...selectColumns].join(', ');
3629
- const query = `
3630
- SELECT ${selectClause}
3631
- FROM "${this.schemaName}".workflow_status
3632
- ${whereClause}
3633
- GROUP BY ${groupByClause}
3634
- `;
3635
- const result = await this.observabilityQuery((client) => client.query(query, params));
3636
- const toIntOrNull = (v) => (v === null || v === undefined ? null : Number(v));
3637
- return result.rows.map((row) => {
3638
- const group = {};
3639
- for (const name of groupNames) {
3640
- const v = row[name];
3641
- group[name] = v === null || v === undefined ? null : String(v);
3642
- }
3643
- return {
3644
- group,
3645
- count: selectNames.includes('count') ? toIntOrNull(row.count) : null,
3646
- minCreatedAt: selectNames.includes('min_created_at') ? toIntOrNull(row.min_created_at) : null,
3647
- maxQueueWaitMs: selectNames.includes('max_queue_wait_ms') ? toIntOrNull(row.max_queue_wait_ms) : null,
3648
- maxTotalLatencyMs: selectNames.includes('max_total_latency_ms') ? toIntOrNull(row.max_total_latency_ms) : null,
3649
- };
3650
- });
3651
- }
3652
- async getStepAggregates(input) {
3653
- if (input.timeBucketSizeMs !== undefined && input.timeBucketSizeMs <= 0) {
3654
- throw new Error('time_bucket_size_ms must be > 0');
3655
- }
3656
- // operation_outputs has no explicit status column; derive it from whether `error` is populated.
3657
- // Child-workflow mapping rows have NULL error, so they appear as SUCCESS — callers filter by function_name.
3658
- const statusExpr = `CASE WHEN error IS NULL THEN 'SUCCESS' ELSE 'ERROR' END`;
3659
- const groupByFlags = [
3660
- ['function_name', input.groupByFunctionName ?? false, 'function_name'],
3661
- ['status', input.groupByStatus ?? false, statusExpr],
3662
- ];
3663
- const groupNames = [];
3664
- const groupColumns = [];
3665
- const groupSelectColumns = [];
3666
- for (const [colName, enabled, expr] of groupByFlags) {
3667
- if (enabled) {
3668
- groupNames.push(colName);
3669
- groupColumns.push(expr);
3670
- groupSelectColumns.push(`${expr} AS ${colName}`);
3671
- }
3672
- }
3673
- const params = [];
3674
- if (input.timeBucketSizeMs !== undefined) {
3675
- // Bucket on completed_at_epoch_ms — it's the indexed timestamp on
3676
- // this table.
3677
- params.push(input.timeBucketSizeMs);
3678
- const bucket = `$${params.length}::bigint`;
3679
- const bucketExpr = `(CAST(FLOOR(completed_at_epoch_ms / ${bucket}) AS BIGINT) * ${bucket})`;
3680
- groupNames.push('time_bucket');
3681
- groupColumns.push(bucketExpr);
3682
- groupSelectColumns.push(`${bucketExpr} AS time_bucket`);
3683
- }
3684
- if (groupColumns.length === 0) {
3685
- throw new Error('At least one group_by flag must be set to True');
3686
- }
3687
- // Build select columns from boolean flags. Child-workflow mapping rows record start and
3688
- // complete at nearly the same instant, so they contribute ~0; DBOS.getResult and DBOS.sleep
3689
- // rows span their whole wait, so those dominate the duration max.
3690
- const selectFlags = [
3691
- ['count', input.selectCount ?? false, 'COUNT(*)'],
3692
- ['max_duration_ms', input.selectMaxDurationMs ?? false, 'MAX(completed_at_epoch_ms - started_at_epoch_ms)'],
3693
- ];
3694
- const selectNames = [];
3695
- const selectColumns = [];
3696
- for (const [name, enabled, expr] of selectFlags) {
3697
- if (enabled) {
3698
- selectNames.push(name);
3699
- selectColumns.push(`${expr} AS ${name}`);
3700
- }
3701
- }
3702
- if (selectColumns.length === 0) {
3703
- throw new Error('At least one select_ flag must be set to True');
3704
- }
3705
- const whereClauses = [];
3706
- let paramIdx = params.length + 1;
3707
- if (input.status && input.status.length > 0) {
3708
- const placeholders = input.status.map((_, i) => `$${paramIdx + i}`).join(', ');
3709
- whereClauses.push(`(${statusExpr}) IN (${placeholders})`);
3710
- params.push(...input.status);
3711
- paramIdx += input.status.length;
3712
- }
3713
- if (input.functionName && input.functionName.length > 0) {
3714
- const placeholders = input.functionName.map((_, i) => `$${paramIdx + i}`).join(', ');
3715
- whereClauses.push(`function_name IN (${placeholders})`);
3716
- params.push(...input.functionName);
3717
- paramIdx += input.functionName.length;
3718
- }
3719
- if (input.workflowIdPrefix && input.workflowIdPrefix.length > 0) {
3720
- const likeClauses = input.workflowIdPrefix.map((p) => {
3721
- params.push(`${p}%`);
3722
- return `workflow_uuid LIKE $${paramIdx++}`;
3723
- });
3724
- whereClauses.push(`(${likeClauses.join(' OR ')})`);
3725
- }
3726
- if (input.completedAfter) {
3727
- whereClauses.push(`completed_at_epoch_ms >= $${paramIdx}`);
3728
- params.push(new Date(input.completedAfter).getTime());
3729
- paramIdx++;
3730
- }
3731
- if (input.completedBefore) {
3732
- whereClauses.push(`completed_at_epoch_ms <= $${paramIdx}`);
3733
- params.push(new Date(input.completedBefore).getTime());
3734
- paramIdx++;
3735
- }
3736
- // Unset scopes to this application, as on every other observability query.
3737
- whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
3738
- paramIdx = params.length + 1;
3739
- const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
3740
- const groupByClause = groupColumns.join(', ');
3741
- const selectClause = [...groupSelectColumns, ...selectColumns].join(', ');
3742
- const query = `
3743
- SELECT ${selectClause}
3744
- FROM "${this.schemaName}".operation_outputs
3745
- ${whereClause}
3746
- GROUP BY ${groupByClause}
3747
- `;
3748
- const result = await this.observabilityQuery((client) => client.query(query, params));
3749
- const toIntOrNull = (v) => (v === null || v === undefined ? null : Number(v));
3750
- return result.rows.map((row) => {
3751
- const group = {};
3752
- for (const name of groupNames) {
3753
- const v = row[name];
3754
- group[name] = v === null || v === undefined ? null : String(v);
3755
- }
3756
- return {
3757
- group,
3758
- count: selectNames.includes('count') ? toIntOrNull(row.count) : null,
3759
- maxDurationMs: selectNames.includes('max_duration_ms') ? toIntOrNull(row.max_duration_ms) : null,
3760
- };
3761
- });
3762
- }
3763
- /** Whether the system database is CockroachDB. Resolved once, since detecting it costs a query. */
3764
- async #cockroach() {
3765
- if (this.#isCockroach === undefined) {
3766
- const client = await this.#connect();
3767
- try {
3768
- this.#isCockroach = await isCockroachDB(client);
3769
- }
3770
- finally {
3771
- client.release();
3772
- }
3773
- }
3774
- return this.#isCockroach;
3775
- }
3776
- /**
3777
- * Take a database-wide lock for one retention round, returning how to release it, or
3778
- * undefined when another round already holds it. The lock is session-scoped, so a round
3779
- * that crashes releases it. CockroachDB has no advisory locks and always takes it, so it
3780
- * collects unprotected rather than not at all.
3781
- */
3782
- async acquireRetentionLock() {
3783
- if (await this.#cockroach()) {
3784
- return { release: () => Promise.resolve() };
3785
- }
3786
- const key = retentionLockKey(this.schemaName).toString();
3787
- // The round holds this connection until it ends: releasing it would drop the lock.
3788
- const client = await this.#borrowRetentionClient();
3789
- let acquired = false;
3790
- try {
3791
- const { rows } = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [key]);
3792
- acquired = rows[0]?.locked === true;
3793
- }
3794
- catch (e) {
3795
- this.#releaseRetentionClient(client);
3796
- throw e;
3797
- }
3798
- if (!acquired) {
3799
- this.#releaseRetentionClient(client);
3800
- return undefined;
3801
- }
3802
- return {
3803
- release: async () => {
3804
- // Already cut by destroy(): the session is gone and took the lock with it.
3805
- if (!this.#retentionClients.has(client)) {
3806
- return;
3807
- }
3808
- try {
3809
- // Explicit, since releasing the client only returns the session to the pool.
3810
- const { rows } = await client.query('SELECT pg_advisory_unlock($1) AS released', [
3811
- key,
3812
- ]);
3813
- if (rows[0]?.released !== true) {
3814
- // False means this session no longer holds it, which a transaction-pooling proxy
3815
- // causes by switching backends.
3816
- this.logger.warn('Could not release the retention lock: this session no longer holds it. Retention will ' +
3817
- 'not proceed until the lock is released, which happens when the holding backend closes. ' +
3818
- 'A transaction-pooling proxy in front of Postgres causes this; run DBOS through a ' +
3819
- 'session-pooled or direct connection.');
3820
- }
3821
- }
3822
- finally {
3823
- this.#releaseRetentionClient(client);
3824
- }
3825
- },
3826
- };
3827
- }
3828
- /** VACUUM the tables a sweep dirtied. No-op where there is no autovacuum to outrun. */
3829
- async #vacuumTables(tables) {
3830
- if (await this.#cockroach()) {
3831
- return;
3832
- }
3833
- const client = await this.#borrowRetentionClient();
3834
- let notices = [];
3835
- const onNotice = (notice) => notices.push(notice.message ?? '');
3836
- client.on('notice', onNotice);
3837
- try {
3838
- for (const table of tables) {
3839
- // Per table, so one refusal does not skip the rest.
3840
- notices = [];
3841
- try {
3842
- await client.query(`VACUUM (INDEX_CLEANUP ON, TRUNCATE OFF, ANALYZE) "${this.schemaName}"."${table}"`);
3843
- }
3844
- catch (e) {
3845
- if (!this.#retentionClients.has(client)) {
3846
- throw e;
3847
- }
3848
- this.logger.warn(`Payload retention could not vacuum ${table}: ${e.message}`);
3849
- continue;
3850
- }
3851
- // A refused or stalled VACUUM does not raise, it says so in a notice; a successful
3852
- // one is silent, so anything here is worth surfacing.
3853
- for (const notice of notices) {
3854
- this.logger.warn(`Payload retention vacuuming ${table}: ${notice}`);
3855
- }
3856
- }
3857
- }
3858
- finally {
3859
- client.removeListener('notice', onNotice);
3860
- this.#releaseRetentionClient(client);
3861
- }
3862
- }
3863
- /** Delete one payload table's orphans below the cutoff, one batch per transaction. */
3864
- async #garbageCollectTable(table, cutoff, batchSize) {
3865
- // A payload below the cutoff belongs to a workflow created before it, so the status side
3866
- // is the few such rows still present, not the whole table.
3867
- const orphaned = `NOT EXISTS (
3868
- SELECT 1 FROM "${this.schemaName}".workflow_status ws
3869
- WHERE ws.workflow_uuid = t.workflow_uuid AND ws.created_at < $1
3870
- )`;
3871
- // Seed from the oldest row in range.
3872
- const oldest = await retryOnSerializationError(async () => {
3873
- const { rows } = await this.pool.query(`SELECT retention_timestamp
3874
- FROM "${this.schemaName}".${table}
3875
- WHERE retention_timestamp < $1
3876
- ORDER BY retention_timestamp
3877
- LIMIT 1`, [cutoff]);
3878
- // retention_timestamp is a bigint, so node-postgres hands it back as a string.
3879
- return rows.length > 0 ? Number(rows[0].retention_timestamp) : undefined;
3880
- });
3881
- if (oldest === undefined) {
3882
- return 0;
3883
- }
3884
- let total = 0;
3885
- let watermark = oldest - 1;
3886
- for (;;) {
3887
- const batch = await retryOnSerializationError(async () => {
3888
- // Borrowed rather than pool.query'd: that releases with the error, which discards the
3889
- // connection on a deadlock, so the retry wrapping this would churn the pool per batch.
3890
- const client = await this.#borrowRetentionClient();
3891
- try {
3892
- // Batches are cut by candidate count, so rows spared by the anti-join only thin one
3893
- // out; they are re-checked next round.
3894
- const { rows } = await client.query(`SELECT retention_timestamp
3895
- FROM "${this.schemaName}".${table}
3896
- WHERE retention_timestamp < $1 AND retention_timestamp > $2
3897
- ORDER BY retention_timestamp
3898
- LIMIT 1 OFFSET ${batchSize - 1}`, [cutoff, watermark]);
3899
- const step = rows.length > 0 ? Number(rows[0].retention_timestamp) : undefined;
3900
- const params = [cutoff, watermark];
3901
- // Timestamp ties may push the batch slightly over batchSize, but never split across two.
3902
- const upperBound = step === undefined ? '' : `AND t.retention_timestamp <= $${params.push(step)} `;
3903
- const result = await client.query(`DELETE FROM "${this.schemaName}".${table} t
3904
- WHERE t.retention_timestamp < $1 AND t.retention_timestamp > $2 ${upperBound}AND ${orphaned}`, params);
3905
- return { step, deleted: result.rowCount ?? 0 };
3906
- }
3907
- finally {
3908
- this.#releaseRetentionClient(client);
3909
- }
3910
- });
3911
- total += batch.deleted;
3912
- if (batch.step === undefined) {
3913
- return total;
3914
- }
3915
- watermark = batch.step;
3916
- }
3917
- }
3918
- /**
3919
- * Delete payload and step rows below the cutoff whose workflow is gone, returning the count
3920
- * removed from each table. Runs after the status sweep, whose orphans all fall in range:
3921
- * every payload is stamped no later than the completion that made the row collectable.
3922
- */
3923
- async garbageCollectPayloads(cutoff, batchSize = exports.DEFAULT_GC_BATCH_SIZE) {
3924
- // A NaN survives a bare `< 1` test and would only fail once it reached SQL.
3925
- if (!Number.isInteger(batchSize) || batchSize < 1) {
3926
- throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
3927
- }
3928
- const tables = ['workflow_input', 'workflow_output', 'operation_outputs'];
3929
- // To optimize performance, vacuum payload tables both before and after collecting them.
3930
- await this.#vacuumTables(['workflow_status', ...tables]);
3931
- // One connection per concurrent sweep, on top of the one the retention lock holds for the
3932
- // whole round. A pool too small for all three runs them sequentially rather than leaving
3933
- // sweeps waiting on a connection that only the round itself would free.
3934
- const poolMax = this.pool.options.max ?? exports.DEFAULT_POOL_SIZE;
3935
- const concurrency = Math.max(1, Math.min(tables.length, poolMax - 1));
3936
- const deleted = new Array(tables.length).fill(0);
3937
- const failures = [];
3938
- let next = 0;
3939
- const sweep = async () => {
3940
- for (;;) {
3941
- const i = next++;
3942
- if (i >= tables.length)
3943
- return;
3944
- try {
3945
- deleted[i] = await this.#garbageCollectTable(tables[i], cutoff, batchSize);
3946
- }
3947
- catch (e) {
3948
- failures.push(e);
3949
- }
3950
- }
3951
- };
3952
- await Promise.all(Array.from({ length: concurrency }, () => sweep()));
3953
- // Only the first can be thrown, so the rest would otherwise be lost.
3954
- for (const extra of failures.slice(1)) {
3955
- this.logger.warn(`Payload retention sweep also failed: ${extra instanceof Error ? extra.message : String(extra)}`);
3956
- }
3957
- if (failures.length > 0) {
3958
- throw failures[0];
3959
- }
3960
- await this.#vacuumTables(tables);
3961
- this.logger.debug(`Payload retention deleted ${deleted[0]} inputs, ${deleted[1]} outputs, and ${deleted[2]} steps`);
3962
- return deleted;
3963
- }
3964
- /**
3965
- * Rows garbage collection may delete. completed_at is set on every terminal transition and
3966
- * cleared on resume, so one predicate covers eligibility: in-flight rows hold NULL and never
3967
- * compare true. Unscoped by application: retention is system-wide.
3968
- */
3969
- #gcFilter(cutoffEpochTimestampMs, params) {
3970
- params.push(cutoffEpochTimestampMs);
3971
- return `completed_at < $${params.length}`;
3972
- }
3973
- /**
3974
- * Delete one batch, returning the watermark to resume from, or undefined once the last one ran.
3975
- * The delete is its own transaction; it re-checks the filter, so it needs no snapshot shared
3976
- * with the select that bounds it.
3977
- */
3978
- async #garbageCollectBatch(cutoffEpochTimestampMs, batchSize, watermark) {
3979
- // Borrowed rather than pool.query'd: that releases with the error, which discards the
3980
- // connection on a deadlock, so the retry wrapping this would churn the pool per batch.
3981
- const client = await this.#borrowRetentionClient();
3982
- try {
3983
- // The batchSize-th oldest eligible row above the watermark bounds this range
3984
- const stepParams = [];
3985
- const stepScope = this.#gcFilter(cutoffEpochTimestampMs, stepParams);
3986
- stepParams.push(watermark);
3987
- const stepResult = await client.query(`SELECT completed_at
3988
- FROM "${this.schemaName}".workflow_status
3989
- WHERE ${stepScope} AND completed_at > $${stepParams.length}
3990
- ORDER BY completed_at
3991
- LIMIT 1 OFFSET ${batchSize - 1}`, stepParams);
3992
- // completed_at is a bigint, so node-postgres hands it back as a string.
3993
- const step = stepResult.rows.length > 0 ? Number(stepResult.rows[0].completed_at) : undefined;
3994
- const deleteParams = [];
3995
- let deleteScope = this.#gcFilter(cutoffEpochTimestampMs, deleteParams);
3996
- if (step !== undefined) {
3997
- // Inclusive upper bound: completed_at ties may push a batch over batchSize, but never split across two.
3998
- deleteParams.push(watermark, step);
3999
- deleteScope = `${deleteScope} AND completed_at > $${deleteParams.length - 1} AND completed_at <= $${deleteParams.length}`;
4000
- }
4001
- // The final batch drops the watermark: unbounded, since an import can land a
4002
- // completed_at below it mid-pass.
4003
- await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE ${deleteScope}`, deleteParams);
4004
- return step;
4005
- }
4006
- finally {
4007
- this.#releaseRetentionClient(client);
4008
- }
4009
- }
4010
- /**
4011
- * Delete old terminal workflows throughout the system database, returning the cutoff
4012
- * actually used, or undefined when there is nothing to collect.
4013
- *
4014
- * Conductor sends cleared retention thresholds as JSON null, so every param is nullish.
4015
- */
4016
- async garbageCollect(cutoffEpochTimestampMs, rowsThreshold, options = {}) {
4017
- const batchSize = options.batchSize ?? exports.DEFAULT_GC_BATCH_SIZE;
4018
- // A NaN survives a bare `< 1` test and would only fail once it reached SQL, leaving GC half-applied.
4019
- if (!Number.isInteger(batchSize) || batchSize < 1) {
4020
- throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
4021
- }
4022
- if (rowsThreshold !== undefined && rowsThreshold !== null) {
4023
- // The completed_at of the rowsThreshold newest completed row
4024
- const result = await retryOnSerializationError(() => this.pool.query(`SELECT completed_at
4025
- FROM "${this.schemaName}".workflow_status
4026
- WHERE completed_at IS NOT NULL
4027
- ORDER BY completed_at DESC
4028
- LIMIT 1 OFFSET $1`, [rowsThreshold - 1]));
4029
- if (result.rows.length > 0) {
4030
- const rowsBasedCutoff = Number(result.rows[0].completed_at);
4031
- // Use the more restrictive cutoff (higher timestamp = more recent = more deletion)
4032
- if (cutoffEpochTimestampMs === undefined ||
4033
- cutoffEpochTimestampMs === null ||
4034
- rowsBasedCutoff > cutoffEpochTimestampMs) {
4035
- cutoffEpochTimestampMs = rowsBasedCutoff;
4036
- }
4037
- }
4038
- }
4039
- if (cutoffEpochTimestampMs === undefined || cutoffEpochTimestampMs === null) {
4040
- return undefined;
4041
- }
4042
- // Narrowed to a constant so the closures below keep it.
4043
- const cutoff = cutoffEpochTimestampMs;
4044
- // Advance a completed_at watermark, one committed transaction per batch, so a long
4045
- // history neither deletes in one transaction nor rescans what it already deleted.
4046
- const oldest = await retryOnSerializationError(async () => {
4047
- const params = [];
4048
- const scope = this.#gcFilter(cutoff, params);
4049
- const { rows } = await this.pool.query(`SELECT completed_at
4050
- FROM "${this.schemaName}".workflow_status
4051
- WHERE ${scope}
4052
- ORDER BY completed_at
4053
- LIMIT 1`, params);
4054
- return rows.length > 0 ? Number(rows[0].completed_at) : undefined;
4055
- });
4056
- let watermark = oldest === undefined ? 0 : oldest - 1;
4057
- for (;;) {
4058
- const next = await retryOnSerializationError(() => this.#garbageCollectBatch(cutoff, batchSize, watermark));
4059
- // Fewer than a full batch remained, so that delete took the rest.
4060
- if (next === undefined)
4061
- return cutoff;
4062
- watermark = next;
4063
- }
4064
- }
4065
- /**
4066
- * IDs of this application's in-flight workflows created at or before the cutoff.
4067
- * Claiming-scoped, so an upgrade still times out its own unclaimed workflows.
4068
- */
4069
- async listTimedOutWorkflowIds(cutoffEpochTimestampMs) {
4070
- const params = [
4071
- cutoffEpochTimestampMs,
4072
- workflow_1.StatusString.PENDING,
4073
- workflow_1.StatusString.ENQUEUED,
4074
- workflow_1.StatusString.DELAYED,
4075
- ];
4076
- const scope = this.#appNameFilter('application_name', this.appName, params);
4077
- const { rows } = await this.pool.query(`SELECT workflow_uuid
4078
- FROM "${this.schemaName}".workflow_status
4079
- WHERE created_at <= $1
4080
- AND status IN ($2, $3, $4)
4081
- AND ${scope}`, params);
4082
- return rows.map((row) => row.workflow_uuid);
4083
- }
4084
- async getMetrics(startTime, endTime, applicationName) {
4085
- const startEpochMs = new Date(startTime).getTime();
4086
- const endEpochMs = new Date(endTime).getTime();
4087
- const workflowParams = [startEpochMs, endEpochMs];
4088
- const workflowScope = this.#observabilityFilter('application_name', applicationName, workflowParams);
4089
- const stepParams = [startEpochMs, endEpochMs];
4090
- const stepScope = this.#observabilityFilter('application_name', applicationName, stepParams);
4091
- const [workflowResult, stepResult] = await this.observabilityQuery(async (client) => [
4092
- await client.query(`SELECT name, COUNT(workflow_uuid) as count
4093
- FROM "${this.schemaName}".workflow_status
4094
- WHERE created_at >= $1 AND created_at < $2 AND ${workflowScope}
4095
- GROUP BY name`, workflowParams),
4096
- await client.query(`SELECT function_name, COUNT(*) as count
4097
- FROM "${this.schemaName}".operation_outputs
4098
- WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2 AND ${stepScope}
4099
- GROUP BY function_name`, stepParams),
4100
- ]);
4101
- const metrics = [];
4102
- for (const row of workflowResult.rows) {
4103
- metrics.push({
4104
- metricType: 'workflow_count',
4105
- metricName: row.name,
4106
- value: Number(row.count),
4107
- });
4108
- }
4109
- for (const row of stepResult.rows) {
4110
- metrics.push({
4111
- metricType: 'step_count',
4112
- metricName: row.function_name,
4113
- value: Number(row.count),
4114
- });
4115
- }
4116
- return metrics;
4117
- }
4118
3132
  // ==================== Scheduling ====================
4119
3133
  async createSchedule(schedule, client) {
4120
3134
  const q = client ?? this.pool;
@@ -4180,7 +3194,7 @@ class SystemDatabase {
4180
3194
  conditions.push(`(${likeClauses.join(' OR ')})`);
4181
3195
  }
4182
3196
  // Unset scopes to this application, as on every other observability query.
4183
- conditions.push(this.#observabilityFilter('application_name', filters?.applicationName, params));
3197
+ conditions.push(this.observabilityFilter('application_name', filters?.applicationName, params));
4184
3198
  paramIdx = params.length + 1;
4185
3199
  const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
4186
3200
  const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
@@ -4247,7 +3261,7 @@ class SystemDatabase {
4247
3261
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE schedule_name = $2`, [lastFiredAt, name]);
4248
3262
  }
4249
3263
  async applySchedules(schedules) {
4250
- const client = await this.#connect();
3264
+ const client = await this.connect();
4251
3265
  try {
4252
3266
  await client.query('BEGIN');
4253
3267
  for (const sched of schedules) {
@@ -4299,7 +3313,7 @@ class SystemDatabase {
4299
3313
  */
4300
3314
  async createApplicationVersion(versionName, applicationName) {
4301
3315
  const owner = applicationName ?? this.appName;
4302
- const client = await this.#connect();
3316
+ const client = await this.connect();
4303
3317
  try {
4304
3318
  await client.query('BEGIN');
4305
3319
  // Claim a pre-upgrade row in place, so the version is not recreated or retimed.
@@ -4330,7 +3344,7 @@ class SystemDatabase {
4330
3344
  */
4331
3345
  async updateApplicationVersionTimestamp(versionName, newTimestamp, applicationName) {
4332
3346
  const owner = applicationName ?? this.appName;
4333
- const client = await this.#connect();
3347
+ const client = await this.connect();
4334
3348
  try {
4335
3349
  await client.query('BEGIN');
4336
3350
  const resolved = await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
@@ -4354,7 +3368,7 @@ class SystemDatabase {
4354
3368
  }
4355
3369
  async listApplicationVersions() {
4356
3370
  const params = [];
4357
- const scope = this.#appNameFilter('application_name', this.appName, params);
3371
+ const scope = this.appNameFilter('application_name', this.appName, params);
4358
3372
  const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
4359
3373
  FROM "${this.schemaName}".application_versions
4360
3374
  WHERE ${scope}
@@ -4368,7 +3382,7 @@ class SystemDatabase {
4368
3382
  async getLatestApplicationVersion(applicationName) {
4369
3383
  const owner = applicationName ?? this.appName;
4370
3384
  const params = [];
4371
- const scope = this.#appNameFilter('application_name', owner, params);
3385
+ const scope = this.appNameFilter('application_name', owner, params);
4372
3386
  const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
4373
3387
  FROM "${this.schemaName}".application_versions
4374
3388
  WHERE ${scope}
@@ -4392,7 +3406,7 @@ class SystemDatabase {
4392
3406
  */
4393
3407
  async listQueues(applicationName) {
4394
3408
  const params = [];
4395
- const scope = this.#observabilityFilter('application_name', applicationName, params);
3409
+ const scope = this.observabilityFilter('application_name', applicationName, params);
4396
3410
  const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
4397
3411
  FROM "${this.schemaName}".queues
4398
3412
  WHERE ${scope}`, params);
@@ -4440,7 +3454,7 @@ class SystemDatabase {
4440
3454
  application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
4441
3455
  : `ON CONFLICT (name) DO NOTHING`;
4442
3456
  const owner = record.applicationName ?? this.appName;
4443
- const client = await this.#connect();
3457
+ const client = await this.connect();
4444
3458
  try {
4445
3459
  await client.query('BEGIN');
4446
3460
  const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE name = $1`, [record.name]);
@@ -4565,7 +3579,7 @@ class SystemDatabase {
4565
3579
  throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
4566
3580
  }
4567
3581
  // Never a merge: queue, schedule, and version names are globally unique whatever their owner, so this cannot collide.
4568
- const client = await this.#connect();
3582
+ const client = await this.connect();
4569
3583
  let queues, schedules, versions, inFlight;
4570
3584
  try {
4571
3585
  await client.query('BEGIN');
@@ -4875,7 +3889,7 @@ class SystemDatabase {
4875
3889
  const startTimeMs = Date.now();
4876
3890
  // Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
4877
3891
  const endTimeMs = startTimeMs + Math.ceil(durationMS);
4878
- const client = await this.#connect();
3892
+ const client = await this.connect();
4879
3893
  try {
4880
3894
  const res = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
4881
3895
  if (res) {
@@ -4942,7 +3956,7 @@ class SystemDatabase {
4942
3956
  };
4943
3957
  let acquired = null;
4944
3958
  try {
4945
- const client = await this.#connect();
3959
+ const client = await this.connect();
4946
3960
  acquired = client;
4947
3961
  if (this.#abandonIfStopped(client))
4948
3962
  return;
@@ -5199,18 +4213,6 @@ __decorate([
5199
4213
  __metadata("design:paramtypes", [String]),
5200
4214
  __metadata("design:returntype", Promise)
5201
4215
  ], SystemDatabase.prototype, "getQueuePartitions", null);
5202
- __decorate([
5203
- dbRetry(),
5204
- __metadata("design:type", Function),
5205
- __metadata("design:paramtypes", [Number]),
5206
- __metadata("design:returntype", Promise)
5207
- ], SystemDatabase.prototype, "listTimedOutWorkflowIds", null);
5208
- __decorate([
5209
- dbRetry(),
5210
- __metadata("design:type", Function),
5211
- __metadata("design:paramtypes", [String, String, Array]),
5212
- __metadata("design:returntype", Promise)
5213
- ], SystemDatabase.prototype, "getMetrics", null);
5214
4216
  __decorate([
5215
4217
  dbRetry(),
5216
4218
  __metadata("design:type", Function),