@dbos-inc/dbos-sdk 5.3.7-test.961f803 → 5.3.8-preview

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
9
9
  if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
10
  };
11
11
  Object.defineProperty(exports, "__esModule", { value: true });
12
- exports.SystemDatabase = exports.retriablePostgresException = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.advisoryLockKey = exports.retentionLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_GC_BATCH_SIZE = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
12
+ exports.SystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
13
13
  const dbos_executor_1 = require("./dbos-executor");
14
14
  const pg_1 = require("pg");
15
15
  const error_1 = require("./error");
@@ -88,8 +88,6 @@ function setsIdleTransactionTimeout(databaseUrl) {
88
88
  }
89
89
  // Workflows re-owned per transaction by a rename.
90
90
  exports.DEFAULT_RENAME_BATCH_SIZE = 10_000;
91
- // Rows deleted per transaction by garbage collection.
92
- exports.DEFAULT_GC_BATCH_SIZE = 50_000;
93
91
  const QUEUE_COLUMN_BY_FIELD = {
94
92
  concurrency: 'concurrency',
95
93
  workerConcurrency: 'worker_concurrency',
@@ -204,16 +202,6 @@ async function releaseSystemDatabaseClient(client, customPool) {
204
202
  }
205
203
  catch (e) { }
206
204
  }
207
- /**
208
- * The pg_try_advisory_lock argument for one schema's retention round: the leading 8 bytes of
209
- * SHA-256 over a fixed string, read as a signed big-endian 64-bit integer. Separate locks for
210
- * separate schemas. Every DBOS SDK derives the key this way, so rounds in different languages
211
- * against one system database contend for the same lock; changing it here changes it everywhere.
212
- */
213
- function retentionLockKey(schemaName) {
214
- return advisoryLockKey(`dbos.retention.${schemaName}`);
215
- }
216
- exports.retentionLockKey = retentionLockKey;
217
205
  /** A Postgres advisory lock key: the leading 8 bytes of SHA-256 over `lockName`, as a signed big-endian integer. */
218
206
  function advisoryLockKey(lockName) {
219
207
  return (0, crypto_1.createHash)('sha256').update(lockName).digest().readBigInt64BE(0);
@@ -223,6 +211,7 @@ async function isCockroachDB(client) {
223
211
  const versionRes = await client.query('SELECT version() AS version');
224
212
  return /cockroachdb/i.test(versionRes.rows[0]?.version ?? '');
225
213
  }
214
+ exports.isCockroachDB = isCockroachDB;
226
215
  async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
227
216
  if (!customPool) {
228
217
  // A custom pool means the database already exists; otherwise, create it if it does not.
@@ -378,14 +367,6 @@ const RETRY_SQLSTATE_CODES = new Set([
378
367
  '25P03', // idle_in_transaction_session_timeout, when the kill lands on an in-flight query; an idle kill surfaces as a message match below
379
368
  '40P01', // deadlock_detected: the victim's transaction rolled back, so rerunning it is safe
380
369
  ]);
381
- /**
382
- * Bulk maintenance retries these a bounded number of times. 40001 is kept out of the sets above,
383
- * which feed `dbRetry` and so retry forever; every other path runs READ COMMITTED and never sees it.
384
- */
385
- const SERIALIZATION_SQLSTATE_CODES = new Set([
386
- '40001', // serialization_failure (MVCC conflict)
387
- '40P01', // deadlock_detected
388
- ]);
389
370
  // Node.js transient network error codes (system call level)
390
371
  const RETRY_NODE_ERRNOS = new Set([
391
372
  'ECONNRESET',
@@ -400,6 +381,7 @@ function isPgDatabaseError(e) {
400
381
  // Matched by shape, not instanceof: a user-supplied pool may throw another pg copy's DatabaseError.
401
382
  return !!e && typeof e === 'object' && typeof e.code === 'string' && e.code.length === 5;
402
383
  }
384
+ exports.isPgDatabaseError = isPgDatabaseError;
403
385
  function sqlStateLooksRetryable(sqlstate) {
404
386
  if (!sqlstate)
405
387
  return false;
@@ -449,6 +431,7 @@ function* unwrapErrors(e) {
449
431
  yield cur;
450
432
  }
451
433
  }
434
+ exports.unwrapErrors = unwrapErrors;
452
435
  // "What could possibly go wrong?"
453
436
  function retriablePostgresException(err) {
454
437
  // Dig into AggregateErrors of various types
@@ -481,45 +464,45 @@ exports.retriablePostgresException = retriablePostgresException;
481
464
  function isStatementTimeout(err) {
482
465
  return !!err && typeof err === 'object' && err.code === '57014';
483
466
  }
484
- function isSerializationError(err) {
485
- for (const e of unwrapErrors(err)) {
486
- const anyErr = e;
487
- if (isPgDatabaseError(anyErr) && !!anyErr.code && SERIALIZATION_SQLSTATE_CODES.has(anyErr.code)) {
488
- return true;
489
- }
490
- }
491
- return false;
492
- }
493
- /**
494
- * Re-run a batch that lost a deadlock or serialization race. The database already rolled it
495
- * back, so replaying it is safe.
496
- */
497
- async function retryOnSerializationError(operation) {
498
- const maxAttempts = 10;
499
- const maxBackoff = 2.0;
500
- let backoff = 0.05;
501
- for (let attempt = 1;; attempt++) {
467
+ /** Run `operation` under the {@link dbRetry} policy, for code that cannot take the decorator. */
468
+ async function withDbRetry(operation, options = {}) {
469
+ // Read the defaults per call so the backoff stays tunable after the decorator is applied.
470
+ const maxBackoff = options.maxBackoff ?? utils_1.dbRetryConfig.maxBackoffSec;
471
+ let retries = 0;
472
+ let backoff = options.initialBackoff ?? utils_1.dbRetryConfig.initialBackoffSec;
473
+ const abortSignal = (0, context_1.currentDbRetryAbortSignal)();
474
+ while (true) {
502
475
  try {
503
476
  return await operation();
504
477
  }
505
478
  catch (e) {
506
- if (!isSerializationError(e)) {
507
- throw e;
479
+ if (retriablePostgresException(e)) {
480
+ if (abortSignal?.aborted)
481
+ throw e;
482
+ retries++;
483
+ // Calculate backoff with jitter
484
+ const actualBackoff = backoff * (0.5 + Math.random());
485
+ dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Database connection failed: ${e instanceof Error ? e.message : String(e)}. ` +
486
+ `Retrying in ${actualBackoff.toFixed(2)}s (attempt ${retries})`);
487
+ // Sleep with backoff
488
+ if (abortSignal) {
489
+ await (0, utils_1.interruptibleSleep)(actualBackoff * 1000, abortSignal);
490
+ if (abortSignal.aborted)
491
+ throw e;
492
+ }
493
+ else {
494
+ await (0, utils_1.sleepms)(actualBackoff * 1000); // Convert to milliseconds
495
+ }
496
+ // Increase backoff for next attempt (exponential)
497
+ backoff = Math.min(backoff * 2, maxBackoff);
508
498
  }
509
- const message = e instanceof Error ? e.message : String(e);
510
- if (attempt === maxAttempts) {
511
- dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Garbage collection failed after ${maxAttempts} attempts: ${message}`);
499
+ else {
512
500
  throw e;
513
501
  }
514
- // Jittered backoff, so peers that collided do not collide again
515
- const actualBackoff = backoff * (0.5 + Math.random());
516
- dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Contention or deadlock detected in workflow garbage collection: ${message}. ` +
517
- `Retrying in ${actualBackoff.toFixed(2)}s (attempt ${attempt})`);
518
- await (0, utils_1.sleepms)(actualBackoff * 1000);
519
- backoff = Math.min(backoff * 2, maxBackoff);
520
502
  }
521
503
  }
522
504
  }
505
+ exports.withDbRetry = withDbRetry;
523
506
  /**
524
507
  * If a workflow encounters a database connection issue while performing an operation,
525
508
  * block the workflow and retry the operation until it reconnects and succeeds.
@@ -533,41 +516,7 @@ function dbRetry(options = {}) {
533
516
  return function (target, propertyName, descriptor) {
534
517
  const method = descriptor.value;
535
518
  descriptor.value = async function (...args) {
536
- // Read the defaults per call so the backoff stays tunable after the decorator is applied.
537
- const maxBackoff = options.maxBackoff ?? utils_1.dbRetryConfig.maxBackoffSec;
538
- let retries = 0;
539
- let backoff = options.initialBackoff ?? utils_1.dbRetryConfig.initialBackoffSec;
540
- const abortSignal = (0, context_1.currentDbRetryAbortSignal)();
541
- while (true) {
542
- try {
543
- return await method.apply(this, args);
544
- }
545
- catch (e) {
546
- if (retriablePostgresException(e)) {
547
- if (abortSignal?.aborted)
548
- throw e;
549
- retries++;
550
- // Calculate backoff with jitter
551
- const actualBackoff = backoff * (0.5 + Math.random());
552
- dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Database connection failed: ${e instanceof Error ? e.message : String(e)}. ` +
553
- `Retrying in ${actualBackoff.toFixed(2)}s (attempt ${retries})`);
554
- // Sleep with backoff
555
- if (abortSignal) {
556
- await (0, utils_1.interruptibleSleep)(actualBackoff * 1000, abortSignal);
557
- if (abortSignal.aborted)
558
- throw e;
559
- }
560
- else {
561
- await (0, utils_1.sleepms)(actualBackoff * 1000); // Convert to milliseconds
562
- }
563
- // Increase backoff for next attempt (exponential)
564
- backoff = Math.min(backoff * 2, maxBackoff);
565
- }
566
- else {
567
- throw e;
568
- }
569
- }
570
- }
519
+ return withDbRetry(() => method.apply(this, args), options);
571
520
  };
572
521
  return descriptor;
573
522
  };
@@ -649,11 +598,12 @@ class SystemDatabase {
649
598
  #batchCreatedAtCursors = new Map();
650
599
  // Set by destroy(), so polling waits end instead of running on against a pool that outlives this handle.
651
600
  #destroyed = false;
652
- // Resolved on first use by #cockroach(), since detecting it costs a query.
653
- #isCockroach = undefined;
601
+ get destroyed() {
602
+ return this.#destroyed;
603
+ }
654
604
  // Connections a retention round holds right now. destroy() cuts them, since closing the
655
605
  // pool would otherwise wait on the lock session and on any statement in flight.
656
- #retentionClients = new Set();
606
+ retentionClients = new Set();
657
607
  constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS,
658
608
  // The application this handle acts for; undefined writes unclaimed rows.
659
609
  appName, observabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS, idleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS) {
@@ -713,25 +663,9 @@ class SystemDatabase {
713
663
  this.logger.warn(`Unexpected error on a system database connection: ${err}`);
714
664
  };
715
665
  /** Check out a pool connection guarded for as long as we hold it. See {@link borrowClient}. */
716
- #connect() {
666
+ connect() {
717
667
  return borrowClient(this.pool, this.#onClientError);
718
668
  }
719
- /** Borrow a connection for a retention round, so destroy() can cut it. */
720
- async #borrowRetentionClient() {
721
- if (this.#destroyed) {
722
- throw new Error('System database shutting down');
723
- }
724
- const client = await this.#connect();
725
- this.#retentionClients.add(client);
726
- return client;
727
- }
728
- /** Return a retention connection, unless destroy() already cut it: a second release would throw. */
729
- #releaseRetentionClient(client) {
730
- if (this.#retentionClients.delete(client)) {
731
- // No error argument: a genuinely dead connection is still evicted by the pool's own check.
732
- client.release();
733
- }
734
- }
735
669
  /**
736
670
  * Cap an introspection read with a statement timeout, so one scanning a huge table cannot hold a
737
671
  * snapshot for minutes and stall autovacuum database-wide. Soft-private so tests can assert the cap
@@ -742,7 +676,7 @@ class SystemDatabase {
742
676
  */
743
677
  async observabilityQuery(fn) {
744
678
  const timeoutMs = this.observabilityQueryTimeoutMs;
745
- const client = await this.#connect();
679
+ const client = await this.connect();
746
680
  try {
747
681
  if (timeoutMs === undefined) {
748
682
  return await fn(client);
@@ -771,7 +705,7 @@ class SystemDatabase {
771
705
  * A predicate matching rows owned by these applications plus unclaimed ones, which belong to
772
706
  * every application; unset or empty matches everything. Appends its bind parameter to `params`.
773
707
  */
774
- #appNameFilter(column, value, params) {
708
+ appNameFilter(column, value, params) {
775
709
  // An empty name is no name: it is not a value any application could be configured with.
776
710
  const names = !value ? [] : Array.isArray(value) ? value : [value];
777
711
  if (names.length === 0)
@@ -784,8 +718,8 @@ class SystemDatabase {
784
718
  * this application owns, not to every application's rows. A handle with no application of its
785
719
  * own still matches every one.
786
720
  */
787
- #observabilityFilter(column, value, params) {
788
- return this.#appNameFilter(column, value ?? this.appName, params);
721
+ observabilityFilter(column, value, params) {
722
+ return this.appNameFilter(column, value ?? this.appName, params);
789
723
  }
790
724
  /**
791
725
  * The version name a dequeue treats as latest: this application's own plus unclaimed
@@ -793,7 +727,7 @@ class SystemDatabase {
793
727
  */
794
728
  async #latestApplicationVersionName(client) {
795
729
  const params = [];
796
- const scope = this.#appNameFilter('application_name', this.appName, params);
730
+ const scope = this.appNameFilter('application_name', this.appName, params);
797
731
  const { rows } = await client.query(`SELECT version_name
798
732
  FROM "${this.schemaName}".application_versions
799
733
  WHERE ${scope}
@@ -856,7 +790,7 @@ class SystemDatabase {
856
790
  // A retention round still running is cut here rather than waited for: returned with an
857
791
  // error, each connection is destroyed at once, the idle lock session lets go of the
858
792
  // advisory lock, and the round fails on its next statement instead of holding shutdown.
859
- for (const client of this.#retentionClients) {
793
+ for (const client of this.retentionClients) {
860
794
  // Cover the release() call itself, which tears the connection down and can surface a socket error.
861
795
  client.on('error', () => { });
862
796
  try {
@@ -866,7 +800,7 @@ class SystemDatabase {
866
800
  this.logger.warn(`Error releasing a retention connection: ${String(e)}`);
867
801
  }
868
802
  }
869
- this.#retentionClients.clear();
803
+ this.retentionClients.clear();
870
804
  // We attached nothing to the pool object itself, so there is nothing to unpick; only close one we own.
871
805
  if (!this.customPool) {
872
806
  await this.pool.end();
@@ -881,7 +815,7 @@ class SystemDatabase {
881
815
  return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
882
816
  }
883
817
  async initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy) {
884
- const client = await this.#connect();
818
+ const client = await this.connect();
885
819
  let shouldCommit = false;
886
820
  try {
887
821
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
@@ -907,7 +841,7 @@ class SystemDatabase {
907
841
  }
908
842
  /** Insert a child workflow's row and record it as the parent's step in one transaction, so a crash leaves both or neither. */
909
843
  async initChildWorkflowStatus(initStatus, creatorXid, parentWorkflowID, parentFunctionID, startTimeEpochMs, endTimeEpochMs, reusePolicy = 'return-existing') {
910
- const client = await this.#connect();
844
+ const client = await this.connect();
911
845
  let shouldCommit = false;
912
846
  try {
913
847
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
@@ -1084,7 +1018,7 @@ class SystemDatabase {
1084
1018
  'schedule_name',
1085
1019
  'application_name',
1086
1020
  ];
1087
- const client = await this.#connect();
1021
+ const client = await this.connect();
1088
1022
  try {
1089
1023
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1090
1024
  // Chunk to stay well under the bind-parameter limit.
@@ -1137,7 +1071,7 @@ class SystemDatabase {
1137
1071
  return inserted;
1138
1072
  }
1139
1073
  async recordWorkflowOutput(workflowID, status, ownerXid) {
1140
- const client = await this.#connect();
1074
+ const client = await this.connect();
1141
1075
  try {
1142
1076
  return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
1143
1077
  }
@@ -1146,7 +1080,7 @@ class SystemDatabase {
1146
1080
  }
1147
1081
  }
1148
1082
  async recordWorkflowError(workflowID, status, ownerXid) {
1149
- const client = await this.#connect();
1083
+ const client = await this.connect();
1150
1084
  try {
1151
1085
  return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
1152
1086
  }
@@ -1194,22 +1128,11 @@ class SystemDatabase {
1194
1128
  }
1195
1129
  }
1196
1130
  }
1197
- async getPendingWorkflows(executorID, appVersion) {
1198
- const params = [workflow_1.StatusString.PENDING, executorID, appVersion];
1199
- // executor_id defaults to "local", so it collides across applications.
1200
- const scope = this.#appNameFilter('application_name', this.appName, params);
1201
- const getWorkflows = await this.pool.query(`SELECT workflow_uuid
1202
- FROM "${this.schemaName}".workflow_status
1203
- WHERE status=$1 AND executor_id=$2 AND application_version=$3 AND ${scope}`, params);
1204
- return getWorkflows.rows.map((i) => ({
1205
- workflowUUID: i.workflow_uuid,
1206
- }));
1207
- }
1208
1131
  // Recovery re-enqueues rather than executing directly so the queue's atomic dequeue admits exactly one runner, and the executor ID predicate rejects sweeps for rows a live executor has already claimed.
1209
1132
  async reenqueueWorkflowsForRecovery(executorID, appVersion, recoveryQueueName) {
1210
1133
  const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
1211
1134
  // executor_id defaults to "local", so it collides across applications.
1212
- const scope = this.#appNameFilter('application_name', this.appName, params);
1135
+ const scope = this.appNameFilter('application_name', this.appName, params);
1213
1136
  const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1214
1137
  SET started_at_epoch_ms = NULL,
1215
1138
  status = $1,
@@ -1230,7 +1153,7 @@ class SystemDatabase {
1230
1153
  return status ? JSON.stringify(status) : null;
1231
1154
  };
1232
1155
  if (callerID && callerFN) {
1233
- const client = await this.#connect();
1156
+ const client = await this.connect();
1234
1157
  try {
1235
1158
  // Check if the operation has been done before for OAOO (only do this inside a workflow).
1236
1159
  const json = await this.#inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, funcGetStatus));
@@ -1266,7 +1189,7 @@ class SystemDatabase {
1266
1189
  }
1267
1190
  // Only used in tests
1268
1191
  async setWorkflowStatus(workflowID, status, resetRecoveryAttempts, internalOptions) {
1269
- const client = await this.#connect();
1192
+ const client = await this.connect();
1270
1193
  try {
1271
1194
  await this.updateWorkflowStatus(client, workflowID, status, {
1272
1195
  update: {
@@ -1284,7 +1207,7 @@ class SystemDatabase {
1284
1207
  }
1285
1208
  // ==================== Step Results ====================
1286
1209
  async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
1287
- const client = await this.#connect();
1210
+ const client = await this.connect();
1288
1211
  try {
1289
1212
  return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
1290
1213
  }
@@ -1292,8 +1215,13 @@ class SystemDatabase {
1292
1215
  client.release();
1293
1216
  }
1294
1217
  }
1295
- async getAllOperationResults(workflowID, limit, offset) {
1296
- let query = `SELECT * FROM "${this.schemaName}".operation_outputs WHERE workflow_uuid=$1 ORDER BY function_id`;
1218
+ async getAllOperationResults(workflowID, limit, offset, loadOutput = true) {
1219
+ // Skip the output/error payload columns when the caller does not need them,
1220
+ // mirroring how listWorkflows skips its payload joins when loadOutput is false.
1221
+ const columns = loadOutput
1222
+ ? '*'
1223
+ : 'workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp';
1224
+ let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE workflow_uuid=$1 ORDER BY function_id`;
1297
1225
  const params = [workflowID];
1298
1226
  if (limit !== undefined) {
1299
1227
  params.push(limit);
@@ -1307,7 +1235,7 @@ class SystemDatabase {
1307
1235
  return rows;
1308
1236
  }
1309
1237
  async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
1310
- const client = await this.#connect();
1238
+ const client = await this.connect();
1311
1239
  try {
1312
1240
  await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options));
1313
1241
  }
@@ -1317,7 +1245,7 @@ class SystemDatabase {
1317
1245
  }
1318
1246
  }
1319
1247
  async runTransactionalStep(workflowID, functionID, functionName, callback) {
1320
- const client = await this.#connect();
1248
+ const client = await this.connect();
1321
1249
  try {
1322
1250
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1323
1251
  const existing = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
@@ -1363,7 +1291,7 @@ class SystemDatabase {
1363
1291
  }
1364
1292
  // Nondeprecated - skip matching entry, unpatched if nonmatching entry,
1365
1293
  // If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
1366
- const client = await this.#connect();
1294
+ const client = await this.connect();
1367
1295
  try {
1368
1296
  return await this.#inTransaction(client, async () => {
1369
1297
  const checkpointName = await readCheckpointName(client);
@@ -1410,7 +1338,7 @@ class SystemDatabase {
1410
1338
  await this.#checkIfCanceled(this.pool, workflowID);
1411
1339
  }
1412
1340
  async resumeWorkflows(workflowIDs, queueName) {
1413
- const client = await this.#connect();
1341
+ const client = await this.connect();
1414
1342
  try {
1415
1343
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1416
1344
  // Check existence separately: a zero-row update also means "already complete", a legal no-op.
@@ -1474,7 +1402,7 @@ class SystemDatabase {
1474
1402
  return await this.debounceDelayedWorkflowStandalone(params);
1475
1403
  }
1476
1404
  async debounceDelayedWorkflowStandalone(params) {
1477
- const client = await this.#connect();
1405
+ const client = await this.connect();
1478
1406
  try {
1479
1407
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1480
1408
  const result = await this.#debounceDelayedWorkflowInternal(client, params);
@@ -1502,7 +1430,7 @@ class SystemDatabase {
1502
1430
  params.applicationName ?? null,
1503
1431
  ];
1504
1432
  // Never extend a workflow the target application doesn't own; falls through to the holder below.
1505
- const ownScope = this.#appNameFilter('application_name', params.applicationName, updateParams);
1433
+ const ownScope = this.appNameFilter('application_name', params.applicationName, updateParams);
1506
1434
  const updated = await client.query(`UPDATE "${this.schemaName}".workflow_status
1507
1435
  SET delay_until_epoch_ms = CASE
1508
1436
  WHEN debounce_deadline_epoch_ms IS NOT NULL AND debounce_deadline_epoch_ms < $1
@@ -1584,7 +1512,7 @@ class SystemDatabase {
1584
1512
  allIds.push(...(await this.getWorkflowChildren(wfid)));
1585
1513
  }
1586
1514
  }
1587
- const client = await this.#connect();
1515
+ const client = await this.connect();
1588
1516
  try {
1589
1517
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1590
1518
  // The payload tables carry no foreign key, so the status delete does not cascade
@@ -1630,7 +1558,7 @@ class SystemDatabase {
1630
1558
  throw new error_1.DBOSError(`startStep must be >= 0, got ${startStep}`);
1631
1559
  }
1632
1560
  const schema = this.schemaName;
1633
- const client = await this.#connect();
1561
+ const client = await this.connect();
1634
1562
  try {
1635
1563
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1636
1564
  const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE workflow_uuid = $1`, [workflowID]);
@@ -1712,62 +1640,6 @@ class SystemDatabase {
1712
1640
  const result = await this.bulkForkWorkflows([workflowID], [newWorkflowID], [startStep], options);
1713
1641
  return result[0];
1714
1642
  }
1715
- async forkFromFailure(workflowIDs, options = {}) {
1716
- const modes = [
1717
- options.fromLastFailure ?? false,
1718
- options.fromLastStep ?? false,
1719
- options.fromStep !== undefined,
1720
- options.fromStepName !== undefined,
1721
- ].filter(Boolean).length;
1722
- if (modes !== 1) {
1723
- throw new Error('Exactly one of fromLastFailure, fromLastStep, fromStep, or fromStepName must be specified');
1724
- }
1725
- let startSteps;
1726
- if (options.fromStep !== undefined) {
1727
- startSteps = Array(workflowIDs.length).fill(options.fromStep);
1728
- }
1729
- else {
1730
- let query;
1731
- const params = [workflowIDs];
1732
- if (options.fromLastFailure) {
1733
- query = `SELECT workflow_uuid,
1734
- COALESCE(
1735
- MAX(function_id) FILTER (WHERE error IS NOT NULL),
1736
- MAX(function_id)
1737
- ) AS start_step
1738
- FROM "${this.schemaName}".operation_outputs
1739
- WHERE workflow_uuid = ANY($1)
1740
- GROUP BY workflow_uuid`;
1741
- }
1742
- else if (options.fromLastStep) {
1743
- query = `SELECT workflow_uuid, MAX(function_id) AS start_step
1744
- FROM "${this.schemaName}".operation_outputs
1745
- WHERE workflow_uuid = ANY($1)
1746
- GROUP BY workflow_uuid`;
1747
- }
1748
- else {
1749
- // fromStepName
1750
- query = `SELECT workflow_uuid, MAX(function_id) AS start_step
1751
- FROM "${this.schemaName}".operation_outputs
1752
- WHERE workflow_uuid = ANY($1) AND function_name = $2
1753
- GROUP BY workflow_uuid`;
1754
- params.push(options.fromStepName);
1755
- }
1756
- const result = await this.pool.query(query, params);
1757
- const startStepByID = new Map(result.rows.map((r) => [r.workflow_uuid, Number(r.start_step)]));
1758
- if (options.fromStepName !== undefined) {
1759
- for (const wid of workflowIDs) {
1760
- if (!startStepByID.has(wid)) {
1761
- throw new Error(`Workflow ${wid} has no step named '${options.fromStepName}'`);
1762
- }
1763
- }
1764
- }
1765
- // A workflow with no recorded steps has nothing to resume from, so restart it from the beginning.
1766
- startSteps = workflowIDs.map((wid) => startStepByID.get(wid) ?? 0);
1767
- }
1768
- const forkedIDs = workflowIDs.map(() => (0, crypto_1.randomUUID)());
1769
- return this.bulkForkWorkflows(workflowIDs, forkedIDs, startSteps, options);
1770
- }
1771
1643
  async bulkForkWorkflows(originalWorkflowIDs, forkedWorkflowIDs, startSteps, options = {}) {
1772
1644
  if (originalWorkflowIDs.length === 0) {
1773
1645
  return [];
@@ -1775,7 +1647,7 @@ class SystemDatabase {
1775
1647
  if (originalWorkflowIDs.length !== forkedWorkflowIDs.length || originalWorkflowIDs.length !== startSteps.length) {
1776
1648
  throw new Error('originalWorkflowIDs, forkedWorkflowIDs, and startSteps must have the same length');
1777
1649
  }
1778
- const client = await this.#connect();
1650
+ const client = await this.connect();
1779
1651
  try {
1780
1652
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1781
1653
  // Fetch the status of all original workflows inside the transaction.
@@ -1927,188 +1799,6 @@ class SystemDatabase {
1927
1799
  client.release();
1928
1800
  }
1929
1801
  }
1930
- async exportWorkflow(workflowID, exportChildren = false) {
1931
- const workflowIDs = [workflowID];
1932
- if (exportChildren) {
1933
- workflowIDs.push(...(await this.getWorkflowChildren(workflowID)));
1934
- }
1935
- const exportedWorkflows = [];
1936
- const client = await this.#connect();
1937
- try {
1938
- for (const wfID of workflowIDs) {
1939
- // Export workflow_status
1940
- const statusResult = await client.query(
1941
- // creator_xid and owner_xid are intentionally omitted: they are transient
1942
- // tokens, not logical workflow state, and a source database's
1943
- // tokens are meaningless in the target.
1944
- `SELECT
1945
- ws.workflow_uuid, ws.status, ws.name, ws.authenticated_user, ws.assumed_role,
1946
- ws.authenticated_roles,
1947
- COALESCE(wo.output, ws.output) AS output, COALESCE(wo.error, ws.error) AS error,
1948
- ws.executor_id,
1949
- ws.created_at, ws.updated_at, ws.application_version, ws.application_id,
1950
- ws.class_name, ws.config_name, ws.recovery_attempts, ws.queue_name,
1951
- ws.workflow_timeout_ms, ws.workflow_deadline_epoch_ms, ws.started_at_epoch_ms,
1952
- ws.deduplication_id, COALESCE(wi.inputs, ws.inputs) AS inputs,
1953
- ws.priority, ws.queue_partition_key, ws.forked_from,
1954
- ws.parent_workflow_id, ws.serialization, ws.delay_until_epoch_ms,
1955
- ws.was_forked_from, ws.rate_limited, ws.completed_at, ws.attributes, ws.schedule_name,
1956
- ws.debounce_deadline_epoch_ms, ws.is_debounced, ws.application_name
1957
- FROM "${this.schemaName}".workflow_status ws
1958
- LEFT JOIN "${this.schemaName}".workflow_input wi ON wi.workflow_uuid = ws.workflow_uuid
1959
- LEFT JOIN "${this.schemaName}".workflow_output wo ON wo.workflow_uuid = ws.workflow_uuid
1960
- WHERE ws.workflow_uuid = $1`, [wfID]);
1961
- if (statusResult.rows.length === 0) {
1962
- throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${wfID} does not exist`);
1963
- }
1964
- const workflowStatus = statusResult.rows[0];
1965
- // Export operation_outputs
1966
- const outputsResult = await client.query(`SELECT
1967
- workflow_uuid, function_id, function_name, output, error,
1968
- child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
1969
- serialization, application_name
1970
- FROM "${this.schemaName}".operation_outputs
1971
- WHERE workflow_uuid = $1`, [wfID]);
1972
- // Export workflow_events
1973
- const eventsResult = await client.query(`SELECT workflow_uuid, key, value, serialization
1974
- FROM "${this.schemaName}".workflow_events
1975
- WHERE workflow_uuid = $1`, [wfID]);
1976
- // Export workflow_events_history
1977
- const historyResult = await client.query(`SELECT workflow_uuid, function_id, key, value, serialization
1978
- FROM "${this.schemaName}".workflow_events_history
1979
- WHERE workflow_uuid = $1`, [wfID]);
1980
- // Export streams
1981
- const streamsResult = await client.query(`SELECT workflow_uuid, key, value, "offset", function_id, serialization
1982
- FROM "${this.schemaName}".streams
1983
- WHERE workflow_uuid = $1`, [wfID]);
1984
- exportedWorkflows.push({
1985
- workflow_status: workflowStatus,
1986
- operation_outputs: outputsResult.rows,
1987
- workflow_events: eventsResult.rows,
1988
- workflow_events_history: historyResult.rows,
1989
- streams: streamsResult.rows,
1990
- });
1991
- }
1992
- }
1993
- finally {
1994
- client.release();
1995
- }
1996
- return exportedWorkflows;
1997
- }
1998
- async importWorkflow(workflows) {
1999
- const client = await this.#connect();
2000
- try {
2001
- await client.query('BEGIN');
2002
- for (const workflow of workflows) {
2003
- const status = workflow.workflow_status;
2004
- // Import workflow_status
2005
- await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
2006
- workflow_uuid, status, name, authenticated_user, assumed_role,
2007
- authenticated_roles, output, error, executor_id,
2008
- created_at, updated_at, application_version, application_id,
2009
- class_name, config_name, recovery_attempts, queue_name,
2010
- workflow_timeout_ms, workflow_deadline_epoch_ms, started_at_epoch_ms,
2011
- deduplication_id, inputs, priority, queue_partition_key, forked_from,
2012
- parent_workflow_id, serialization, delay_until_epoch_ms,
2013
- was_forked_from, rate_limited, completed_at, attributes, schedule_name,
2014
- debounce_deadline_epoch_ms, is_debounced, application_name
2015
- ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)`, [
2016
- status.workflow_uuid,
2017
- status.status,
2018
- status.name,
2019
- status.authenticated_user,
2020
- status.assumed_role,
2021
- status.authenticated_roles,
2022
- // Legacy columns: the payloads live in their own tables.
2023
- null,
2024
- null,
2025
- status.executor_id,
2026
- status.created_at,
2027
- status.updated_at,
2028
- status.application_version,
2029
- status.application_id,
2030
- status.class_name,
2031
- status.config_name,
2032
- status.recovery_attempts,
2033
- status.queue_name,
2034
- status.workflow_timeout_ms,
2035
- status.workflow_deadline_epoch_ms,
2036
- status.started_at_epoch_ms,
2037
- status.deduplication_id,
2038
- null,
2039
- status.priority,
2040
- status.queue_partition_key,
2041
- status.forked_from,
2042
- status.parent_workflow_id,
2043
- status.serialization,
2044
- status.delay_until_epoch_ms ?? null,
2045
- // NOT NULL columns: fall back to FALSE for payloads exported before
2046
- // these fields were included.
2047
- status.was_forked_from ?? false,
2048
- status.rate_limited ?? false,
2049
- status.completed_at ?? null,
2050
- status.attributes ? JSON.stringify(status.attributes) : null,
2051
- status.schedule_name ?? null,
2052
- status.debounce_deadline_epoch_ms ?? null,
2053
- status.is_debounced ?? false,
2054
- status.application_name ?? null,
2055
- ]);
2056
- // Retention starts at import: the original timestamps are long past the cutoff
2057
- // and would be collected immediately.
2058
- await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
2059
- VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [status.workflow_uuid, status.inputs]);
2060
- if (status.output !== null || status.error !== null) {
2061
- await client.query(`INSERT INTO "${this.schemaName}".workflow_output (workflow_uuid, output, error, retention_timestamp)
2062
- VALUES ($1, $2, $3, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [status.workflow_uuid, status.output, status.error]);
2063
- }
2064
- // Import operation_outputs
2065
- for (const output of workflow.operation_outputs) {
2066
- await client.query(`INSERT INTO "${this.schemaName}".operation_outputs (
2067
- workflow_uuid, function_id, function_name, output, error,
2068
- child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
2069
- serialization, application_name, retention_timestamp
2070
- ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [
2071
- output.workflow_uuid,
2072
- output.function_id,
2073
- output.function_name,
2074
- output.output,
2075
- output.error,
2076
- output.child_workflow_id,
2077
- output.started_at_epoch_ms,
2078
- output.completed_at_epoch_ms,
2079
- output.serialization,
2080
- output.application_name ?? null,
2081
- ]);
2082
- }
2083
- // Import workflow_events
2084
- for (const event of workflow.workflow_events) {
2085
- await client.query(`INSERT INTO "${this.schemaName}".workflow_events (
2086
- workflow_uuid, key, value, serialization
2087
- ) VALUES ($1, $2, $3, $4)`, [event.workflow_uuid, event.key, event.value, event.serialization]);
2088
- }
2089
- // Import workflow_events_history
2090
- for (const history of workflow.workflow_events_history) {
2091
- await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (
2092
- workflow_uuid, function_id, key, value, serialization
2093
- ) VALUES ($1, $2, $3, $4, $5)`, [history.workflow_uuid, history.function_id, history.key, history.value, history.serialization]);
2094
- }
2095
- // Import streams
2096
- for (const stream of workflow.streams) {
2097
- await client.query(`INSERT INTO "${this.schemaName}".streams (
2098
- workflow_uuid, key, value, "offset", function_id, serialization
2099
- ) VALUES ($1, $2, $3, $4, $5, $6)`, [stream.workflow_uuid, stream.key, stream.value, stream.offset, stream.function_id, stream.serialization]);
2100
- }
2101
- }
2102
- await client.query('COMMIT');
2103
- }
2104
- catch (error) {
2105
- await client.query('ROLLBACK');
2106
- throw error;
2107
- }
2108
- finally {
2109
- client.release();
2110
- }
2111
- }
2112
1802
  // ==================== Awaiting Workflows ====================
2113
1803
  /** Register one execution; the returned release is idempotent, so it can run both before a park and once the run settles. */
2114
1804
  registerRunningWorkflow(workflowID, queueName, queuePartitionKey) {
@@ -2396,7 +2086,7 @@ class SystemDatabase {
2396
2086
  async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey) {
2397
2087
  topic = topic ?? this.nullTopic;
2398
2088
  const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
2399
- const client = await this.#connect();
2089
+ const client = await this.connect();
2400
2090
  try {
2401
2091
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
2402
2092
  await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
@@ -2432,7 +2122,7 @@ class SystemDatabase {
2432
2122
  /** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
2433
2123
  async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
2434
2124
  const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
2435
- const client = await this.#connect();
2125
+ const client = await this.connect();
2436
2126
  try {
2437
2127
  await this.#inTransaction(client, async () => {
2438
2128
  await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
@@ -2520,7 +2210,7 @@ class SystemDatabase {
2520
2210
  // Transactionally consume and return the message if it's in the DB, otherwise return null.
2521
2211
  let message = null;
2522
2212
  let serialization = null;
2523
- const client = await this.#connect();
2213
+ const client = await this.connect();
2524
2214
  try {
2525
2215
  await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
2526
2216
  const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
@@ -2560,7 +2250,7 @@ class SystemDatabase {
2560
2250
  }
2561
2251
  // ==================== Events ====================
2562
2252
  async setEvent(workflowID, functionID, key, message, serialization) {
2563
- const client = await this.#connect();
2253
+ const client = await this.connect();
2564
2254
  try {
2565
2255
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
2566
2256
  // Only a real write (not a replay) should wake readers.
@@ -2668,7 +2358,7 @@ class SystemDatabase {
2668
2358
  // ==================== Streams ====================
2669
2359
  async writeStreamFromStep(workflowID, functionID, key, serializedValue, serialization) {
2670
2360
  const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
2671
- const client = await this.#connect();
2361
+ const client = await this.connect();
2672
2362
  try {
2673
2363
  while (true) {
2674
2364
  try {
@@ -2705,7 +2395,7 @@ class SystemDatabase {
2705
2395
  }
2706
2396
  }
2707
2397
  async writeStreamFromWorkflow(workflowID, functionID, key, serializedValue, serialization, functionName) {
2708
- const client = await this.#connect();
2398
+ const client = await this.connect();
2709
2399
  try {
2710
2400
  while (true) {
2711
2401
  // Only a real insert (not a replay) should wake readers.
@@ -2839,7 +2529,7 @@ class SystemDatabase {
2839
2529
  }
2840
2530
  try {
2841
2531
  // One statement: one round trip, one async-notify queue-lock acquisition; unnest emits one notification per payload.
2842
- const client = await this.#connect();
2532
+ const client = await this.connect();
2843
2533
  try {
2844
2534
  await client.query(`SELECT pg_notify($1, p) FROM unnest($2::text[]) AS p`, [channel, Array.from(batch)]);
2845
2535
  }
@@ -2854,49 +2544,6 @@ class SystemDatabase {
2854
2544
  }
2855
2545
  }
2856
2546
  // ==================== Observability: Workflow Communications ====================
2857
- async getAllEvents(workflowID) {
2858
- const { rows } = await this.observabilityQuery((client) => client.query(`SELECT key, value, serialization FROM "${this.schemaName}".workflow_events
2859
- WHERE workflow_uuid = $1`, [workflowID]));
2860
- const events = {};
2861
- for (const row of rows) {
2862
- events[row.key] = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
2863
- }
2864
- return events;
2865
- }
2866
- async getAllNotifications(workflowID) {
2867
- const { rows } = await this.observabilityQuery((client) => client.query(`SELECT topic, message, serialization, created_at_epoch_ms, consumed
2868
- FROM "${this.schemaName}".notifications
2869
- WHERE destination_uuid = $1
2870
- ORDER BY created_at_epoch_ms`, [workflowID]));
2871
- return await Promise.all(rows.map(async (row) => ({
2872
- topic: row.topic === this.nullTopic ? null : row.topic,
2873
- message: await (0, serialization_1.safeParse)(this.serializer, row.message, row.serialization),
2874
- createdAtEpochMs: Number(row.created_at_epoch_ms),
2875
- consumed: row.consumed,
2876
- })));
2877
- }
2878
- async getAllStreamEntries(workflowID) {
2879
- const { rows } = await this.observabilityQuery((client) => client.query(`SELECT key, value, serialization FROM "${this.schemaName}".streams
2880
- WHERE workflow_uuid = $1
2881
- ORDER BY key, "offset"`, [workflowID]));
2882
- const streams = {};
2883
- const closed = new Set();
2884
- for (const row of rows) {
2885
- if (closed.has(row.key)) {
2886
- continue;
2887
- }
2888
- // safeParse yields the raw string for the legacy unserialized marker, which does not parse.
2889
- const value = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
2890
- if (isStreamClosedSentinel(value)) {
2891
- // End the stream where readStream does, so the two never disagree.
2892
- closed.add(row.key);
2893
- streams[row.key] ??= [];
2894
- continue;
2895
- }
2896
- (streams[row.key] ??= []).push(value);
2897
- }
2898
- return streams;
2899
- }
2900
2547
  // ==================== Queues ====================
2901
2548
  async transitionDelayedWorkflows() {
2902
2549
  // Transition workflows from DELAYED to ENQUEUED when their delay has expired.
@@ -2904,7 +2551,7 @@ class SystemDatabase {
2904
2551
  // debounce key held only while DELAYED, so a later same-key debounce starts a fresh workflow.
2905
2552
  const params = [workflow_1.StatusString.ENQUEUED, Date.now(), workflow_1.StatusString.DELAYED];
2906
2553
  // Only what this application would dequeue: a peer's debounce key is not ours to clear.
2907
- const scope = this.#appNameFilter('application_name', this.appName, params);
2554
+ const scope = this.appNameFilter('application_name', this.appName, params);
2908
2555
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
2909
2556
  SET status = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
2910
2557
  deduplication_id = CASE WHEN is_debounced THEN NULL ELSE deduplication_id END
@@ -2913,7 +2560,7 @@ class SystemDatabase {
2913
2560
  /** Cancel up to `limit` of this application's active workflows whose deadline has passed, returning their IDs. */
2914
2561
  async cancelTimedOutWorkflows(limit) {
2915
2562
  const params = [workflow_1.StatusString.CANCELLED, Date.now(), limit];
2916
- const scope = this.#appNameFilter('application_name', this.appName, params);
2563
+ const scope = this.appNameFilter('application_name', this.appName, params);
2917
2564
  // Literal statuses let the planner match idx_workflow_status_deadline under any plan mode.
2918
2565
  // SKIP LOCKED leaves a row a dequeue or a peer's sweep holds for the next sweep.
2919
2566
  const { rows } = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
@@ -2944,7 +2591,7 @@ class SystemDatabase {
2944
2591
  // Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v3, so cost scales with the number of partitions rather than the backlog depth.
2945
2592
  const params = [queueName, workflow_1.StatusString.ENQUEUED];
2946
2593
  // Only partitions this application can actually dequeue from.
2947
- const scope = this.#appNameFilter('application_name', this.appName, params);
2594
+ const scope = this.appNameFilter('application_name', this.appName, params);
2948
2595
  const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
2949
2596
  (SELECT MIN(queue_partition_key) AS pk
2950
2597
  FROM "${this.schemaName}".workflow_status
@@ -2971,7 +2618,7 @@ class SystemDatabase {
2971
2618
  queue.partitionRateLimit !== undefined;
2972
2619
  // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
2973
2620
  const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
2974
- const client = await this.#connect();
2621
+ const client = await this.connect();
2975
2622
  try {
2976
2623
  // Default to READ COMMITTED except with a budget shared across executors
2977
2624
  if (hasSharedBudget) {
@@ -2984,7 +2631,7 @@ class SystemDatabase {
2984
2631
  const rateLimitRemaining = async (rateLimit, partitionScoped) => {
2985
2632
  const params = [queue.name, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED, rateLimit.periodSec * 1000];
2986
2633
  // Count only what this application would dequeue, matching the select below.
2987
- const scope = this.#appNameFilter('application_name', this.appName, params);
2634
+ const scope = this.appNameFilter('application_name', this.appName, params);
2988
2635
  const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
2989
2636
  const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
2990
2637
  WHERE queue_name = $1
@@ -3003,7 +2650,7 @@ class SystemDatabase {
3003
2650
  */
3004
2651
  const pendingCount = async (partitionScoped) => {
3005
2652
  const params = [queue.name, workflow_1.StatusString.PENDING];
3006
- const scope = this.#appNameFilter('application_name', this.appName, params);
2653
+ const scope = this.appNameFilter('application_name', this.appName, params);
3007
2654
  const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
3008
2655
  const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
3009
2656
  WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
@@ -3064,7 +2711,7 @@ class SystemDatabase {
3064
2711
  const lockMode = hasSharedBudget ? 'FOR UPDATE NOWAIT' : 'FOR UPDATE SKIP LOCKED';
3065
2712
  const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
3066
2713
  const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
3067
- const selectScope = this.#appNameFilter('application_name', this.appName, selectParams);
2714
+ const selectScope = this.appNameFilter('application_name', this.appName, selectParams);
3068
2715
  const selectQuery = `
3069
2716
  SELECT workflow_uuid
3070
2717
  FROM "${this.schemaName}".workflow_status
@@ -3097,7 +2744,7 @@ class SystemDatabase {
3097
2744
  ownerXid,
3098
2745
  ];
3099
2746
  // Re-check ownership alongside status, as the partitioned claim guard does.
3100
- const claimScope = this.#appNameFilter('application_name', this.appName, updateParams);
2747
+ const claimScope = this.appNameFilter('application_name', this.appName, updateParams);
3101
2748
  // RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
3102
2749
  const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
3103
2750
  SET status = $1,
@@ -3147,7 +2794,7 @@ class SystemDatabase {
3147
2794
  throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
3148
2795
  }
3149
2796
  // partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
3150
- const client = await this.#connect();
2797
+ const client = await this.connect();
3151
2798
  try {
3152
2799
  await client.query('BEGIN');
3153
2800
  const latestVersion = await this.#latestApplicationVersionName(client);
@@ -3166,7 +2813,7 @@ class SystemDatabase {
3166
2813
  workflow_1.StatusString.PENDING,
3167
2814
  sweepLimit,
3168
2815
  ];
3169
- const candidateScope = this.#appNameFilter('application_name', this.appName, candidateParams);
2816
+ const candidateScope = this.appNameFilter('application_name', this.appName, candidateParams);
3170
2817
  // Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
3171
2818
  const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
3172
2819
  (SELECT MIN(queue_partition_key) AS pk
@@ -3223,7 +2870,7 @@ class SystemDatabase {
3223
2870
  AND ${versionClause(version)}
3224
2871
  AND ${scope}`;
3225
2872
  const lockedParams = [candidateIDs, workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
3226
- const lockedScope = this.#appNameFilter('application_name', this.appName, lockedParams);
2873
+ const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
3227
2874
  // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
3228
2875
  const lockedResult = await client.query(`SELECT workflow_uuid
3229
2876
  FROM "${this.schemaName}".workflow_status
@@ -3247,7 +2894,7 @@ class SystemDatabase {
3247
2894
  this.appName ?? null,
3248
2895
  ownerXid,
3249
2896
  ];
3250
- const flipScope = this.#appNameFilter('application_name', this.appName, flipParams);
2897
+ const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
3251
2898
  // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
3252
2899
  const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
3253
2900
  SET status = $1,
@@ -3373,8 +3020,8 @@ class SystemDatabase {
3373
3020
  // Falsy, not just undefined: a Conductor request carries an omitted list as JSON null.
3374
3021
  const idKeyed = (input.workflowIDs?.length ?? 0) > 0;
3375
3022
  whereClauses.push(idKeyed
3376
- ? this.#appNameFilter('application_name', input.applicationName, params)
3377
- : this.#observabilityFilter('application_name', input.applicationName, params));
3023
+ ? this.appNameFilter('application_name', input.applicationName, params)
3024
+ : this.observabilityFilter('application_name', input.applicationName, params));
3378
3025
  paramCounter = params.length + 1;
3379
3026
  if (input.workflow_id_prefix) {
3380
3027
  if (Array.isArray(input.workflow_id_prefix)) {
@@ -3487,634 +3134,6 @@ class SystemDatabase {
3487
3134
  : await this.observabilityQuery((client) => client.query(query, params));
3488
3135
  return result.rows.map(mapWorkflowStatus);
3489
3136
  }
3490
- async getWorkflowAggregates(input) {
3491
- if (input.timeBucketSizeMs !== undefined && input.timeBucketSizeMs <= 0) {
3492
- throw new Error('time_bucket_size_ms must be > 0');
3493
- }
3494
- const groupByFlags = [
3495
- ['status', input.groupByStatus ?? false, 'status'],
3496
- ['name', input.groupByName ?? false, 'name'],
3497
- ['queue_name', input.groupByQueueName ?? false, 'queue_name'],
3498
- ['executor_id', input.groupByExecutorId ?? false, 'executor_id'],
3499
- ['application_version', input.groupByApplicationVersion ?? false, 'application_version'],
3500
- ['application_name', input.groupByApplicationName ?? false, 'application_name'],
3501
- ];
3502
- const groupNames = [];
3503
- const groupColumns = [];
3504
- const groupSelectColumns = [];
3505
- for (const [colName, enabled, col] of groupByFlags) {
3506
- if (enabled) {
3507
- groupNames.push(colName);
3508
- groupColumns.push(col);
3509
- groupSelectColumns.push(col);
3510
- }
3511
- }
3512
- const params = [];
3513
- if (input.timeBucketSizeMs !== undefined) {
3514
- // Bucket on created_at — the indexed wall-clock timestamp on workflow_status.
3515
- // One placeholder shared by SELECT and GROUP BY, so Postgres sees the two expressions as identical.
3516
- params.push(input.timeBucketSizeMs);
3517
- const bucket = `$${params.length}::bigint`;
3518
- const bucketExpr = `(CAST(FLOOR(created_at / ${bucket}) AS BIGINT) * ${bucket})`;
3519
- groupNames.push('time_bucket');
3520
- groupColumns.push(bucketExpr);
3521
- groupSelectColumns.push(`${bucketExpr} AS time_bucket`);
3522
- }
3523
- if (groupColumns.length === 0) {
3524
- throw new Error('At least one group_by flag must be set to True');
3525
- }
3526
- // Build select columns from boolean flags. MAX ignores NULLs, so rows
3527
- // missing started_at_epoch_ms or completed_at naturally drop out of the
3528
- // latency maxes.
3529
- const selectFlags = [
3530
- ['count', input.selectCount ?? false, 'COUNT(*)'],
3531
- ['min_created_at', input.selectMinCreatedAt ?? false, 'MIN(created_at)'],
3532
- ['max_queue_wait_ms', input.selectMaxQueueWaitMs ?? false, 'MAX(started_at_epoch_ms - created_at)'],
3533
- ['max_total_latency_ms', input.selectMaxTotalLatencyMs ?? false, 'MAX(completed_at - created_at)'],
3534
- ];
3535
- const selectNames = [];
3536
- const selectColumns = [];
3537
- for (const [name, enabled, expr] of selectFlags) {
3538
- if (enabled) {
3539
- selectNames.push(name);
3540
- selectColumns.push(`${expr} AS ${name}`);
3541
- }
3542
- }
3543
- if (selectColumns.length === 0) {
3544
- throw new Error('At least one select_ flag must be set to True');
3545
- }
3546
- const whereClauses = [];
3547
- let paramIdx = params.length + 1;
3548
- const addFilter = (column, values) => {
3549
- if (!values || values.length === 0)
3550
- return;
3551
- const placeholders = values.map((_, i) => `$${paramIdx + i}`).join(', ');
3552
- whereClauses.push(`${column} IN (${placeholders})`);
3553
- params.push(...values);
3554
- paramIdx += values.length;
3555
- };
3556
- addFilter('status', input.status);
3557
- addFilter('name', input.name);
3558
- addFilter('application_version', input.appVersion);
3559
- addFilter('executor_id', input.executorId);
3560
- addFilter('queue_name', input.queueName);
3561
- if (input.workflowIdPrefix && input.workflowIdPrefix.length > 0) {
3562
- const likeClauses = input.workflowIdPrefix.map((p) => {
3563
- params.push(`${p}%`);
3564
- return `workflow_uuid LIKE $${paramIdx++}`;
3565
- });
3566
- whereClauses.push(`(${likeClauses.join(' OR ')})`);
3567
- }
3568
- addFilter('workflow_uuid', input.workflowIDs);
3569
- addFilter('authenticated_user', input.authenticatedUser);
3570
- addFilter('forked_from', input.forkedFrom);
3571
- addFilter('parent_workflow_id', input.parentWorkflowID);
3572
- addFilter('schedule_name', input.scheduleName);
3573
- // Unset scopes to this application, as on every other observability query.
3574
- whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
3575
- paramIdx = params.length + 1;
3576
- if (input.wasForkedFrom !== undefined) {
3577
- whereClauses.push(`was_forked_from = $${paramIdx}`);
3578
- params.push(input.wasForkedFrom);
3579
- paramIdx++;
3580
- }
3581
- // Matches the forks themselves, as opposed to wasForkedFrom, which matches the workflows they were forked from.
3582
- if (input.isFork !== undefined) {
3583
- whereClauses.push(input.isFork ? `forked_from IS NOT NULL` : `forked_from IS NULL`);
3584
- }
3585
- if (input.hasParent !== undefined) {
3586
- whereClauses.push(input.hasParent ? `parent_workflow_id IS NOT NULL` : `parent_workflow_id IS NULL`);
3587
- }
3588
- // Match workflows whose attributes JSONB contains all the given key-value pairs.
3589
- if (input.attributes && Object.keys(input.attributes).length > 0) {
3590
- whereClauses.push(`attributes @> $${paramIdx}::jsonb`);
3591
- params.push(JSON.stringify(input.attributes));
3592
- paramIdx++;
3593
- }
3594
- if (input.startTime) {
3595
- whereClauses.push(`created_at >= $${paramIdx}`);
3596
- params.push(new Date(input.startTime).getTime());
3597
- paramIdx++;
3598
- }
3599
- if (input.endTime) {
3600
- whereClauses.push(`created_at <= $${paramIdx}`);
3601
- params.push(new Date(input.endTime).getTime());
3602
- paramIdx++;
3603
- }
3604
- if (input.completedAfter) {
3605
- whereClauses.push(`completed_at >= $${paramIdx}`);
3606
- params.push(new Date(input.completedAfter).getTime());
3607
- paramIdx++;
3608
- }
3609
- if (input.completedBefore) {
3610
- whereClauses.push(`completed_at <= $${paramIdx}`);
3611
- params.push(new Date(input.completedBefore).getTime());
3612
- paramIdx++;
3613
- }
3614
- // dequeuedAfter/Before filter on started_at_epoch_ms: that column is
3615
- // populated on dequeue and surfaced as WorkflowStatus.dequeuedAt.
3616
- if (input.dequeuedAfter) {
3617
- whereClauses.push(`started_at_epoch_ms >= $${paramIdx}`);
3618
- params.push(new Date(input.dequeuedAfter).getTime());
3619
- paramIdx++;
3620
- }
3621
- if (input.dequeuedBefore) {
3622
- whereClauses.push(`started_at_epoch_ms <= $${paramIdx}`);
3623
- params.push(new Date(input.dequeuedBefore).getTime());
3624
- paramIdx++;
3625
- }
3626
- const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
3627
- const groupByClause = groupColumns.join(', ');
3628
- const selectClause = [...groupSelectColumns, ...selectColumns].join(', ');
3629
- const query = `
3630
- SELECT ${selectClause}
3631
- FROM "${this.schemaName}".workflow_status
3632
- ${whereClause}
3633
- GROUP BY ${groupByClause}
3634
- `;
3635
- const result = await this.observabilityQuery((client) => client.query(query, params));
3636
- const toIntOrNull = (v) => (v === null || v === undefined ? null : Number(v));
3637
- return result.rows.map((row) => {
3638
- const group = {};
3639
- for (const name of groupNames) {
3640
- const v = row[name];
3641
- group[name] = v === null || v === undefined ? null : String(v);
3642
- }
3643
- return {
3644
- group,
3645
- count: selectNames.includes('count') ? toIntOrNull(row.count) : null,
3646
- minCreatedAt: selectNames.includes('min_created_at') ? toIntOrNull(row.min_created_at) : null,
3647
- maxQueueWaitMs: selectNames.includes('max_queue_wait_ms') ? toIntOrNull(row.max_queue_wait_ms) : null,
3648
- maxTotalLatencyMs: selectNames.includes('max_total_latency_ms') ? toIntOrNull(row.max_total_latency_ms) : null,
3649
- };
3650
- });
3651
- }
3652
- async getStepAggregates(input) {
3653
- if (input.timeBucketSizeMs !== undefined && input.timeBucketSizeMs <= 0) {
3654
- throw new Error('time_bucket_size_ms must be > 0');
3655
- }
3656
- // operation_outputs has no explicit status column; derive it from whether `error` is populated.
3657
- // Child-workflow mapping rows have NULL error, so they appear as SUCCESS — callers filter by function_name.
3658
- const statusExpr = `CASE WHEN error IS NULL THEN 'SUCCESS' ELSE 'ERROR' END`;
3659
- const groupByFlags = [
3660
- ['function_name', input.groupByFunctionName ?? false, 'function_name'],
3661
- ['status', input.groupByStatus ?? false, statusExpr],
3662
- ];
3663
- const groupNames = [];
3664
- const groupColumns = [];
3665
- const groupSelectColumns = [];
3666
- for (const [colName, enabled, expr] of groupByFlags) {
3667
- if (enabled) {
3668
- groupNames.push(colName);
3669
- groupColumns.push(expr);
3670
- groupSelectColumns.push(`${expr} AS ${colName}`);
3671
- }
3672
- }
3673
- const params = [];
3674
- if (input.timeBucketSizeMs !== undefined) {
3675
- // Bucket on completed_at_epoch_ms — it's the indexed timestamp on
3676
- // this table.
3677
- params.push(input.timeBucketSizeMs);
3678
- const bucket = `$${params.length}::bigint`;
3679
- const bucketExpr = `(CAST(FLOOR(completed_at_epoch_ms / ${bucket}) AS BIGINT) * ${bucket})`;
3680
- groupNames.push('time_bucket');
3681
- groupColumns.push(bucketExpr);
3682
- groupSelectColumns.push(`${bucketExpr} AS time_bucket`);
3683
- }
3684
- if (groupColumns.length === 0) {
3685
- throw new Error('At least one group_by flag must be set to True');
3686
- }
3687
- // Build select columns from boolean flags. Child-workflow mapping rows record start and
3688
- // complete at nearly the same instant, so they contribute ~0; DBOS.getResult and DBOS.sleep
3689
- // rows span their whole wait, so those dominate the duration max.
3690
- const selectFlags = [
3691
- ['count', input.selectCount ?? false, 'COUNT(*)'],
3692
- ['max_duration_ms', input.selectMaxDurationMs ?? false, 'MAX(completed_at_epoch_ms - started_at_epoch_ms)'],
3693
- ];
3694
- const selectNames = [];
3695
- const selectColumns = [];
3696
- for (const [name, enabled, expr] of selectFlags) {
3697
- if (enabled) {
3698
- selectNames.push(name);
3699
- selectColumns.push(`${expr} AS ${name}`);
3700
- }
3701
- }
3702
- if (selectColumns.length === 0) {
3703
- throw new Error('At least one select_ flag must be set to True');
3704
- }
3705
- const whereClauses = [];
3706
- let paramIdx = params.length + 1;
3707
- if (input.status && input.status.length > 0) {
3708
- const placeholders = input.status.map((_, i) => `$${paramIdx + i}`).join(', ');
3709
- whereClauses.push(`(${statusExpr}) IN (${placeholders})`);
3710
- params.push(...input.status);
3711
- paramIdx += input.status.length;
3712
- }
3713
- if (input.functionName && input.functionName.length > 0) {
3714
- const placeholders = input.functionName.map((_, i) => `$${paramIdx + i}`).join(', ');
3715
- whereClauses.push(`function_name IN (${placeholders})`);
3716
- params.push(...input.functionName);
3717
- paramIdx += input.functionName.length;
3718
- }
3719
- if (input.workflowIdPrefix && input.workflowIdPrefix.length > 0) {
3720
- const likeClauses = input.workflowIdPrefix.map((p) => {
3721
- params.push(`${p}%`);
3722
- return `workflow_uuid LIKE $${paramIdx++}`;
3723
- });
3724
- whereClauses.push(`(${likeClauses.join(' OR ')})`);
3725
- }
3726
- if (input.completedAfter) {
3727
- whereClauses.push(`completed_at_epoch_ms >= $${paramIdx}`);
3728
- params.push(new Date(input.completedAfter).getTime());
3729
- paramIdx++;
3730
- }
3731
- if (input.completedBefore) {
3732
- whereClauses.push(`completed_at_epoch_ms <= $${paramIdx}`);
3733
- params.push(new Date(input.completedBefore).getTime());
3734
- paramIdx++;
3735
- }
3736
- // Unset scopes to this application, as on every other observability query.
3737
- whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
3738
- paramIdx = params.length + 1;
3739
- const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
3740
- const groupByClause = groupColumns.join(', ');
3741
- const selectClause = [...groupSelectColumns, ...selectColumns].join(', ');
3742
- const query = `
3743
- SELECT ${selectClause}
3744
- FROM "${this.schemaName}".operation_outputs
3745
- ${whereClause}
3746
- GROUP BY ${groupByClause}
3747
- `;
3748
- const result = await this.observabilityQuery((client) => client.query(query, params));
3749
- const toIntOrNull = (v) => (v === null || v === undefined ? null : Number(v));
3750
- return result.rows.map((row) => {
3751
- const group = {};
3752
- for (const name of groupNames) {
3753
- const v = row[name];
3754
- group[name] = v === null || v === undefined ? null : String(v);
3755
- }
3756
- return {
3757
- group,
3758
- count: selectNames.includes('count') ? toIntOrNull(row.count) : null,
3759
- maxDurationMs: selectNames.includes('max_duration_ms') ? toIntOrNull(row.max_duration_ms) : null,
3760
- };
3761
- });
3762
- }
3763
- /** Whether the system database is CockroachDB. Resolved once, since detecting it costs a query. */
3764
- async #cockroach() {
3765
- if (this.#isCockroach === undefined) {
3766
- const client = await this.#connect();
3767
- try {
3768
- this.#isCockroach = await isCockroachDB(client);
3769
- }
3770
- finally {
3771
- client.release();
3772
- }
3773
- }
3774
- return this.#isCockroach;
3775
- }
3776
- /**
3777
- * Take a database-wide lock for one retention round, returning how to release it, or
3778
- * undefined when another round already holds it. The lock is session-scoped, so a round
3779
- * that crashes releases it. CockroachDB has no advisory locks and always takes it, so it
3780
- * collects unprotected rather than not at all.
3781
- */
3782
- async acquireRetentionLock() {
3783
- if (await this.#cockroach()) {
3784
- return { release: () => Promise.resolve() };
3785
- }
3786
- const key = retentionLockKey(this.schemaName).toString();
3787
- // The round holds this connection until it ends: releasing it would drop the lock.
3788
- const client = await this.#borrowRetentionClient();
3789
- let acquired = false;
3790
- try {
3791
- const { rows } = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [key]);
3792
- acquired = rows[0]?.locked === true;
3793
- }
3794
- catch (e) {
3795
- this.#releaseRetentionClient(client);
3796
- throw e;
3797
- }
3798
- if (!acquired) {
3799
- this.#releaseRetentionClient(client);
3800
- return undefined;
3801
- }
3802
- return {
3803
- release: async () => {
3804
- // Already cut by destroy(): the session is gone and took the lock with it.
3805
- if (!this.#retentionClients.has(client)) {
3806
- return;
3807
- }
3808
- try {
3809
- // Explicit, since releasing the client only returns the session to the pool.
3810
- const { rows } = await client.query('SELECT pg_advisory_unlock($1) AS released', [
3811
- key,
3812
- ]);
3813
- if (rows[0]?.released !== true) {
3814
- // False means this session no longer holds it, which a transaction-pooling proxy
3815
- // causes by switching backends.
3816
- this.logger.warn('Could not release the retention lock: this session no longer holds it. Retention will ' +
3817
- 'not proceed until the lock is released, which happens when the holding backend closes. ' +
3818
- 'A transaction-pooling proxy in front of Postgres causes this; run DBOS through a ' +
3819
- 'session-pooled or direct connection.');
3820
- }
3821
- }
3822
- finally {
3823
- this.#releaseRetentionClient(client);
3824
- }
3825
- },
3826
- };
3827
- }
3828
- /** VACUUM the tables a sweep dirtied. No-op where there is no autovacuum to outrun. */
3829
- async #vacuumTables(tables) {
3830
- if (await this.#cockroach()) {
3831
- return;
3832
- }
3833
- const client = await this.#borrowRetentionClient();
3834
- let notices = [];
3835
- const onNotice = (notice) => notices.push(notice.message ?? '');
3836
- client.on('notice', onNotice);
3837
- try {
3838
- for (const table of tables) {
3839
- // Per table, so one refusal does not skip the rest.
3840
- notices = [];
3841
- try {
3842
- await client.query(`VACUUM (INDEX_CLEANUP ON, TRUNCATE OFF, ANALYZE) "${this.schemaName}"."${table}"`);
3843
- }
3844
- catch (e) {
3845
- if (!this.#retentionClients.has(client)) {
3846
- throw e;
3847
- }
3848
- this.logger.warn(`Payload retention could not vacuum ${table}: ${e.message}`);
3849
- continue;
3850
- }
3851
- // A refused or stalled VACUUM does not raise, it says so in a notice; a successful
3852
- // one is silent, so anything here is worth surfacing.
3853
- for (const notice of notices) {
3854
- this.logger.warn(`Payload retention vacuuming ${table}: ${notice}`);
3855
- }
3856
- }
3857
- }
3858
- finally {
3859
- client.removeListener('notice', onNotice);
3860
- this.#releaseRetentionClient(client);
3861
- }
3862
- }
3863
- /** Delete one payload table's orphans below the cutoff, one batch per transaction. */
3864
- async #garbageCollectTable(table, cutoff, batchSize) {
3865
- // A payload below the cutoff belongs to a workflow created before it, so the status side
3866
- // is the few such rows still present, not the whole table.
3867
- const orphaned = `NOT EXISTS (
3868
- SELECT 1 FROM "${this.schemaName}".workflow_status ws
3869
- WHERE ws.workflow_uuid = t.workflow_uuid AND ws.created_at < $1
3870
- )`;
3871
- // Seed from the oldest row in range.
3872
- const oldest = await retryOnSerializationError(async () => {
3873
- const { rows } = await this.pool.query(`SELECT retention_timestamp
3874
- FROM "${this.schemaName}".${table}
3875
- WHERE retention_timestamp < $1
3876
- ORDER BY retention_timestamp
3877
- LIMIT 1`, [cutoff]);
3878
- // retention_timestamp is a bigint, so node-postgres hands it back as a string.
3879
- return rows.length > 0 ? Number(rows[0].retention_timestamp) : undefined;
3880
- });
3881
- if (oldest === undefined) {
3882
- return 0;
3883
- }
3884
- let total = 0;
3885
- let watermark = oldest - 1;
3886
- for (;;) {
3887
- const batch = await retryOnSerializationError(async () => {
3888
- // Borrowed rather than pool.query'd: that releases with the error, which discards the
3889
- // connection on a deadlock, so the retry wrapping this would churn the pool per batch.
3890
- const client = await this.#borrowRetentionClient();
3891
- try {
3892
- // Batches are cut by candidate count, so rows spared by the anti-join only thin one
3893
- // out; they are re-checked next round.
3894
- const { rows } = await client.query(`SELECT retention_timestamp
3895
- FROM "${this.schemaName}".${table}
3896
- WHERE retention_timestamp < $1 AND retention_timestamp > $2
3897
- ORDER BY retention_timestamp
3898
- LIMIT 1 OFFSET ${batchSize - 1}`, [cutoff, watermark]);
3899
- const step = rows.length > 0 ? Number(rows[0].retention_timestamp) : undefined;
3900
- const params = [cutoff, watermark];
3901
- // Timestamp ties may push the batch slightly over batchSize, but never split across two.
3902
- const upperBound = step === undefined ? '' : `AND t.retention_timestamp <= $${params.push(step)} `;
3903
- const result = await client.query(`DELETE FROM "${this.schemaName}".${table} t
3904
- WHERE t.retention_timestamp < $1 AND t.retention_timestamp > $2 ${upperBound}AND ${orphaned}`, params);
3905
- return { step, deleted: result.rowCount ?? 0 };
3906
- }
3907
- finally {
3908
- this.#releaseRetentionClient(client);
3909
- }
3910
- });
3911
- total += batch.deleted;
3912
- if (batch.step === undefined) {
3913
- return total;
3914
- }
3915
- watermark = batch.step;
3916
- }
3917
- }
3918
- /**
3919
- * Delete payload and step rows below the cutoff whose workflow is gone, returning the count
3920
- * removed from each table. Runs after the status sweep, whose orphans all fall in range:
3921
- * every payload is stamped no later than the completion that made the row collectable.
3922
- */
3923
- async garbageCollectPayloads(cutoff, batchSize = exports.DEFAULT_GC_BATCH_SIZE) {
3924
- // A NaN survives a bare `< 1` test and would only fail once it reached SQL.
3925
- if (!Number.isInteger(batchSize) || batchSize < 1) {
3926
- throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
3927
- }
3928
- const tables = ['workflow_input', 'workflow_output', 'operation_outputs'];
3929
- // To optimize performance, vacuum payload tables both before and after collecting them.
3930
- await this.#vacuumTables(['workflow_status', ...tables]);
3931
- // One connection per concurrent sweep, on top of the one the retention lock holds for the
3932
- // whole round. A pool too small for all three runs them sequentially rather than leaving
3933
- // sweeps waiting on a connection that only the round itself would free.
3934
- const poolMax = this.pool.options.max ?? exports.DEFAULT_POOL_SIZE;
3935
- const concurrency = Math.max(1, Math.min(tables.length, poolMax - 1));
3936
- const deleted = new Array(tables.length).fill(0);
3937
- const failures = [];
3938
- let next = 0;
3939
- const sweep = async () => {
3940
- for (;;) {
3941
- const i = next++;
3942
- if (i >= tables.length)
3943
- return;
3944
- try {
3945
- deleted[i] = await this.#garbageCollectTable(tables[i], cutoff, batchSize);
3946
- }
3947
- catch (e) {
3948
- failures.push(e);
3949
- }
3950
- }
3951
- };
3952
- await Promise.all(Array.from({ length: concurrency }, () => sweep()));
3953
- // Only the first can be thrown, so the rest would otherwise be lost.
3954
- for (const extra of failures.slice(1)) {
3955
- this.logger.warn(`Payload retention sweep also failed: ${extra instanceof Error ? extra.message : String(extra)}`);
3956
- }
3957
- if (failures.length > 0) {
3958
- throw failures[0];
3959
- }
3960
- await this.#vacuumTables(tables);
3961
- this.logger.debug(`Payload retention deleted ${deleted[0]} inputs, ${deleted[1]} outputs, and ${deleted[2]} steps`);
3962
- return deleted;
3963
- }
3964
- /**
3965
- * Rows garbage collection may delete. completed_at is set on every terminal transition and
3966
- * cleared on resume, so one predicate covers eligibility: in-flight rows hold NULL and never
3967
- * compare true. Unscoped by application: retention is system-wide.
3968
- */
3969
- #gcFilter(cutoffEpochTimestampMs, params) {
3970
- params.push(cutoffEpochTimestampMs);
3971
- return `completed_at < $${params.length}`;
3972
- }
3973
- /**
3974
- * Delete one batch, returning the watermark to resume from, or undefined once the last one ran.
3975
- * The delete is its own transaction; it re-checks the filter, so it needs no snapshot shared
3976
- * with the select that bounds it.
3977
- */
3978
- async #garbageCollectBatch(cutoffEpochTimestampMs, batchSize, watermark) {
3979
- // Borrowed rather than pool.query'd: that releases with the error, which discards the
3980
- // connection on a deadlock, so the retry wrapping this would churn the pool per batch.
3981
- const client = await this.#borrowRetentionClient();
3982
- try {
3983
- // The batchSize-th oldest eligible row above the watermark bounds this range
3984
- const stepParams = [];
3985
- const stepScope = this.#gcFilter(cutoffEpochTimestampMs, stepParams);
3986
- stepParams.push(watermark);
3987
- const stepResult = await client.query(`SELECT completed_at
3988
- FROM "${this.schemaName}".workflow_status
3989
- WHERE ${stepScope} AND completed_at > $${stepParams.length}
3990
- ORDER BY completed_at
3991
- LIMIT 1 OFFSET ${batchSize - 1}`, stepParams);
3992
- // completed_at is a bigint, so node-postgres hands it back as a string.
3993
- const step = stepResult.rows.length > 0 ? Number(stepResult.rows[0].completed_at) : undefined;
3994
- const deleteParams = [];
3995
- let deleteScope = this.#gcFilter(cutoffEpochTimestampMs, deleteParams);
3996
- if (step !== undefined) {
3997
- // Inclusive upper bound: completed_at ties may push a batch over batchSize, but never split across two.
3998
- deleteParams.push(watermark, step);
3999
- deleteScope = `${deleteScope} AND completed_at > $${deleteParams.length - 1} AND completed_at <= $${deleteParams.length}`;
4000
- }
4001
- // The final batch drops the watermark: unbounded, since an import can land a
4002
- // completed_at below it mid-pass.
4003
- await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE ${deleteScope}`, deleteParams);
4004
- return step;
4005
- }
4006
- finally {
4007
- this.#releaseRetentionClient(client);
4008
- }
4009
- }
4010
- /**
4011
- * Delete old terminal workflows throughout the system database, returning the cutoff
4012
- * actually used, or undefined when there is nothing to collect.
4013
- *
4014
- * Conductor sends cleared retention thresholds as JSON null, so every param is nullish.
4015
- */
4016
- async garbageCollect(cutoffEpochTimestampMs, rowsThreshold, options = {}) {
4017
- const batchSize = options.batchSize ?? exports.DEFAULT_GC_BATCH_SIZE;
4018
- // A NaN survives a bare `< 1` test and would only fail once it reached SQL, leaving GC half-applied.
4019
- if (!Number.isInteger(batchSize) || batchSize < 1) {
4020
- throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
4021
- }
4022
- if (rowsThreshold !== undefined && rowsThreshold !== null) {
4023
- // The completed_at of the rowsThreshold newest completed row
4024
- const result = await retryOnSerializationError(() => this.pool.query(`SELECT completed_at
4025
- FROM "${this.schemaName}".workflow_status
4026
- WHERE completed_at IS NOT NULL
4027
- ORDER BY completed_at DESC
4028
- LIMIT 1 OFFSET $1`, [rowsThreshold - 1]));
4029
- if (result.rows.length > 0) {
4030
- const rowsBasedCutoff = Number(result.rows[0].completed_at);
4031
- // Use the more restrictive cutoff (higher timestamp = more recent = more deletion)
4032
- if (cutoffEpochTimestampMs === undefined ||
4033
- cutoffEpochTimestampMs === null ||
4034
- rowsBasedCutoff > cutoffEpochTimestampMs) {
4035
- cutoffEpochTimestampMs = rowsBasedCutoff;
4036
- }
4037
- }
4038
- }
4039
- if (cutoffEpochTimestampMs === undefined || cutoffEpochTimestampMs === null) {
4040
- return undefined;
4041
- }
4042
- // Narrowed to a constant so the closures below keep it.
4043
- const cutoff = cutoffEpochTimestampMs;
4044
- // Advance a completed_at watermark, one committed transaction per batch, so a long
4045
- // history neither deletes in one transaction nor rescans what it already deleted.
4046
- const oldest = await retryOnSerializationError(async () => {
4047
- const params = [];
4048
- const scope = this.#gcFilter(cutoff, params);
4049
- const { rows } = await this.pool.query(`SELECT completed_at
4050
- FROM "${this.schemaName}".workflow_status
4051
- WHERE ${scope}
4052
- ORDER BY completed_at
4053
- LIMIT 1`, params);
4054
- return rows.length > 0 ? Number(rows[0].completed_at) : undefined;
4055
- });
4056
- let watermark = oldest === undefined ? 0 : oldest - 1;
4057
- for (;;) {
4058
- const next = await retryOnSerializationError(() => this.#garbageCollectBatch(cutoff, batchSize, watermark));
4059
- // Fewer than a full batch remained, so that delete took the rest.
4060
- if (next === undefined)
4061
- return cutoff;
4062
- watermark = next;
4063
- }
4064
- }
4065
- /**
4066
- * IDs of this application's in-flight workflows created at or before the cutoff.
4067
- * Claiming-scoped, so an upgrade still times out its own unclaimed workflows.
4068
- */
4069
- async listTimedOutWorkflowIds(cutoffEpochTimestampMs) {
4070
- const params = [
4071
- cutoffEpochTimestampMs,
4072
- workflow_1.StatusString.PENDING,
4073
- workflow_1.StatusString.ENQUEUED,
4074
- workflow_1.StatusString.DELAYED,
4075
- ];
4076
- const scope = this.#appNameFilter('application_name', this.appName, params);
4077
- const { rows } = await this.pool.query(`SELECT workflow_uuid
4078
- FROM "${this.schemaName}".workflow_status
4079
- WHERE created_at <= $1
4080
- AND status IN ($2, $3, $4)
4081
- AND ${scope}`, params);
4082
- return rows.map((row) => row.workflow_uuid);
4083
- }
4084
- async getMetrics(startTime, endTime, applicationName) {
4085
- const startEpochMs = new Date(startTime).getTime();
4086
- const endEpochMs = new Date(endTime).getTime();
4087
- const workflowParams = [startEpochMs, endEpochMs];
4088
- const workflowScope = this.#observabilityFilter('application_name', applicationName, workflowParams);
4089
- const stepParams = [startEpochMs, endEpochMs];
4090
- const stepScope = this.#observabilityFilter('application_name', applicationName, stepParams);
4091
- const [workflowResult, stepResult] = await this.observabilityQuery(async (client) => [
4092
- await client.query(`SELECT name, COUNT(workflow_uuid) as count
4093
- FROM "${this.schemaName}".workflow_status
4094
- WHERE created_at >= $1 AND created_at < $2 AND ${workflowScope}
4095
- GROUP BY name`, workflowParams),
4096
- await client.query(`SELECT function_name, COUNT(*) as count
4097
- FROM "${this.schemaName}".operation_outputs
4098
- WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2 AND ${stepScope}
4099
- GROUP BY function_name`, stepParams),
4100
- ]);
4101
- const metrics = [];
4102
- for (const row of workflowResult.rows) {
4103
- metrics.push({
4104
- metricType: 'workflow_count',
4105
- metricName: row.name,
4106
- value: Number(row.count),
4107
- });
4108
- }
4109
- for (const row of stepResult.rows) {
4110
- metrics.push({
4111
- metricType: 'step_count',
4112
- metricName: row.function_name,
4113
- value: Number(row.count),
4114
- });
4115
- }
4116
- return metrics;
4117
- }
4118
3137
  // ==================== Scheduling ====================
4119
3138
  async createSchedule(schedule, client) {
4120
3139
  const q = client ?? this.pool;
@@ -4180,7 +3199,7 @@ class SystemDatabase {
4180
3199
  conditions.push(`(${likeClauses.join(' OR ')})`);
4181
3200
  }
4182
3201
  // Unset scopes to this application, as on every other observability query.
4183
- conditions.push(this.#observabilityFilter('application_name', filters?.applicationName, params));
3202
+ conditions.push(this.observabilityFilter('application_name', filters?.applicationName, params));
4184
3203
  paramIdx = params.length + 1;
4185
3204
  const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
4186
3205
  const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
@@ -4247,7 +3266,7 @@ class SystemDatabase {
4247
3266
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE schedule_name = $2`, [lastFiredAt, name]);
4248
3267
  }
4249
3268
  async applySchedules(schedules) {
4250
- const client = await this.#connect();
3269
+ const client = await this.connect();
4251
3270
  try {
4252
3271
  await client.query('BEGIN');
4253
3272
  for (const sched of schedules) {
@@ -4299,7 +3318,7 @@ class SystemDatabase {
4299
3318
  */
4300
3319
  async createApplicationVersion(versionName, applicationName) {
4301
3320
  const owner = applicationName ?? this.appName;
4302
- const client = await this.#connect();
3321
+ const client = await this.connect();
4303
3322
  try {
4304
3323
  await client.query('BEGIN');
4305
3324
  // Claim a pre-upgrade row in place, so the version is not recreated or retimed.
@@ -4330,7 +3349,7 @@ class SystemDatabase {
4330
3349
  */
4331
3350
  async updateApplicationVersionTimestamp(versionName, newTimestamp, applicationName) {
4332
3351
  const owner = applicationName ?? this.appName;
4333
- const client = await this.#connect();
3352
+ const client = await this.connect();
4334
3353
  try {
4335
3354
  await client.query('BEGIN');
4336
3355
  const resolved = await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
@@ -4354,7 +3373,7 @@ class SystemDatabase {
4354
3373
  }
4355
3374
  async listApplicationVersions() {
4356
3375
  const params = [];
4357
- const scope = this.#appNameFilter('application_name', this.appName, params);
3376
+ const scope = this.appNameFilter('application_name', this.appName, params);
4358
3377
  const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
4359
3378
  FROM "${this.schemaName}".application_versions
4360
3379
  WHERE ${scope}
@@ -4368,7 +3387,7 @@ class SystemDatabase {
4368
3387
  async getLatestApplicationVersion(applicationName) {
4369
3388
  const owner = applicationName ?? this.appName;
4370
3389
  const params = [];
4371
- const scope = this.#appNameFilter('application_name', owner, params);
3390
+ const scope = this.appNameFilter('application_name', owner, params);
4372
3391
  const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
4373
3392
  FROM "${this.schemaName}".application_versions
4374
3393
  WHERE ${scope}
@@ -4392,7 +3411,7 @@ class SystemDatabase {
4392
3411
  */
4393
3412
  async listQueues(applicationName) {
4394
3413
  const params = [];
4395
- const scope = this.#observabilityFilter('application_name', applicationName, params);
3414
+ const scope = this.observabilityFilter('application_name', applicationName, params);
4396
3415
  const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
4397
3416
  FROM "${this.schemaName}".queues
4398
3417
  WHERE ${scope}`, params);
@@ -4440,7 +3459,7 @@ class SystemDatabase {
4440
3459
  application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
4441
3460
  : `ON CONFLICT (name) DO NOTHING`;
4442
3461
  const owner = record.applicationName ?? this.appName;
4443
- const client = await this.#connect();
3462
+ const client = await this.connect();
4444
3463
  try {
4445
3464
  await client.query('BEGIN');
4446
3465
  const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE name = $1`, [record.name]);
@@ -4565,7 +3584,7 @@ class SystemDatabase {
4565
3584
  throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
4566
3585
  }
4567
3586
  // Never a merge: queue, schedule, and version names are globally unique whatever their owner, so this cannot collide.
4568
- const client = await this.#connect();
3587
+ const client = await this.connect();
4569
3588
  let queues, schedules, versions, inFlight;
4570
3589
  try {
4571
3590
  await client.query('BEGIN');
@@ -4875,7 +3894,7 @@ class SystemDatabase {
4875
3894
  const startTimeMs = Date.now();
4876
3895
  // Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
4877
3896
  const endTimeMs = startTimeMs + Math.ceil(durationMS);
4878
- const client = await this.#connect();
3897
+ const client = await this.connect();
4879
3898
  try {
4880
3899
  const res = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
4881
3900
  if (res) {
@@ -4942,7 +3961,7 @@ class SystemDatabase {
4942
3961
  };
4943
3962
  let acquired = null;
4944
3963
  try {
4945
- const client = await this.#connect();
3964
+ const client = await this.connect();
4946
3965
  acquired = client;
4947
3966
  if (this.#abandonIfStopped(client))
4948
3967
  return;
@@ -5199,18 +4218,6 @@ __decorate([
5199
4218
  __metadata("design:paramtypes", [String]),
5200
4219
  __metadata("design:returntype", Promise)
5201
4220
  ], SystemDatabase.prototype, "getQueuePartitions", null);
5202
- __decorate([
5203
- dbRetry(),
5204
- __metadata("design:type", Function),
5205
- __metadata("design:paramtypes", [Number]),
5206
- __metadata("design:returntype", Promise)
5207
- ], SystemDatabase.prototype, "listTimedOutWorkflowIds", null);
5208
- __decorate([
5209
- dbRetry(),
5210
- __metadata("design:type", Function),
5211
- __metadata("design:paramtypes", [String, String, Array]),
5212
- __metadata("design:returntype", Promise)
5213
- ], SystemDatabase.prototype, "getMetrics", null);
5214
4221
  __decorate([
5215
4222
  dbRetry(),
5216
4223
  __metadata("design:type", Function),