@dbos-inc/dbos-sdk 5.3.6-preview → 5.3.7-preview
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/dbos-executor.d.ts +2 -2
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts.map +1 -1
- package/dist/src/dbos.js +45 -12
- package/dist/src/dbos.js.map +1 -1
- package/dist/src/enterprise.d.ts +26 -0
- package/dist/src/enterprise.d.ts.map +1 -0
- package/dist/src/enterprise.js +36 -0
- package/dist/src/enterprise.js.map +1 -0
- package/dist/src/system_database.d.ts +38 -133
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +96 -1094
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/workflow.d.ts +0 -3
- package/dist/src/workflow.d.ts.map +1 -1
- package/dist/src/workflow.js.map +1 -1
- package/dist/src/workflow_management.d.ts +0 -5
- package/dist/src/workflow_management.d.ts.map +1 -1
- package/dist/src/workflow_management.js +1 -35
- package/dist/src/workflow_management.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +10 -3
- package/dist/src/conductor/conductor.d.ts +0 -38
- package/dist/src/conductor/conductor.d.ts.map +0 -1
- package/dist/src/conductor/conductor.js +0 -944
- package/dist/src/conductor/conductor.js.map +0 -1
- package/dist/src/conductor/protocol.d.ts +0 -594
- package/dist/src/conductor/protocol.d.ts.map +0 -1
- package/dist/src/conductor/protocol.js +0 -447
- package/dist/src/conductor/protocol.js.map +0 -1
|
@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
9
9
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.SystemDatabase = exports.retriablePostgresException = exports.
|
|
12
|
+
exports.SystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
13
13
|
const dbos_executor_1 = require("./dbos-executor");
|
|
14
14
|
const pg_1 = require("pg");
|
|
15
15
|
const error_1 = require("./error");
|
|
@@ -88,8 +88,6 @@ function setsIdleTransactionTimeout(databaseUrl) {
|
|
|
88
88
|
}
|
|
89
89
|
// Workflows re-owned per transaction by a rename.
|
|
90
90
|
exports.DEFAULT_RENAME_BATCH_SIZE = 10_000;
|
|
91
|
-
// Rows deleted per transaction by garbage collection.
|
|
92
|
-
exports.DEFAULT_GC_BATCH_SIZE = 50_000;
|
|
93
91
|
const QUEUE_COLUMN_BY_FIELD = {
|
|
94
92
|
concurrency: 'concurrency',
|
|
95
93
|
workerConcurrency: 'worker_concurrency',
|
|
@@ -204,16 +202,6 @@ async function releaseSystemDatabaseClient(client, customPool) {
|
|
|
204
202
|
}
|
|
205
203
|
catch (e) { }
|
|
206
204
|
}
|
|
207
|
-
/**
|
|
208
|
-
* The pg_try_advisory_lock argument for one schema's retention round: the leading 8 bytes of
|
|
209
|
-
* SHA-256 over a fixed string, read as a signed big-endian 64-bit integer. Separate locks for
|
|
210
|
-
* separate schemas. Every DBOS SDK derives the key this way, so rounds in different languages
|
|
211
|
-
* against one system database contend for the same lock; changing it here changes it everywhere.
|
|
212
|
-
*/
|
|
213
|
-
function retentionLockKey(schemaName) {
|
|
214
|
-
return advisoryLockKey(`dbos.retention.${schemaName}`);
|
|
215
|
-
}
|
|
216
|
-
exports.retentionLockKey = retentionLockKey;
|
|
217
205
|
/** A Postgres advisory lock key: the leading 8 bytes of SHA-256 over `lockName`, as a signed big-endian integer. */
|
|
218
206
|
function advisoryLockKey(lockName) {
|
|
219
207
|
return (0, crypto_1.createHash)('sha256').update(lockName).digest().readBigInt64BE(0);
|
|
@@ -223,6 +211,7 @@ async function isCockroachDB(client) {
|
|
|
223
211
|
const versionRes = await client.query('SELECT version() AS version');
|
|
224
212
|
return /cockroachdb/i.test(versionRes.rows[0]?.version ?? '');
|
|
225
213
|
}
|
|
214
|
+
exports.isCockroachDB = isCockroachDB;
|
|
226
215
|
async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
|
|
227
216
|
if (!customPool) {
|
|
228
217
|
// A custom pool means the database already exists; otherwise, create it if it does not.
|
|
@@ -378,14 +367,6 @@ const RETRY_SQLSTATE_CODES = new Set([
|
|
|
378
367
|
'25P03', // idle_in_transaction_session_timeout, when the kill lands on an in-flight query; an idle kill surfaces as a message match below
|
|
379
368
|
'40P01', // deadlock_detected: the victim's transaction rolled back, so rerunning it is safe
|
|
380
369
|
]);
|
|
381
|
-
/**
|
|
382
|
-
* Bulk maintenance retries these a bounded number of times. 40001 is kept out of the sets above,
|
|
383
|
-
* which feed `dbRetry` and so retry forever; every other path runs READ COMMITTED and never sees it.
|
|
384
|
-
*/
|
|
385
|
-
const SERIALIZATION_SQLSTATE_CODES = new Set([
|
|
386
|
-
'40001', // serialization_failure (MVCC conflict)
|
|
387
|
-
'40P01', // deadlock_detected
|
|
388
|
-
]);
|
|
389
370
|
// Node.js transient network error codes (system call level)
|
|
390
371
|
const RETRY_NODE_ERRNOS = new Set([
|
|
391
372
|
'ECONNRESET',
|
|
@@ -400,6 +381,7 @@ function isPgDatabaseError(e) {
|
|
|
400
381
|
// Matched by shape, not instanceof: a user-supplied pool may throw another pg copy's DatabaseError.
|
|
401
382
|
return !!e && typeof e === 'object' && typeof e.code === 'string' && e.code.length === 5;
|
|
402
383
|
}
|
|
384
|
+
exports.isPgDatabaseError = isPgDatabaseError;
|
|
403
385
|
function sqlStateLooksRetryable(sqlstate) {
|
|
404
386
|
if (!sqlstate)
|
|
405
387
|
return false;
|
|
@@ -449,6 +431,7 @@ function* unwrapErrors(e) {
|
|
|
449
431
|
yield cur;
|
|
450
432
|
}
|
|
451
433
|
}
|
|
434
|
+
exports.unwrapErrors = unwrapErrors;
|
|
452
435
|
// "What could possibly go wrong?"
|
|
453
436
|
function retriablePostgresException(err) {
|
|
454
437
|
// Dig into AggregateErrors of various types
|
|
@@ -481,45 +464,45 @@ exports.retriablePostgresException = retriablePostgresException;
|
|
|
481
464
|
function isStatementTimeout(err) {
|
|
482
465
|
return !!err && typeof err === 'object' && err.code === '57014';
|
|
483
466
|
}
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
}
|
|
493
|
-
/**
|
|
494
|
-
* Re-run a batch that lost a deadlock or serialization race. The database already rolled it
|
|
495
|
-
* back, so replaying it is safe.
|
|
496
|
-
*/
|
|
497
|
-
async function retryOnSerializationError(operation) {
|
|
498
|
-
const maxAttempts = 10;
|
|
499
|
-
const maxBackoff = 2.0;
|
|
500
|
-
let backoff = 0.05;
|
|
501
|
-
for (let attempt = 1;; attempt++) {
|
|
467
|
+
/** Run `operation` under the {@link dbRetry} policy, for code that cannot take the decorator. */
|
|
468
|
+
async function withDbRetry(operation, options = {}) {
|
|
469
|
+
// Read the defaults per call so the backoff stays tunable after the decorator is applied.
|
|
470
|
+
const maxBackoff = options.maxBackoff ?? utils_1.dbRetryConfig.maxBackoffSec;
|
|
471
|
+
let retries = 0;
|
|
472
|
+
let backoff = options.initialBackoff ?? utils_1.dbRetryConfig.initialBackoffSec;
|
|
473
|
+
const abortSignal = (0, context_1.currentDbRetryAbortSignal)();
|
|
474
|
+
while (true) {
|
|
502
475
|
try {
|
|
503
476
|
return await operation();
|
|
504
477
|
}
|
|
505
478
|
catch (e) {
|
|
506
|
-
if (
|
|
507
|
-
|
|
479
|
+
if (retriablePostgresException(e)) {
|
|
480
|
+
if (abortSignal?.aborted)
|
|
481
|
+
throw e;
|
|
482
|
+
retries++;
|
|
483
|
+
// Calculate backoff with jitter
|
|
484
|
+
const actualBackoff = backoff * (0.5 + Math.random());
|
|
485
|
+
dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Database connection failed: ${e instanceof Error ? e.message : String(e)}. ` +
|
|
486
|
+
`Retrying in ${actualBackoff.toFixed(2)}s (attempt ${retries})`);
|
|
487
|
+
// Sleep with backoff
|
|
488
|
+
if (abortSignal) {
|
|
489
|
+
await (0, utils_1.interruptibleSleep)(actualBackoff * 1000, abortSignal);
|
|
490
|
+
if (abortSignal.aborted)
|
|
491
|
+
throw e;
|
|
492
|
+
}
|
|
493
|
+
else {
|
|
494
|
+
await (0, utils_1.sleepms)(actualBackoff * 1000); // Convert to milliseconds
|
|
495
|
+
}
|
|
496
|
+
// Increase backoff for next attempt (exponential)
|
|
497
|
+
backoff = Math.min(backoff * 2, maxBackoff);
|
|
508
498
|
}
|
|
509
|
-
|
|
510
|
-
if (attempt === maxAttempts) {
|
|
511
|
-
dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Garbage collection failed after ${maxAttempts} attempts: ${message}`);
|
|
499
|
+
else {
|
|
512
500
|
throw e;
|
|
513
501
|
}
|
|
514
|
-
// Jittered backoff, so peers that collided do not collide again
|
|
515
|
-
const actualBackoff = backoff * (0.5 + Math.random());
|
|
516
|
-
dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Contention or deadlock detected in workflow garbage collection: ${message}. ` +
|
|
517
|
-
`Retrying in ${actualBackoff.toFixed(2)}s (attempt ${attempt})`);
|
|
518
|
-
await (0, utils_1.sleepms)(actualBackoff * 1000);
|
|
519
|
-
backoff = Math.min(backoff * 2, maxBackoff);
|
|
520
502
|
}
|
|
521
503
|
}
|
|
522
504
|
}
|
|
505
|
+
exports.withDbRetry = withDbRetry;
|
|
523
506
|
/**
|
|
524
507
|
* If a workflow encounters a database connection issue while performing an operation,
|
|
525
508
|
* block the workflow and retry the operation until it reconnects and succeeds.
|
|
@@ -533,41 +516,7 @@ function dbRetry(options = {}) {
|
|
|
533
516
|
return function (target, propertyName, descriptor) {
|
|
534
517
|
const method = descriptor.value;
|
|
535
518
|
descriptor.value = async function (...args) {
|
|
536
|
-
|
|
537
|
-
const maxBackoff = options.maxBackoff ?? utils_1.dbRetryConfig.maxBackoffSec;
|
|
538
|
-
let retries = 0;
|
|
539
|
-
let backoff = options.initialBackoff ?? utils_1.dbRetryConfig.initialBackoffSec;
|
|
540
|
-
const abortSignal = (0, context_1.currentDbRetryAbortSignal)();
|
|
541
|
-
while (true) {
|
|
542
|
-
try {
|
|
543
|
-
return await method.apply(this, args);
|
|
544
|
-
}
|
|
545
|
-
catch (e) {
|
|
546
|
-
if (retriablePostgresException(e)) {
|
|
547
|
-
if (abortSignal?.aborted)
|
|
548
|
-
throw e;
|
|
549
|
-
retries++;
|
|
550
|
-
// Calculate backoff with jitter
|
|
551
|
-
const actualBackoff = backoff * (0.5 + Math.random());
|
|
552
|
-
dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Database connection failed: ${e instanceof Error ? e.message : String(e)}. ` +
|
|
553
|
-
`Retrying in ${actualBackoff.toFixed(2)}s (attempt ${retries})`);
|
|
554
|
-
// Sleep with backoff
|
|
555
|
-
if (abortSignal) {
|
|
556
|
-
await (0, utils_1.interruptibleSleep)(actualBackoff * 1000, abortSignal);
|
|
557
|
-
if (abortSignal.aborted)
|
|
558
|
-
throw e;
|
|
559
|
-
}
|
|
560
|
-
else {
|
|
561
|
-
await (0, utils_1.sleepms)(actualBackoff * 1000); // Convert to milliseconds
|
|
562
|
-
}
|
|
563
|
-
// Increase backoff for next attempt (exponential)
|
|
564
|
-
backoff = Math.min(backoff * 2, maxBackoff);
|
|
565
|
-
}
|
|
566
|
-
else {
|
|
567
|
-
throw e;
|
|
568
|
-
}
|
|
569
|
-
}
|
|
570
|
-
}
|
|
519
|
+
return withDbRetry(() => method.apply(this, args), options);
|
|
571
520
|
};
|
|
572
521
|
return descriptor;
|
|
573
522
|
};
|
|
@@ -649,11 +598,12 @@ class SystemDatabase {
|
|
|
649
598
|
#batchCreatedAtCursors = new Map();
|
|
650
599
|
// Set by destroy(), so polling waits end instead of running on against a pool that outlives this handle.
|
|
651
600
|
#destroyed = false;
|
|
652
|
-
|
|
653
|
-
|
|
601
|
+
get destroyed() {
|
|
602
|
+
return this.#destroyed;
|
|
603
|
+
}
|
|
654
604
|
// Connections a retention round holds right now. destroy() cuts them, since closing the
|
|
655
605
|
// pool would otherwise wait on the lock session and on any statement in flight.
|
|
656
|
-
|
|
606
|
+
retentionClients = new Set();
|
|
657
607
|
constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS,
|
|
658
608
|
// The application this handle acts for; undefined writes unclaimed rows.
|
|
659
609
|
appName, observabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS, idleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS) {
|
|
@@ -713,25 +663,9 @@ class SystemDatabase {
|
|
|
713
663
|
this.logger.warn(`Unexpected error on a system database connection: ${err}`);
|
|
714
664
|
};
|
|
715
665
|
/** Check out a pool connection guarded for as long as we hold it. See {@link borrowClient}. */
|
|
716
|
-
|
|
666
|
+
connect() {
|
|
717
667
|
return borrowClient(this.pool, this.#onClientError);
|
|
718
668
|
}
|
|
719
|
-
/** Borrow a connection for a retention round, so destroy() can cut it. */
|
|
720
|
-
async #borrowRetentionClient() {
|
|
721
|
-
if (this.#destroyed) {
|
|
722
|
-
throw new Error('System database shutting down');
|
|
723
|
-
}
|
|
724
|
-
const client = await this.#connect();
|
|
725
|
-
this.#retentionClients.add(client);
|
|
726
|
-
return client;
|
|
727
|
-
}
|
|
728
|
-
/** Return a retention connection, unless destroy() already cut it: a second release would throw. */
|
|
729
|
-
#releaseRetentionClient(client) {
|
|
730
|
-
if (this.#retentionClients.delete(client)) {
|
|
731
|
-
// No error argument: a genuinely dead connection is still evicted by the pool's own check.
|
|
732
|
-
client.release();
|
|
733
|
-
}
|
|
734
|
-
}
|
|
735
669
|
/**
|
|
736
670
|
* Cap an introspection read with a statement timeout, so one scanning a huge table cannot hold a
|
|
737
671
|
* snapshot for minutes and stall autovacuum database-wide. Soft-private so tests can assert the cap
|
|
@@ -742,7 +676,7 @@ class SystemDatabase {
|
|
|
742
676
|
*/
|
|
743
677
|
async observabilityQuery(fn) {
|
|
744
678
|
const timeoutMs = this.observabilityQueryTimeoutMs;
|
|
745
|
-
const client = await this
|
|
679
|
+
const client = await this.connect();
|
|
746
680
|
try {
|
|
747
681
|
if (timeoutMs === undefined) {
|
|
748
682
|
return await fn(client);
|
|
@@ -771,7 +705,7 @@ class SystemDatabase {
|
|
|
771
705
|
* A predicate matching rows owned by these applications plus unclaimed ones, which belong to
|
|
772
706
|
* every application; unset or empty matches everything. Appends its bind parameter to `params`.
|
|
773
707
|
*/
|
|
774
|
-
|
|
708
|
+
appNameFilter(column, value, params) {
|
|
775
709
|
// An empty name is no name: it is not a value any application could be configured with.
|
|
776
710
|
const names = !value ? [] : Array.isArray(value) ? value : [value];
|
|
777
711
|
if (names.length === 0)
|
|
@@ -784,8 +718,8 @@ class SystemDatabase {
|
|
|
784
718
|
* this application owns, not to every application's rows. A handle with no application of its
|
|
785
719
|
* own still matches every one.
|
|
786
720
|
*/
|
|
787
|
-
|
|
788
|
-
return this
|
|
721
|
+
observabilityFilter(column, value, params) {
|
|
722
|
+
return this.appNameFilter(column, value ?? this.appName, params);
|
|
789
723
|
}
|
|
790
724
|
/**
|
|
791
725
|
* The version name a dequeue treats as latest: this application's own plus unclaimed
|
|
@@ -793,7 +727,7 @@ class SystemDatabase {
|
|
|
793
727
|
*/
|
|
794
728
|
async #latestApplicationVersionName(client) {
|
|
795
729
|
const params = [];
|
|
796
|
-
const scope = this
|
|
730
|
+
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
797
731
|
const { rows } = await client.query(`SELECT version_name
|
|
798
732
|
FROM "${this.schemaName}".application_versions
|
|
799
733
|
WHERE ${scope}
|
|
@@ -856,7 +790,7 @@ class SystemDatabase {
|
|
|
856
790
|
// A retention round still running is cut here rather than waited for: returned with an
|
|
857
791
|
// error, each connection is destroyed at once, the idle lock session lets go of the
|
|
858
792
|
// advisory lock, and the round fails on its next statement instead of holding shutdown.
|
|
859
|
-
for (const client of this
|
|
793
|
+
for (const client of this.retentionClients) {
|
|
860
794
|
// Cover the release() call itself, which tears the connection down and can surface a socket error.
|
|
861
795
|
client.on('error', () => { });
|
|
862
796
|
try {
|
|
@@ -866,7 +800,7 @@ class SystemDatabase {
|
|
|
866
800
|
this.logger.warn(`Error releasing a retention connection: ${String(e)}`);
|
|
867
801
|
}
|
|
868
802
|
}
|
|
869
|
-
this
|
|
803
|
+
this.retentionClients.clear();
|
|
870
804
|
// We attached nothing to the pool object itself, so there is nothing to unpick; only close one we own.
|
|
871
805
|
if (!this.customPool) {
|
|
872
806
|
await this.pool.end();
|
|
@@ -881,7 +815,7 @@ class SystemDatabase {
|
|
|
881
815
|
return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
|
|
882
816
|
}
|
|
883
817
|
async initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy) {
|
|
884
|
-
const client = await this
|
|
818
|
+
const client = await this.connect();
|
|
885
819
|
let shouldCommit = false;
|
|
886
820
|
try {
|
|
887
821
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
@@ -907,7 +841,7 @@ class SystemDatabase {
|
|
|
907
841
|
}
|
|
908
842
|
/** Insert a child workflow's row and record it as the parent's step in one transaction, so a crash leaves both or neither. */
|
|
909
843
|
async initChildWorkflowStatus(initStatus, creatorXid, parentWorkflowID, parentFunctionID, startTimeEpochMs, endTimeEpochMs, reusePolicy = 'return-existing') {
|
|
910
|
-
const client = await this
|
|
844
|
+
const client = await this.connect();
|
|
911
845
|
let shouldCommit = false;
|
|
912
846
|
try {
|
|
913
847
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
@@ -1084,7 +1018,7 @@ class SystemDatabase {
|
|
|
1084
1018
|
'schedule_name',
|
|
1085
1019
|
'application_name',
|
|
1086
1020
|
];
|
|
1087
|
-
const client = await this
|
|
1021
|
+
const client = await this.connect();
|
|
1088
1022
|
try {
|
|
1089
1023
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1090
1024
|
// Chunk to stay well under the bind-parameter limit.
|
|
@@ -1137,7 +1071,7 @@ class SystemDatabase {
|
|
|
1137
1071
|
return inserted;
|
|
1138
1072
|
}
|
|
1139
1073
|
async recordWorkflowOutput(workflowID, status, ownerXid) {
|
|
1140
|
-
const client = await this
|
|
1074
|
+
const client = await this.connect();
|
|
1141
1075
|
try {
|
|
1142
1076
|
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
|
|
1143
1077
|
}
|
|
@@ -1146,7 +1080,7 @@ class SystemDatabase {
|
|
|
1146
1080
|
}
|
|
1147
1081
|
}
|
|
1148
1082
|
async recordWorkflowError(workflowID, status, ownerXid) {
|
|
1149
|
-
const client = await this
|
|
1083
|
+
const client = await this.connect();
|
|
1150
1084
|
try {
|
|
1151
1085
|
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
|
|
1152
1086
|
}
|
|
@@ -1194,22 +1128,11 @@ class SystemDatabase {
|
|
|
1194
1128
|
}
|
|
1195
1129
|
}
|
|
1196
1130
|
}
|
|
1197
|
-
async getPendingWorkflows(executorID, appVersion) {
|
|
1198
|
-
const params = [workflow_1.StatusString.PENDING, executorID, appVersion];
|
|
1199
|
-
// executor_id defaults to "local", so it collides across applications.
|
|
1200
|
-
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
1201
|
-
const getWorkflows = await this.pool.query(`SELECT workflow_uuid
|
|
1202
|
-
FROM "${this.schemaName}".workflow_status
|
|
1203
|
-
WHERE status=$1 AND executor_id=$2 AND application_version=$3 AND ${scope}`, params);
|
|
1204
|
-
return getWorkflows.rows.map((i) => ({
|
|
1205
|
-
workflowUUID: i.workflow_uuid,
|
|
1206
|
-
}));
|
|
1207
|
-
}
|
|
1208
1131
|
// Recovery re-enqueues rather than executing directly so the queue's atomic dequeue admits exactly one runner, and the executor ID predicate rejects sweeps for rows a live executor has already claimed.
|
|
1209
1132
|
async reenqueueWorkflowsForRecovery(executorID, appVersion, recoveryQueueName) {
|
|
1210
1133
|
const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
|
|
1211
1134
|
// executor_id defaults to "local", so it collides across applications.
|
|
1212
|
-
const scope = this
|
|
1135
|
+
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
1213
1136
|
const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1214
1137
|
SET started_at_epoch_ms = NULL,
|
|
1215
1138
|
status = $1,
|
|
@@ -1230,7 +1153,7 @@ class SystemDatabase {
|
|
|
1230
1153
|
return status ? JSON.stringify(status) : null;
|
|
1231
1154
|
};
|
|
1232
1155
|
if (callerID && callerFN) {
|
|
1233
|
-
const client = await this
|
|
1156
|
+
const client = await this.connect();
|
|
1234
1157
|
try {
|
|
1235
1158
|
// Check if the operation has been done before for OAOO (only do this inside a workflow).
|
|
1236
1159
|
const json = await this.#inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, funcGetStatus));
|
|
@@ -1266,7 +1189,7 @@ class SystemDatabase {
|
|
|
1266
1189
|
}
|
|
1267
1190
|
// Only used in tests
|
|
1268
1191
|
async setWorkflowStatus(workflowID, status, resetRecoveryAttempts, internalOptions) {
|
|
1269
|
-
const client = await this
|
|
1192
|
+
const client = await this.connect();
|
|
1270
1193
|
try {
|
|
1271
1194
|
await this.updateWorkflowStatus(client, workflowID, status, {
|
|
1272
1195
|
update: {
|
|
@@ -1284,7 +1207,7 @@ class SystemDatabase {
|
|
|
1284
1207
|
}
|
|
1285
1208
|
// ==================== Step Results ====================
|
|
1286
1209
|
async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
|
|
1287
|
-
const client = await this
|
|
1210
|
+
const client = await this.connect();
|
|
1288
1211
|
try {
|
|
1289
1212
|
return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
1290
1213
|
}
|
|
@@ -1307,7 +1230,7 @@ class SystemDatabase {
|
|
|
1307
1230
|
return rows;
|
|
1308
1231
|
}
|
|
1309
1232
|
async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
1310
|
-
const client = await this
|
|
1233
|
+
const client = await this.connect();
|
|
1311
1234
|
try {
|
|
1312
1235
|
await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options));
|
|
1313
1236
|
}
|
|
@@ -1317,7 +1240,7 @@ class SystemDatabase {
|
|
|
1317
1240
|
}
|
|
1318
1241
|
}
|
|
1319
1242
|
async runTransactionalStep(workflowID, functionID, functionName, callback) {
|
|
1320
|
-
const client = await this
|
|
1243
|
+
const client = await this.connect();
|
|
1321
1244
|
try {
|
|
1322
1245
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1323
1246
|
const existing = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
@@ -1363,7 +1286,7 @@ class SystemDatabase {
|
|
|
1363
1286
|
}
|
|
1364
1287
|
// Nondeprecated - skip matching entry, unpatched if nonmatching entry,
|
|
1365
1288
|
// If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
|
|
1366
|
-
const client = await this
|
|
1289
|
+
const client = await this.connect();
|
|
1367
1290
|
try {
|
|
1368
1291
|
return await this.#inTransaction(client, async () => {
|
|
1369
1292
|
const checkpointName = await readCheckpointName(client);
|
|
@@ -1410,7 +1333,7 @@ class SystemDatabase {
|
|
|
1410
1333
|
await this.#checkIfCanceled(this.pool, workflowID);
|
|
1411
1334
|
}
|
|
1412
1335
|
async resumeWorkflows(workflowIDs, queueName) {
|
|
1413
|
-
const client = await this
|
|
1336
|
+
const client = await this.connect();
|
|
1414
1337
|
try {
|
|
1415
1338
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1416
1339
|
// Check existence separately: a zero-row update also means "already complete", a legal no-op.
|
|
@@ -1474,7 +1397,7 @@ class SystemDatabase {
|
|
|
1474
1397
|
return await this.debounceDelayedWorkflowStandalone(params);
|
|
1475
1398
|
}
|
|
1476
1399
|
async debounceDelayedWorkflowStandalone(params) {
|
|
1477
|
-
const client = await this
|
|
1400
|
+
const client = await this.connect();
|
|
1478
1401
|
try {
|
|
1479
1402
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1480
1403
|
const result = await this.#debounceDelayedWorkflowInternal(client, params);
|
|
@@ -1502,7 +1425,7 @@ class SystemDatabase {
|
|
|
1502
1425
|
params.applicationName ?? null,
|
|
1503
1426
|
];
|
|
1504
1427
|
// Never extend a workflow the target application doesn't own; falls through to the holder below.
|
|
1505
|
-
const ownScope = this
|
|
1428
|
+
const ownScope = this.appNameFilter('application_name', params.applicationName, updateParams);
|
|
1506
1429
|
const updated = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1507
1430
|
SET delay_until_epoch_ms = CASE
|
|
1508
1431
|
WHEN debounce_deadline_epoch_ms IS NOT NULL AND debounce_deadline_epoch_ms < $1
|
|
@@ -1584,7 +1507,7 @@ class SystemDatabase {
|
|
|
1584
1507
|
allIds.push(...(await this.getWorkflowChildren(wfid)));
|
|
1585
1508
|
}
|
|
1586
1509
|
}
|
|
1587
|
-
const client = await this
|
|
1510
|
+
const client = await this.connect();
|
|
1588
1511
|
try {
|
|
1589
1512
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1590
1513
|
// The payload tables carry no foreign key, so the status delete does not cascade
|
|
@@ -1630,7 +1553,7 @@ class SystemDatabase {
|
|
|
1630
1553
|
throw new error_1.DBOSError(`startStep must be >= 0, got ${startStep}`);
|
|
1631
1554
|
}
|
|
1632
1555
|
const schema = this.schemaName;
|
|
1633
|
-
const client = await this
|
|
1556
|
+
const client = await this.connect();
|
|
1634
1557
|
try {
|
|
1635
1558
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1636
1559
|
const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE workflow_uuid = $1`, [workflowID]);
|
|
@@ -1712,62 +1635,6 @@ class SystemDatabase {
|
|
|
1712
1635
|
const result = await this.bulkForkWorkflows([workflowID], [newWorkflowID], [startStep], options);
|
|
1713
1636
|
return result[0];
|
|
1714
1637
|
}
|
|
1715
|
-
async forkFromFailure(workflowIDs, options = {}) {
|
|
1716
|
-
const modes = [
|
|
1717
|
-
options.fromLastFailure ?? false,
|
|
1718
|
-
options.fromLastStep ?? false,
|
|
1719
|
-
options.fromStep !== undefined,
|
|
1720
|
-
options.fromStepName !== undefined,
|
|
1721
|
-
].filter(Boolean).length;
|
|
1722
|
-
if (modes !== 1) {
|
|
1723
|
-
throw new Error('Exactly one of fromLastFailure, fromLastStep, fromStep, or fromStepName must be specified');
|
|
1724
|
-
}
|
|
1725
|
-
let startSteps;
|
|
1726
|
-
if (options.fromStep !== undefined) {
|
|
1727
|
-
startSteps = Array(workflowIDs.length).fill(options.fromStep);
|
|
1728
|
-
}
|
|
1729
|
-
else {
|
|
1730
|
-
let query;
|
|
1731
|
-
const params = [workflowIDs];
|
|
1732
|
-
if (options.fromLastFailure) {
|
|
1733
|
-
query = `SELECT workflow_uuid,
|
|
1734
|
-
COALESCE(
|
|
1735
|
-
MAX(function_id) FILTER (WHERE error IS NOT NULL),
|
|
1736
|
-
MAX(function_id)
|
|
1737
|
-
) AS start_step
|
|
1738
|
-
FROM "${this.schemaName}".operation_outputs
|
|
1739
|
-
WHERE workflow_uuid = ANY($1)
|
|
1740
|
-
GROUP BY workflow_uuid`;
|
|
1741
|
-
}
|
|
1742
|
-
else if (options.fromLastStep) {
|
|
1743
|
-
query = `SELECT workflow_uuid, MAX(function_id) AS start_step
|
|
1744
|
-
FROM "${this.schemaName}".operation_outputs
|
|
1745
|
-
WHERE workflow_uuid = ANY($1)
|
|
1746
|
-
GROUP BY workflow_uuid`;
|
|
1747
|
-
}
|
|
1748
|
-
else {
|
|
1749
|
-
// fromStepName
|
|
1750
|
-
query = `SELECT workflow_uuid, MAX(function_id) AS start_step
|
|
1751
|
-
FROM "${this.schemaName}".operation_outputs
|
|
1752
|
-
WHERE workflow_uuid = ANY($1) AND function_name = $2
|
|
1753
|
-
GROUP BY workflow_uuid`;
|
|
1754
|
-
params.push(options.fromStepName);
|
|
1755
|
-
}
|
|
1756
|
-
const result = await this.pool.query(query, params);
|
|
1757
|
-
const startStepByID = new Map(result.rows.map((r) => [r.workflow_uuid, Number(r.start_step)]));
|
|
1758
|
-
if (options.fromStepName !== undefined) {
|
|
1759
|
-
for (const wid of workflowIDs) {
|
|
1760
|
-
if (!startStepByID.has(wid)) {
|
|
1761
|
-
throw new Error(`Workflow ${wid} has no step named '${options.fromStepName}'`);
|
|
1762
|
-
}
|
|
1763
|
-
}
|
|
1764
|
-
}
|
|
1765
|
-
// A workflow with no recorded steps has nothing to resume from, so restart it from the beginning.
|
|
1766
|
-
startSteps = workflowIDs.map((wid) => startStepByID.get(wid) ?? 0);
|
|
1767
|
-
}
|
|
1768
|
-
const forkedIDs = workflowIDs.map(() => (0, crypto_1.randomUUID)());
|
|
1769
|
-
return this.bulkForkWorkflows(workflowIDs, forkedIDs, startSteps, options);
|
|
1770
|
-
}
|
|
1771
1638
|
async bulkForkWorkflows(originalWorkflowIDs, forkedWorkflowIDs, startSteps, options = {}) {
|
|
1772
1639
|
if (originalWorkflowIDs.length === 0) {
|
|
1773
1640
|
return [];
|
|
@@ -1775,7 +1642,7 @@ class SystemDatabase {
|
|
|
1775
1642
|
if (originalWorkflowIDs.length !== forkedWorkflowIDs.length || originalWorkflowIDs.length !== startSteps.length) {
|
|
1776
1643
|
throw new Error('originalWorkflowIDs, forkedWorkflowIDs, and startSteps must have the same length');
|
|
1777
1644
|
}
|
|
1778
|
-
const client = await this
|
|
1645
|
+
const client = await this.connect();
|
|
1779
1646
|
try {
|
|
1780
1647
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1781
1648
|
// Fetch the status of all original workflows inside the transaction.
|
|
@@ -1927,188 +1794,6 @@ class SystemDatabase {
|
|
|
1927
1794
|
client.release();
|
|
1928
1795
|
}
|
|
1929
1796
|
}
|
|
1930
|
-
async exportWorkflow(workflowID, exportChildren = false) {
|
|
1931
|
-
const workflowIDs = [workflowID];
|
|
1932
|
-
if (exportChildren) {
|
|
1933
|
-
workflowIDs.push(...(await this.getWorkflowChildren(workflowID)));
|
|
1934
|
-
}
|
|
1935
|
-
const exportedWorkflows = [];
|
|
1936
|
-
const client = await this.#connect();
|
|
1937
|
-
try {
|
|
1938
|
-
for (const wfID of workflowIDs) {
|
|
1939
|
-
// Export workflow_status
|
|
1940
|
-
const statusResult = await client.query(
|
|
1941
|
-
// creator_xid and owner_xid are intentionally omitted: they are transient
|
|
1942
|
-
// tokens, not logical workflow state, and a source database's
|
|
1943
|
-
// tokens are meaningless in the target.
|
|
1944
|
-
`SELECT
|
|
1945
|
-
ws.workflow_uuid, ws.status, ws.name, ws.authenticated_user, ws.assumed_role,
|
|
1946
|
-
ws.authenticated_roles,
|
|
1947
|
-
COALESCE(wo.output, ws.output) AS output, COALESCE(wo.error, ws.error) AS error,
|
|
1948
|
-
ws.executor_id,
|
|
1949
|
-
ws.created_at, ws.updated_at, ws.application_version, ws.application_id,
|
|
1950
|
-
ws.class_name, ws.config_name, ws.recovery_attempts, ws.queue_name,
|
|
1951
|
-
ws.workflow_timeout_ms, ws.workflow_deadline_epoch_ms, ws.started_at_epoch_ms,
|
|
1952
|
-
ws.deduplication_id, COALESCE(wi.inputs, ws.inputs) AS inputs,
|
|
1953
|
-
ws.priority, ws.queue_partition_key, ws.forked_from,
|
|
1954
|
-
ws.parent_workflow_id, ws.serialization, ws.delay_until_epoch_ms,
|
|
1955
|
-
ws.was_forked_from, ws.rate_limited, ws.completed_at, ws.attributes, ws.schedule_name,
|
|
1956
|
-
ws.debounce_deadline_epoch_ms, ws.is_debounced, ws.application_name
|
|
1957
|
-
FROM "${this.schemaName}".workflow_status ws
|
|
1958
|
-
LEFT JOIN "${this.schemaName}".workflow_input wi ON wi.workflow_uuid = ws.workflow_uuid
|
|
1959
|
-
LEFT JOIN "${this.schemaName}".workflow_output wo ON wo.workflow_uuid = ws.workflow_uuid
|
|
1960
|
-
WHERE ws.workflow_uuid = $1`, [wfID]);
|
|
1961
|
-
if (statusResult.rows.length === 0) {
|
|
1962
|
-
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${wfID} does not exist`);
|
|
1963
|
-
}
|
|
1964
|
-
const workflowStatus = statusResult.rows[0];
|
|
1965
|
-
// Export operation_outputs
|
|
1966
|
-
const outputsResult = await client.query(`SELECT
|
|
1967
|
-
workflow_uuid, function_id, function_name, output, error,
|
|
1968
|
-
child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
|
|
1969
|
-
serialization, application_name
|
|
1970
|
-
FROM "${this.schemaName}".operation_outputs
|
|
1971
|
-
WHERE workflow_uuid = $1`, [wfID]);
|
|
1972
|
-
// Export workflow_events
|
|
1973
|
-
const eventsResult = await client.query(`SELECT workflow_uuid, key, value, serialization
|
|
1974
|
-
FROM "${this.schemaName}".workflow_events
|
|
1975
|
-
WHERE workflow_uuid = $1`, [wfID]);
|
|
1976
|
-
// Export workflow_events_history
|
|
1977
|
-
const historyResult = await client.query(`SELECT workflow_uuid, function_id, key, value, serialization
|
|
1978
|
-
FROM "${this.schemaName}".workflow_events_history
|
|
1979
|
-
WHERE workflow_uuid = $1`, [wfID]);
|
|
1980
|
-
// Export streams
|
|
1981
|
-
const streamsResult = await client.query(`SELECT workflow_uuid, key, value, "offset", function_id, serialization
|
|
1982
|
-
FROM "${this.schemaName}".streams
|
|
1983
|
-
WHERE workflow_uuid = $1`, [wfID]);
|
|
1984
|
-
exportedWorkflows.push({
|
|
1985
|
-
workflow_status: workflowStatus,
|
|
1986
|
-
operation_outputs: outputsResult.rows,
|
|
1987
|
-
workflow_events: eventsResult.rows,
|
|
1988
|
-
workflow_events_history: historyResult.rows,
|
|
1989
|
-
streams: streamsResult.rows,
|
|
1990
|
-
});
|
|
1991
|
-
}
|
|
1992
|
-
}
|
|
1993
|
-
finally {
|
|
1994
|
-
client.release();
|
|
1995
|
-
}
|
|
1996
|
-
return exportedWorkflows;
|
|
1997
|
-
}
|
|
1998
|
-
async importWorkflow(workflows) {
|
|
1999
|
-
const client = await this.#connect();
|
|
2000
|
-
try {
|
|
2001
|
-
await client.query('BEGIN');
|
|
2002
|
-
for (const workflow of workflows) {
|
|
2003
|
-
const status = workflow.workflow_status;
|
|
2004
|
-
// Import workflow_status
|
|
2005
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
|
|
2006
|
-
workflow_uuid, status, name, authenticated_user, assumed_role,
|
|
2007
|
-
authenticated_roles, output, error, executor_id,
|
|
2008
|
-
created_at, updated_at, application_version, application_id,
|
|
2009
|
-
class_name, config_name, recovery_attempts, queue_name,
|
|
2010
|
-
workflow_timeout_ms, workflow_deadline_epoch_ms, started_at_epoch_ms,
|
|
2011
|
-
deduplication_id, inputs, priority, queue_partition_key, forked_from,
|
|
2012
|
-
parent_workflow_id, serialization, delay_until_epoch_ms,
|
|
2013
|
-
was_forked_from, rate_limited, completed_at, attributes, schedule_name,
|
|
2014
|
-
debounce_deadline_epoch_ms, is_debounced, application_name
|
|
2015
|
-
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)`, [
|
|
2016
|
-
status.workflow_uuid,
|
|
2017
|
-
status.status,
|
|
2018
|
-
status.name,
|
|
2019
|
-
status.authenticated_user,
|
|
2020
|
-
status.assumed_role,
|
|
2021
|
-
status.authenticated_roles,
|
|
2022
|
-
// Legacy columns: the payloads live in their own tables.
|
|
2023
|
-
null,
|
|
2024
|
-
null,
|
|
2025
|
-
status.executor_id,
|
|
2026
|
-
status.created_at,
|
|
2027
|
-
status.updated_at,
|
|
2028
|
-
status.application_version,
|
|
2029
|
-
status.application_id,
|
|
2030
|
-
status.class_name,
|
|
2031
|
-
status.config_name,
|
|
2032
|
-
status.recovery_attempts,
|
|
2033
|
-
status.queue_name,
|
|
2034
|
-
status.workflow_timeout_ms,
|
|
2035
|
-
status.workflow_deadline_epoch_ms,
|
|
2036
|
-
status.started_at_epoch_ms,
|
|
2037
|
-
status.deduplication_id,
|
|
2038
|
-
null,
|
|
2039
|
-
status.priority,
|
|
2040
|
-
status.queue_partition_key,
|
|
2041
|
-
status.forked_from,
|
|
2042
|
-
status.parent_workflow_id,
|
|
2043
|
-
status.serialization,
|
|
2044
|
-
status.delay_until_epoch_ms ?? null,
|
|
2045
|
-
// NOT NULL columns: fall back to FALSE for payloads exported before
|
|
2046
|
-
// these fields were included.
|
|
2047
|
-
status.was_forked_from ?? false,
|
|
2048
|
-
status.rate_limited ?? false,
|
|
2049
|
-
status.completed_at ?? null,
|
|
2050
|
-
status.attributes ? JSON.stringify(status.attributes) : null,
|
|
2051
|
-
status.schedule_name ?? null,
|
|
2052
|
-
status.debounce_deadline_epoch_ms ?? null,
|
|
2053
|
-
status.is_debounced ?? false,
|
|
2054
|
-
status.application_name ?? null,
|
|
2055
|
-
]);
|
|
2056
|
-
// Retention starts at import: the original timestamps are long past the cutoff
|
|
2057
|
-
// and would be collected immediately.
|
|
2058
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
|
|
2059
|
-
VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [status.workflow_uuid, status.inputs]);
|
|
2060
|
-
if (status.output !== null || status.error !== null) {
|
|
2061
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_output (workflow_uuid, output, error, retention_timestamp)
|
|
2062
|
-
VALUES ($1, $2, $3, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [status.workflow_uuid, status.output, status.error]);
|
|
2063
|
-
}
|
|
2064
|
-
// Import operation_outputs
|
|
2065
|
-
for (const output of workflow.operation_outputs) {
|
|
2066
|
-
await client.query(`INSERT INTO "${this.schemaName}".operation_outputs (
|
|
2067
|
-
workflow_uuid, function_id, function_name, output, error,
|
|
2068
|
-
child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
|
|
2069
|
-
serialization, application_name, retention_timestamp
|
|
2070
|
-
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [
|
|
2071
|
-
output.workflow_uuid,
|
|
2072
|
-
output.function_id,
|
|
2073
|
-
output.function_name,
|
|
2074
|
-
output.output,
|
|
2075
|
-
output.error,
|
|
2076
|
-
output.child_workflow_id,
|
|
2077
|
-
output.started_at_epoch_ms,
|
|
2078
|
-
output.completed_at_epoch_ms,
|
|
2079
|
-
output.serialization,
|
|
2080
|
-
output.application_name ?? null,
|
|
2081
|
-
]);
|
|
2082
|
-
}
|
|
2083
|
-
// Import workflow_events
|
|
2084
|
-
for (const event of workflow.workflow_events) {
|
|
2085
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_events (
|
|
2086
|
-
workflow_uuid, key, value, serialization
|
|
2087
|
-
) VALUES ($1, $2, $3, $4)`, [event.workflow_uuid, event.key, event.value, event.serialization]);
|
|
2088
|
-
}
|
|
2089
|
-
// Import workflow_events_history
|
|
2090
|
-
for (const history of workflow.workflow_events_history) {
|
|
2091
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (
|
|
2092
|
-
workflow_uuid, function_id, key, value, serialization
|
|
2093
|
-
) VALUES ($1, $2, $3, $4, $5)`, [history.workflow_uuid, history.function_id, history.key, history.value, history.serialization]);
|
|
2094
|
-
}
|
|
2095
|
-
// Import streams
|
|
2096
|
-
for (const stream of workflow.streams) {
|
|
2097
|
-
await client.query(`INSERT INTO "${this.schemaName}".streams (
|
|
2098
|
-
workflow_uuid, key, value, "offset", function_id, serialization
|
|
2099
|
-
) VALUES ($1, $2, $3, $4, $5, $6)`, [stream.workflow_uuid, stream.key, stream.value, stream.offset, stream.function_id, stream.serialization]);
|
|
2100
|
-
}
|
|
2101
|
-
}
|
|
2102
|
-
await client.query('COMMIT');
|
|
2103
|
-
}
|
|
2104
|
-
catch (error) {
|
|
2105
|
-
await client.query('ROLLBACK');
|
|
2106
|
-
throw error;
|
|
2107
|
-
}
|
|
2108
|
-
finally {
|
|
2109
|
-
client.release();
|
|
2110
|
-
}
|
|
2111
|
-
}
|
|
2112
1797
|
// ==================== Awaiting Workflows ====================
|
|
2113
1798
|
/** Register one execution; the returned release is idempotent, so it can run both before a park and once the run settles. */
|
|
2114
1799
|
registerRunningWorkflow(workflowID, queueName, queuePartitionKey) {
|
|
@@ -2396,7 +2081,7 @@ class SystemDatabase {
|
|
|
2396
2081
|
async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey) {
|
|
2397
2082
|
topic = topic ?? this.nullTopic;
|
|
2398
2083
|
const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
|
|
2399
|
-
const client = await this
|
|
2084
|
+
const client = await this.connect();
|
|
2400
2085
|
try {
|
|
2401
2086
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
2402
2087
|
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
|
|
@@ -2432,7 +2117,7 @@ class SystemDatabase {
|
|
|
2432
2117
|
/** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
|
|
2433
2118
|
async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
|
|
2434
2119
|
const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
|
|
2435
|
-
const client = await this
|
|
2120
|
+
const client = await this.connect();
|
|
2436
2121
|
try {
|
|
2437
2122
|
await this.#inTransaction(client, async () => {
|
|
2438
2123
|
await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
|
|
@@ -2520,7 +2205,7 @@ class SystemDatabase {
|
|
|
2520
2205
|
// Transactionally consume and return the message if it's in the DB, otherwise return null.
|
|
2521
2206
|
let message = null;
|
|
2522
2207
|
let serialization = null;
|
|
2523
|
-
const client = await this
|
|
2208
|
+
const client = await this.connect();
|
|
2524
2209
|
try {
|
|
2525
2210
|
await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
|
|
2526
2211
|
const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
|
|
@@ -2560,7 +2245,7 @@ class SystemDatabase {
|
|
|
2560
2245
|
}
|
|
2561
2246
|
// ==================== Events ====================
|
|
2562
2247
|
async setEvent(workflowID, functionID, key, message, serialization) {
|
|
2563
|
-
const client = await this
|
|
2248
|
+
const client = await this.connect();
|
|
2564
2249
|
try {
|
|
2565
2250
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
2566
2251
|
// Only a real write (not a replay) should wake readers.
|
|
@@ -2668,7 +2353,7 @@ class SystemDatabase {
|
|
|
2668
2353
|
// ==================== Streams ====================
|
|
2669
2354
|
async writeStreamFromStep(workflowID, functionID, key, serializedValue, serialization) {
|
|
2670
2355
|
const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
|
|
2671
|
-
const client = await this
|
|
2356
|
+
const client = await this.connect();
|
|
2672
2357
|
try {
|
|
2673
2358
|
while (true) {
|
|
2674
2359
|
try {
|
|
@@ -2705,7 +2390,7 @@ class SystemDatabase {
|
|
|
2705
2390
|
}
|
|
2706
2391
|
}
|
|
2707
2392
|
async writeStreamFromWorkflow(workflowID, functionID, key, serializedValue, serialization, functionName) {
|
|
2708
|
-
const client = await this
|
|
2393
|
+
const client = await this.connect();
|
|
2709
2394
|
try {
|
|
2710
2395
|
while (true) {
|
|
2711
2396
|
// Only a real insert (not a replay) should wake readers.
|
|
@@ -2839,7 +2524,7 @@ class SystemDatabase {
|
|
|
2839
2524
|
}
|
|
2840
2525
|
try {
|
|
2841
2526
|
// One statement: one round trip, one async-notify queue-lock acquisition; unnest emits one notification per payload.
|
|
2842
|
-
const client = await this
|
|
2527
|
+
const client = await this.connect();
|
|
2843
2528
|
try {
|
|
2844
2529
|
await client.query(`SELECT pg_notify($1, p) FROM unnest($2::text[]) AS p`, [channel, Array.from(batch)]);
|
|
2845
2530
|
}
|
|
@@ -2854,49 +2539,6 @@ class SystemDatabase {
|
|
|
2854
2539
|
}
|
|
2855
2540
|
}
|
|
2856
2541
|
// ==================== Observability: Workflow Communications ====================
|
|
2857
|
-
async getAllEvents(workflowID) {
|
|
2858
|
-
const { rows } = await this.observabilityQuery((client) => client.query(`SELECT key, value, serialization FROM "${this.schemaName}".workflow_events
|
|
2859
|
-
WHERE workflow_uuid = $1`, [workflowID]));
|
|
2860
|
-
const events = {};
|
|
2861
|
-
for (const row of rows) {
|
|
2862
|
-
events[row.key] = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
|
|
2863
|
-
}
|
|
2864
|
-
return events;
|
|
2865
|
-
}
|
|
2866
|
-
async getAllNotifications(workflowID) {
|
|
2867
|
-
const { rows } = await this.observabilityQuery((client) => client.query(`SELECT topic, message, serialization, created_at_epoch_ms, consumed
|
|
2868
|
-
FROM "${this.schemaName}".notifications
|
|
2869
|
-
WHERE destination_uuid = $1
|
|
2870
|
-
ORDER BY created_at_epoch_ms`, [workflowID]));
|
|
2871
|
-
return await Promise.all(rows.map(async (row) => ({
|
|
2872
|
-
topic: row.topic === this.nullTopic ? null : row.topic,
|
|
2873
|
-
message: await (0, serialization_1.safeParse)(this.serializer, row.message, row.serialization),
|
|
2874
|
-
createdAtEpochMs: Number(row.created_at_epoch_ms),
|
|
2875
|
-
consumed: row.consumed,
|
|
2876
|
-
})));
|
|
2877
|
-
}
|
|
2878
|
-
async getAllStreamEntries(workflowID) {
|
|
2879
|
-
const { rows } = await this.observabilityQuery((client) => client.query(`SELECT key, value, serialization FROM "${this.schemaName}".streams
|
|
2880
|
-
WHERE workflow_uuid = $1
|
|
2881
|
-
ORDER BY key, "offset"`, [workflowID]));
|
|
2882
|
-
const streams = {};
|
|
2883
|
-
const closed = new Set();
|
|
2884
|
-
for (const row of rows) {
|
|
2885
|
-
if (closed.has(row.key)) {
|
|
2886
|
-
continue;
|
|
2887
|
-
}
|
|
2888
|
-
// safeParse yields the raw string for the legacy unserialized marker, which does not parse.
|
|
2889
|
-
const value = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
|
|
2890
|
-
if (isStreamClosedSentinel(value)) {
|
|
2891
|
-
// End the stream where readStream does, so the two never disagree.
|
|
2892
|
-
closed.add(row.key);
|
|
2893
|
-
streams[row.key] ??= [];
|
|
2894
|
-
continue;
|
|
2895
|
-
}
|
|
2896
|
-
(streams[row.key] ??= []).push(value);
|
|
2897
|
-
}
|
|
2898
|
-
return streams;
|
|
2899
|
-
}
|
|
2900
2542
|
// ==================== Queues ====================
|
|
2901
2543
|
async transitionDelayedWorkflows() {
|
|
2902
2544
|
// Transition workflows from DELAYED to ENQUEUED when their delay has expired.
|
|
@@ -2904,7 +2546,7 @@ class SystemDatabase {
|
|
|
2904
2546
|
// debounce key held only while DELAYED, so a later same-key debounce starts a fresh workflow.
|
|
2905
2547
|
const params = [workflow_1.StatusString.ENQUEUED, Date.now(), workflow_1.StatusString.DELAYED];
|
|
2906
2548
|
// Only what this application would dequeue: a peer's debounce key is not ours to clear.
|
|
2907
|
-
const scope = this
|
|
2549
|
+
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
2908
2550
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2909
2551
|
SET status = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
2910
2552
|
deduplication_id = CASE WHEN is_debounced THEN NULL ELSE deduplication_id END
|
|
@@ -2913,7 +2555,7 @@ class SystemDatabase {
|
|
|
2913
2555
|
/** Cancel up to `limit` of this application's active workflows whose deadline has passed, returning their IDs. */
|
|
2914
2556
|
async cancelTimedOutWorkflows(limit) {
|
|
2915
2557
|
const params = [workflow_1.StatusString.CANCELLED, Date.now(), limit];
|
|
2916
|
-
const scope = this
|
|
2558
|
+
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
2917
2559
|
// Literal statuses let the planner match idx_workflow_status_deadline under any plan mode.
|
|
2918
2560
|
// SKIP LOCKED leaves a row a dequeue or a peer's sweep holds for the next sweep.
|
|
2919
2561
|
const { rows } = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
@@ -2944,7 +2586,7 @@ class SystemDatabase {
|
|
|
2944
2586
|
// Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v3, so cost scales with the number of partitions rather than the backlog depth.
|
|
2945
2587
|
const params = [queueName, workflow_1.StatusString.ENQUEUED];
|
|
2946
2588
|
// Only partitions this application can actually dequeue from.
|
|
2947
|
-
const scope = this
|
|
2589
|
+
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
2948
2590
|
const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
|
|
2949
2591
|
(SELECT MIN(queue_partition_key) AS pk
|
|
2950
2592
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -2971,7 +2613,7 @@ class SystemDatabase {
|
|
|
2971
2613
|
queue.partitionRateLimit !== undefined;
|
|
2972
2614
|
// Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
|
|
2973
2615
|
const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
|
|
2974
|
-
const client = await this
|
|
2616
|
+
const client = await this.connect();
|
|
2975
2617
|
try {
|
|
2976
2618
|
// Default to READ COMMITTED except with a budget shared across executors
|
|
2977
2619
|
if (hasSharedBudget) {
|
|
@@ -2984,7 +2626,7 @@ class SystemDatabase {
|
|
|
2984
2626
|
const rateLimitRemaining = async (rateLimit, partitionScoped) => {
|
|
2985
2627
|
const params = [queue.name, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED, rateLimit.periodSec * 1000];
|
|
2986
2628
|
// Count only what this application would dequeue, matching the select below.
|
|
2987
|
-
const scope = this
|
|
2629
|
+
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
2988
2630
|
const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
|
|
2989
2631
|
const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
2990
2632
|
WHERE queue_name = $1
|
|
@@ -3003,7 +2645,7 @@ class SystemDatabase {
|
|
|
3003
2645
|
*/
|
|
3004
2646
|
const pendingCount = async (partitionScoped) => {
|
|
3005
2647
|
const params = [queue.name, workflow_1.StatusString.PENDING];
|
|
3006
|
-
const scope = this
|
|
2648
|
+
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
3007
2649
|
const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
|
|
3008
2650
|
const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
3009
2651
|
WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
|
|
@@ -3064,7 +2706,7 @@ class SystemDatabase {
|
|
|
3064
2706
|
const lockMode = hasSharedBudget ? 'FOR UPDATE NOWAIT' : 'FOR UPDATE SKIP LOCKED';
|
|
3065
2707
|
const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
|
|
3066
2708
|
const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
|
|
3067
|
-
const selectScope = this
|
|
2709
|
+
const selectScope = this.appNameFilter('application_name', this.appName, selectParams);
|
|
3068
2710
|
const selectQuery = `
|
|
3069
2711
|
SELECT workflow_uuid
|
|
3070
2712
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -3097,7 +2739,7 @@ class SystemDatabase {
|
|
|
3097
2739
|
ownerXid,
|
|
3098
2740
|
];
|
|
3099
2741
|
// Re-check ownership alongside status, as the partitioned claim guard does.
|
|
3100
|
-
const claimScope = this
|
|
2742
|
+
const claimScope = this.appNameFilter('application_name', this.appName, updateParams);
|
|
3101
2743
|
// RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
|
|
3102
2744
|
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
3103
2745
|
SET status = $1,
|
|
@@ -3147,7 +2789,7 @@ class SystemDatabase {
|
|
|
3147
2789
|
throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
|
|
3148
2790
|
}
|
|
3149
2791
|
// partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
|
|
3150
|
-
const client = await this
|
|
2792
|
+
const client = await this.connect();
|
|
3151
2793
|
try {
|
|
3152
2794
|
await client.query('BEGIN');
|
|
3153
2795
|
const latestVersion = await this.#latestApplicationVersionName(client);
|
|
@@ -3166,7 +2808,7 @@ class SystemDatabase {
|
|
|
3166
2808
|
workflow_1.StatusString.PENDING,
|
|
3167
2809
|
sweepLimit,
|
|
3168
2810
|
];
|
|
3169
|
-
const candidateScope = this
|
|
2811
|
+
const candidateScope = this.appNameFilter('application_name', this.appName, candidateParams);
|
|
3170
2812
|
// Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
|
|
3171
2813
|
const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
|
|
3172
2814
|
(SELECT MIN(queue_partition_key) AS pk
|
|
@@ -3223,7 +2865,7 @@ class SystemDatabase {
|
|
|
3223
2865
|
AND ${versionClause(version)}
|
|
3224
2866
|
AND ${scope}`;
|
|
3225
2867
|
const lockedParams = [candidateIDs, workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
|
|
3226
|
-
const lockedScope = this
|
|
2868
|
+
const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
|
|
3227
2869
|
// Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
|
|
3228
2870
|
const lockedResult = await client.query(`SELECT workflow_uuid
|
|
3229
2871
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -3247,7 +2889,7 @@ class SystemDatabase {
|
|
|
3247
2889
|
this.appName ?? null,
|
|
3248
2890
|
ownerXid,
|
|
3249
2891
|
];
|
|
3250
|
-
const flipScope = this
|
|
2892
|
+
const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
|
|
3251
2893
|
// Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
|
|
3252
2894
|
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
3253
2895
|
SET status = $1,
|
|
@@ -3373,8 +3015,8 @@ class SystemDatabase {
|
|
|
3373
3015
|
// Falsy, not just undefined: a Conductor request carries an omitted list as JSON null.
|
|
3374
3016
|
const idKeyed = (input.workflowIDs?.length ?? 0) > 0;
|
|
3375
3017
|
whereClauses.push(idKeyed
|
|
3376
|
-
? this
|
|
3377
|
-
: this
|
|
3018
|
+
? this.appNameFilter('application_name', input.applicationName, params)
|
|
3019
|
+
: this.observabilityFilter('application_name', input.applicationName, params));
|
|
3378
3020
|
paramCounter = params.length + 1;
|
|
3379
3021
|
if (input.workflow_id_prefix) {
|
|
3380
3022
|
if (Array.isArray(input.workflow_id_prefix)) {
|
|
@@ -3487,634 +3129,6 @@ class SystemDatabase {
|
|
|
3487
3129
|
: await this.observabilityQuery((client) => client.query(query, params));
|
|
3488
3130
|
return result.rows.map(mapWorkflowStatus);
|
|
3489
3131
|
}
|
|
3490
|
-
async getWorkflowAggregates(input) {
|
|
3491
|
-
if (input.timeBucketSizeMs !== undefined && input.timeBucketSizeMs <= 0) {
|
|
3492
|
-
throw new Error('time_bucket_size_ms must be > 0');
|
|
3493
|
-
}
|
|
3494
|
-
const groupByFlags = [
|
|
3495
|
-
['status', input.groupByStatus ?? false, 'status'],
|
|
3496
|
-
['name', input.groupByName ?? false, 'name'],
|
|
3497
|
-
['queue_name', input.groupByQueueName ?? false, 'queue_name'],
|
|
3498
|
-
['executor_id', input.groupByExecutorId ?? false, 'executor_id'],
|
|
3499
|
-
['application_version', input.groupByApplicationVersion ?? false, 'application_version'],
|
|
3500
|
-
['application_name', input.groupByApplicationName ?? false, 'application_name'],
|
|
3501
|
-
];
|
|
3502
|
-
const groupNames = [];
|
|
3503
|
-
const groupColumns = [];
|
|
3504
|
-
const groupSelectColumns = [];
|
|
3505
|
-
for (const [colName, enabled, col] of groupByFlags) {
|
|
3506
|
-
if (enabled) {
|
|
3507
|
-
groupNames.push(colName);
|
|
3508
|
-
groupColumns.push(col);
|
|
3509
|
-
groupSelectColumns.push(col);
|
|
3510
|
-
}
|
|
3511
|
-
}
|
|
3512
|
-
const params = [];
|
|
3513
|
-
if (input.timeBucketSizeMs !== undefined) {
|
|
3514
|
-
// Bucket on created_at — the indexed wall-clock timestamp on workflow_status.
|
|
3515
|
-
// One placeholder shared by SELECT and GROUP BY, so Postgres sees the two expressions as identical.
|
|
3516
|
-
params.push(input.timeBucketSizeMs);
|
|
3517
|
-
const bucket = `$${params.length}::bigint`;
|
|
3518
|
-
const bucketExpr = `(CAST(FLOOR(created_at / ${bucket}) AS BIGINT) * ${bucket})`;
|
|
3519
|
-
groupNames.push('time_bucket');
|
|
3520
|
-
groupColumns.push(bucketExpr);
|
|
3521
|
-
groupSelectColumns.push(`${bucketExpr} AS time_bucket`);
|
|
3522
|
-
}
|
|
3523
|
-
if (groupColumns.length === 0) {
|
|
3524
|
-
throw new Error('At least one group_by flag must be set to True');
|
|
3525
|
-
}
|
|
3526
|
-
// Build select columns from boolean flags. MAX ignores NULLs, so rows
|
|
3527
|
-
// missing started_at_epoch_ms or completed_at naturally drop out of the
|
|
3528
|
-
// latency maxes.
|
|
3529
|
-
const selectFlags = [
|
|
3530
|
-
['count', input.selectCount ?? false, 'COUNT(*)'],
|
|
3531
|
-
['min_created_at', input.selectMinCreatedAt ?? false, 'MIN(created_at)'],
|
|
3532
|
-
['max_queue_wait_ms', input.selectMaxQueueWaitMs ?? false, 'MAX(started_at_epoch_ms - created_at)'],
|
|
3533
|
-
['max_total_latency_ms', input.selectMaxTotalLatencyMs ?? false, 'MAX(completed_at - created_at)'],
|
|
3534
|
-
];
|
|
3535
|
-
const selectNames = [];
|
|
3536
|
-
const selectColumns = [];
|
|
3537
|
-
for (const [name, enabled, expr] of selectFlags) {
|
|
3538
|
-
if (enabled) {
|
|
3539
|
-
selectNames.push(name);
|
|
3540
|
-
selectColumns.push(`${expr} AS ${name}`);
|
|
3541
|
-
}
|
|
3542
|
-
}
|
|
3543
|
-
if (selectColumns.length === 0) {
|
|
3544
|
-
throw new Error('At least one select_ flag must be set to True');
|
|
3545
|
-
}
|
|
3546
|
-
const whereClauses = [];
|
|
3547
|
-
let paramIdx = params.length + 1;
|
|
3548
|
-
const addFilter = (column, values) => {
|
|
3549
|
-
if (!values || values.length === 0)
|
|
3550
|
-
return;
|
|
3551
|
-
const placeholders = values.map((_, i) => `$${paramIdx + i}`).join(', ');
|
|
3552
|
-
whereClauses.push(`${column} IN (${placeholders})`);
|
|
3553
|
-
params.push(...values);
|
|
3554
|
-
paramIdx += values.length;
|
|
3555
|
-
};
|
|
3556
|
-
addFilter('status', input.status);
|
|
3557
|
-
addFilter('name', input.name);
|
|
3558
|
-
addFilter('application_version', input.appVersion);
|
|
3559
|
-
addFilter('executor_id', input.executorId);
|
|
3560
|
-
addFilter('queue_name', input.queueName);
|
|
3561
|
-
if (input.workflowIdPrefix && input.workflowIdPrefix.length > 0) {
|
|
3562
|
-
const likeClauses = input.workflowIdPrefix.map((p) => {
|
|
3563
|
-
params.push(`${p}%`);
|
|
3564
|
-
return `workflow_uuid LIKE $${paramIdx++}`;
|
|
3565
|
-
});
|
|
3566
|
-
whereClauses.push(`(${likeClauses.join(' OR ')})`);
|
|
3567
|
-
}
|
|
3568
|
-
addFilter('workflow_uuid', input.workflowIDs);
|
|
3569
|
-
addFilter('authenticated_user', input.authenticatedUser);
|
|
3570
|
-
addFilter('forked_from', input.forkedFrom);
|
|
3571
|
-
addFilter('parent_workflow_id', input.parentWorkflowID);
|
|
3572
|
-
addFilter('schedule_name', input.scheduleName);
|
|
3573
|
-
// Unset scopes to this application, as on every other observability query.
|
|
3574
|
-
whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
|
|
3575
|
-
paramIdx = params.length + 1;
|
|
3576
|
-
if (input.wasForkedFrom !== undefined) {
|
|
3577
|
-
whereClauses.push(`was_forked_from = $${paramIdx}`);
|
|
3578
|
-
params.push(input.wasForkedFrom);
|
|
3579
|
-
paramIdx++;
|
|
3580
|
-
}
|
|
3581
|
-
// Matches the forks themselves, as opposed to wasForkedFrom, which matches the workflows they were forked from.
|
|
3582
|
-
if (input.isFork !== undefined) {
|
|
3583
|
-
whereClauses.push(input.isFork ? `forked_from IS NOT NULL` : `forked_from IS NULL`);
|
|
3584
|
-
}
|
|
3585
|
-
if (input.hasParent !== undefined) {
|
|
3586
|
-
whereClauses.push(input.hasParent ? `parent_workflow_id IS NOT NULL` : `parent_workflow_id IS NULL`);
|
|
3587
|
-
}
|
|
3588
|
-
// Match workflows whose attributes JSONB contains all the given key-value pairs.
|
|
3589
|
-
if (input.attributes && Object.keys(input.attributes).length > 0) {
|
|
3590
|
-
whereClauses.push(`attributes @> $${paramIdx}::jsonb`);
|
|
3591
|
-
params.push(JSON.stringify(input.attributes));
|
|
3592
|
-
paramIdx++;
|
|
3593
|
-
}
|
|
3594
|
-
if (input.startTime) {
|
|
3595
|
-
whereClauses.push(`created_at >= $${paramIdx}`);
|
|
3596
|
-
params.push(new Date(input.startTime).getTime());
|
|
3597
|
-
paramIdx++;
|
|
3598
|
-
}
|
|
3599
|
-
if (input.endTime) {
|
|
3600
|
-
whereClauses.push(`created_at <= $${paramIdx}`);
|
|
3601
|
-
params.push(new Date(input.endTime).getTime());
|
|
3602
|
-
paramIdx++;
|
|
3603
|
-
}
|
|
3604
|
-
if (input.completedAfter) {
|
|
3605
|
-
whereClauses.push(`completed_at >= $${paramIdx}`);
|
|
3606
|
-
params.push(new Date(input.completedAfter).getTime());
|
|
3607
|
-
paramIdx++;
|
|
3608
|
-
}
|
|
3609
|
-
if (input.completedBefore) {
|
|
3610
|
-
whereClauses.push(`completed_at <= $${paramIdx}`);
|
|
3611
|
-
params.push(new Date(input.completedBefore).getTime());
|
|
3612
|
-
paramIdx++;
|
|
3613
|
-
}
|
|
3614
|
-
// dequeuedAfter/Before filter on started_at_epoch_ms: that column is
|
|
3615
|
-
// populated on dequeue and surfaced as WorkflowStatus.dequeuedAt.
|
|
3616
|
-
if (input.dequeuedAfter) {
|
|
3617
|
-
whereClauses.push(`started_at_epoch_ms >= $${paramIdx}`);
|
|
3618
|
-
params.push(new Date(input.dequeuedAfter).getTime());
|
|
3619
|
-
paramIdx++;
|
|
3620
|
-
}
|
|
3621
|
-
if (input.dequeuedBefore) {
|
|
3622
|
-
whereClauses.push(`started_at_epoch_ms <= $${paramIdx}`);
|
|
3623
|
-
params.push(new Date(input.dequeuedBefore).getTime());
|
|
3624
|
-
paramIdx++;
|
|
3625
|
-
}
|
|
3626
|
-
const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
|
|
3627
|
-
const groupByClause = groupColumns.join(', ');
|
|
3628
|
-
const selectClause = [...groupSelectColumns, ...selectColumns].join(', ');
|
|
3629
|
-
const query = `
|
|
3630
|
-
SELECT ${selectClause}
|
|
3631
|
-
FROM "${this.schemaName}".workflow_status
|
|
3632
|
-
${whereClause}
|
|
3633
|
-
GROUP BY ${groupByClause}
|
|
3634
|
-
`;
|
|
3635
|
-
const result = await this.observabilityQuery((client) => client.query(query, params));
|
|
3636
|
-
const toIntOrNull = (v) => (v === null || v === undefined ? null : Number(v));
|
|
3637
|
-
return result.rows.map((row) => {
|
|
3638
|
-
const group = {};
|
|
3639
|
-
for (const name of groupNames) {
|
|
3640
|
-
const v = row[name];
|
|
3641
|
-
group[name] = v === null || v === undefined ? null : String(v);
|
|
3642
|
-
}
|
|
3643
|
-
return {
|
|
3644
|
-
group,
|
|
3645
|
-
count: selectNames.includes('count') ? toIntOrNull(row.count) : null,
|
|
3646
|
-
minCreatedAt: selectNames.includes('min_created_at') ? toIntOrNull(row.min_created_at) : null,
|
|
3647
|
-
maxQueueWaitMs: selectNames.includes('max_queue_wait_ms') ? toIntOrNull(row.max_queue_wait_ms) : null,
|
|
3648
|
-
maxTotalLatencyMs: selectNames.includes('max_total_latency_ms') ? toIntOrNull(row.max_total_latency_ms) : null,
|
|
3649
|
-
};
|
|
3650
|
-
});
|
|
3651
|
-
}
|
|
3652
|
-
async getStepAggregates(input) {
|
|
3653
|
-
if (input.timeBucketSizeMs !== undefined && input.timeBucketSizeMs <= 0) {
|
|
3654
|
-
throw new Error('time_bucket_size_ms must be > 0');
|
|
3655
|
-
}
|
|
3656
|
-
// operation_outputs has no explicit status column; derive it from whether `error` is populated.
|
|
3657
|
-
// Child-workflow mapping rows have NULL error, so they appear as SUCCESS — callers filter by function_name.
|
|
3658
|
-
const statusExpr = `CASE WHEN error IS NULL THEN 'SUCCESS' ELSE 'ERROR' END`;
|
|
3659
|
-
const groupByFlags = [
|
|
3660
|
-
['function_name', input.groupByFunctionName ?? false, 'function_name'],
|
|
3661
|
-
['status', input.groupByStatus ?? false, statusExpr],
|
|
3662
|
-
];
|
|
3663
|
-
const groupNames = [];
|
|
3664
|
-
const groupColumns = [];
|
|
3665
|
-
const groupSelectColumns = [];
|
|
3666
|
-
for (const [colName, enabled, expr] of groupByFlags) {
|
|
3667
|
-
if (enabled) {
|
|
3668
|
-
groupNames.push(colName);
|
|
3669
|
-
groupColumns.push(expr);
|
|
3670
|
-
groupSelectColumns.push(`${expr} AS ${colName}`);
|
|
3671
|
-
}
|
|
3672
|
-
}
|
|
3673
|
-
const params = [];
|
|
3674
|
-
if (input.timeBucketSizeMs !== undefined) {
|
|
3675
|
-
// Bucket on completed_at_epoch_ms — it's the indexed timestamp on
|
|
3676
|
-
// this table.
|
|
3677
|
-
params.push(input.timeBucketSizeMs);
|
|
3678
|
-
const bucket = `$${params.length}::bigint`;
|
|
3679
|
-
const bucketExpr = `(CAST(FLOOR(completed_at_epoch_ms / ${bucket}) AS BIGINT) * ${bucket})`;
|
|
3680
|
-
groupNames.push('time_bucket');
|
|
3681
|
-
groupColumns.push(bucketExpr);
|
|
3682
|
-
groupSelectColumns.push(`${bucketExpr} AS time_bucket`);
|
|
3683
|
-
}
|
|
3684
|
-
if (groupColumns.length === 0) {
|
|
3685
|
-
throw new Error('At least one group_by flag must be set to True');
|
|
3686
|
-
}
|
|
3687
|
-
// Build select columns from boolean flags. Child-workflow mapping rows record start and
|
|
3688
|
-
// complete at nearly the same instant, so they contribute ~0; DBOS.getResult and DBOS.sleep
|
|
3689
|
-
// rows span their whole wait, so those dominate the duration max.
|
|
3690
|
-
const selectFlags = [
|
|
3691
|
-
['count', input.selectCount ?? false, 'COUNT(*)'],
|
|
3692
|
-
['max_duration_ms', input.selectMaxDurationMs ?? false, 'MAX(completed_at_epoch_ms - started_at_epoch_ms)'],
|
|
3693
|
-
];
|
|
3694
|
-
const selectNames = [];
|
|
3695
|
-
const selectColumns = [];
|
|
3696
|
-
for (const [name, enabled, expr] of selectFlags) {
|
|
3697
|
-
if (enabled) {
|
|
3698
|
-
selectNames.push(name);
|
|
3699
|
-
selectColumns.push(`${expr} AS ${name}`);
|
|
3700
|
-
}
|
|
3701
|
-
}
|
|
3702
|
-
if (selectColumns.length === 0) {
|
|
3703
|
-
throw new Error('At least one select_ flag must be set to True');
|
|
3704
|
-
}
|
|
3705
|
-
const whereClauses = [];
|
|
3706
|
-
let paramIdx = params.length + 1;
|
|
3707
|
-
if (input.status && input.status.length > 0) {
|
|
3708
|
-
const placeholders = input.status.map((_, i) => `$${paramIdx + i}`).join(', ');
|
|
3709
|
-
whereClauses.push(`(${statusExpr}) IN (${placeholders})`);
|
|
3710
|
-
params.push(...input.status);
|
|
3711
|
-
paramIdx += input.status.length;
|
|
3712
|
-
}
|
|
3713
|
-
if (input.functionName && input.functionName.length > 0) {
|
|
3714
|
-
const placeholders = input.functionName.map((_, i) => `$${paramIdx + i}`).join(', ');
|
|
3715
|
-
whereClauses.push(`function_name IN (${placeholders})`);
|
|
3716
|
-
params.push(...input.functionName);
|
|
3717
|
-
paramIdx += input.functionName.length;
|
|
3718
|
-
}
|
|
3719
|
-
if (input.workflowIdPrefix && input.workflowIdPrefix.length > 0) {
|
|
3720
|
-
const likeClauses = input.workflowIdPrefix.map((p) => {
|
|
3721
|
-
params.push(`${p}%`);
|
|
3722
|
-
return `workflow_uuid LIKE $${paramIdx++}`;
|
|
3723
|
-
});
|
|
3724
|
-
whereClauses.push(`(${likeClauses.join(' OR ')})`);
|
|
3725
|
-
}
|
|
3726
|
-
if (input.completedAfter) {
|
|
3727
|
-
whereClauses.push(`completed_at_epoch_ms >= $${paramIdx}`);
|
|
3728
|
-
params.push(new Date(input.completedAfter).getTime());
|
|
3729
|
-
paramIdx++;
|
|
3730
|
-
}
|
|
3731
|
-
if (input.completedBefore) {
|
|
3732
|
-
whereClauses.push(`completed_at_epoch_ms <= $${paramIdx}`);
|
|
3733
|
-
params.push(new Date(input.completedBefore).getTime());
|
|
3734
|
-
paramIdx++;
|
|
3735
|
-
}
|
|
3736
|
-
// Unset scopes to this application, as on every other observability query.
|
|
3737
|
-
whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
|
|
3738
|
-
paramIdx = params.length + 1;
|
|
3739
|
-
const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
|
|
3740
|
-
const groupByClause = groupColumns.join(', ');
|
|
3741
|
-
const selectClause = [...groupSelectColumns, ...selectColumns].join(', ');
|
|
3742
|
-
const query = `
|
|
3743
|
-
SELECT ${selectClause}
|
|
3744
|
-
FROM "${this.schemaName}".operation_outputs
|
|
3745
|
-
${whereClause}
|
|
3746
|
-
GROUP BY ${groupByClause}
|
|
3747
|
-
`;
|
|
3748
|
-
const result = await this.observabilityQuery((client) => client.query(query, params));
|
|
3749
|
-
const toIntOrNull = (v) => (v === null || v === undefined ? null : Number(v));
|
|
3750
|
-
return result.rows.map((row) => {
|
|
3751
|
-
const group = {};
|
|
3752
|
-
for (const name of groupNames) {
|
|
3753
|
-
const v = row[name];
|
|
3754
|
-
group[name] = v === null || v === undefined ? null : String(v);
|
|
3755
|
-
}
|
|
3756
|
-
return {
|
|
3757
|
-
group,
|
|
3758
|
-
count: selectNames.includes('count') ? toIntOrNull(row.count) : null,
|
|
3759
|
-
maxDurationMs: selectNames.includes('max_duration_ms') ? toIntOrNull(row.max_duration_ms) : null,
|
|
3760
|
-
};
|
|
3761
|
-
});
|
|
3762
|
-
}
|
|
3763
|
-
/** Whether the system database is CockroachDB. Resolved once, since detecting it costs a query. */
|
|
3764
|
-
async #cockroach() {
|
|
3765
|
-
if (this.#isCockroach === undefined) {
|
|
3766
|
-
const client = await this.#connect();
|
|
3767
|
-
try {
|
|
3768
|
-
this.#isCockroach = await isCockroachDB(client);
|
|
3769
|
-
}
|
|
3770
|
-
finally {
|
|
3771
|
-
client.release();
|
|
3772
|
-
}
|
|
3773
|
-
}
|
|
3774
|
-
return this.#isCockroach;
|
|
3775
|
-
}
|
|
3776
|
-
/**
|
|
3777
|
-
* Take a database-wide lock for one retention round, returning how to release it, or
|
|
3778
|
-
* undefined when another round already holds it. The lock is session-scoped, so a round
|
|
3779
|
-
* that crashes releases it. CockroachDB has no advisory locks and always takes it, so it
|
|
3780
|
-
* collects unprotected rather than not at all.
|
|
3781
|
-
*/
|
|
3782
|
-
async acquireRetentionLock() {
|
|
3783
|
-
if (await this.#cockroach()) {
|
|
3784
|
-
return { release: () => Promise.resolve() };
|
|
3785
|
-
}
|
|
3786
|
-
const key = retentionLockKey(this.schemaName).toString();
|
|
3787
|
-
// The round holds this connection until it ends: releasing it would drop the lock.
|
|
3788
|
-
const client = await this.#borrowRetentionClient();
|
|
3789
|
-
let acquired = false;
|
|
3790
|
-
try {
|
|
3791
|
-
const { rows } = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [key]);
|
|
3792
|
-
acquired = rows[0]?.locked === true;
|
|
3793
|
-
}
|
|
3794
|
-
catch (e) {
|
|
3795
|
-
this.#releaseRetentionClient(client);
|
|
3796
|
-
throw e;
|
|
3797
|
-
}
|
|
3798
|
-
if (!acquired) {
|
|
3799
|
-
this.#releaseRetentionClient(client);
|
|
3800
|
-
return undefined;
|
|
3801
|
-
}
|
|
3802
|
-
return {
|
|
3803
|
-
release: async () => {
|
|
3804
|
-
// Already cut by destroy(): the session is gone and took the lock with it.
|
|
3805
|
-
if (!this.#retentionClients.has(client)) {
|
|
3806
|
-
return;
|
|
3807
|
-
}
|
|
3808
|
-
try {
|
|
3809
|
-
// Explicit, since releasing the client only returns the session to the pool.
|
|
3810
|
-
const { rows } = await client.query('SELECT pg_advisory_unlock($1) AS released', [
|
|
3811
|
-
key,
|
|
3812
|
-
]);
|
|
3813
|
-
if (rows[0]?.released !== true) {
|
|
3814
|
-
// False means this session no longer holds it, which a transaction-pooling proxy
|
|
3815
|
-
// causes by switching backends.
|
|
3816
|
-
this.logger.warn('Could not release the retention lock: this session no longer holds it. Retention will ' +
|
|
3817
|
-
'not proceed until the lock is released, which happens when the holding backend closes. ' +
|
|
3818
|
-
'A transaction-pooling proxy in front of Postgres causes this; run DBOS through a ' +
|
|
3819
|
-
'session-pooled or direct connection.');
|
|
3820
|
-
}
|
|
3821
|
-
}
|
|
3822
|
-
finally {
|
|
3823
|
-
this.#releaseRetentionClient(client);
|
|
3824
|
-
}
|
|
3825
|
-
},
|
|
3826
|
-
};
|
|
3827
|
-
}
|
|
3828
|
-
/** VACUUM the tables a sweep dirtied. No-op where there is no autovacuum to outrun. */
|
|
3829
|
-
async #vacuumTables(tables) {
|
|
3830
|
-
if (await this.#cockroach()) {
|
|
3831
|
-
return;
|
|
3832
|
-
}
|
|
3833
|
-
const client = await this.#borrowRetentionClient();
|
|
3834
|
-
let notices = [];
|
|
3835
|
-
const onNotice = (notice) => notices.push(notice.message ?? '');
|
|
3836
|
-
client.on('notice', onNotice);
|
|
3837
|
-
try {
|
|
3838
|
-
for (const table of tables) {
|
|
3839
|
-
// Per table, so one refusal does not skip the rest.
|
|
3840
|
-
notices = [];
|
|
3841
|
-
try {
|
|
3842
|
-
await client.query(`VACUUM (INDEX_CLEANUP ON, TRUNCATE OFF, ANALYZE) "${this.schemaName}"."${table}"`);
|
|
3843
|
-
}
|
|
3844
|
-
catch (e) {
|
|
3845
|
-
if (!this.#retentionClients.has(client)) {
|
|
3846
|
-
throw e;
|
|
3847
|
-
}
|
|
3848
|
-
this.logger.warn(`Payload retention could not vacuum ${table}: ${e.message}`);
|
|
3849
|
-
continue;
|
|
3850
|
-
}
|
|
3851
|
-
// A refused or stalled VACUUM does not raise, it says so in a notice; a successful
|
|
3852
|
-
// one is silent, so anything here is worth surfacing.
|
|
3853
|
-
for (const notice of notices) {
|
|
3854
|
-
this.logger.warn(`Payload retention vacuuming ${table}: ${notice}`);
|
|
3855
|
-
}
|
|
3856
|
-
}
|
|
3857
|
-
}
|
|
3858
|
-
finally {
|
|
3859
|
-
client.removeListener('notice', onNotice);
|
|
3860
|
-
this.#releaseRetentionClient(client);
|
|
3861
|
-
}
|
|
3862
|
-
}
|
|
3863
|
-
/** Delete one payload table's orphans below the cutoff, one batch per transaction. */
|
|
3864
|
-
async #garbageCollectTable(table, cutoff, batchSize) {
|
|
3865
|
-
// A payload below the cutoff belongs to a workflow created before it, so the status side
|
|
3866
|
-
// is the few such rows still present, not the whole table.
|
|
3867
|
-
const orphaned = `NOT EXISTS (
|
|
3868
|
-
SELECT 1 FROM "${this.schemaName}".workflow_status ws
|
|
3869
|
-
WHERE ws.workflow_uuid = t.workflow_uuid AND ws.created_at < $1
|
|
3870
|
-
)`;
|
|
3871
|
-
// Seed from the oldest row in range.
|
|
3872
|
-
const oldest = await retryOnSerializationError(async () => {
|
|
3873
|
-
const { rows } = await this.pool.query(`SELECT retention_timestamp
|
|
3874
|
-
FROM "${this.schemaName}".${table}
|
|
3875
|
-
WHERE retention_timestamp < $1
|
|
3876
|
-
ORDER BY retention_timestamp
|
|
3877
|
-
LIMIT 1`, [cutoff]);
|
|
3878
|
-
// retention_timestamp is a bigint, so node-postgres hands it back as a string.
|
|
3879
|
-
return rows.length > 0 ? Number(rows[0].retention_timestamp) : undefined;
|
|
3880
|
-
});
|
|
3881
|
-
if (oldest === undefined) {
|
|
3882
|
-
return 0;
|
|
3883
|
-
}
|
|
3884
|
-
let total = 0;
|
|
3885
|
-
let watermark = oldest - 1;
|
|
3886
|
-
for (;;) {
|
|
3887
|
-
const batch = await retryOnSerializationError(async () => {
|
|
3888
|
-
// Borrowed rather than pool.query'd: that releases with the error, which discards the
|
|
3889
|
-
// connection on a deadlock, so the retry wrapping this would churn the pool per batch.
|
|
3890
|
-
const client = await this.#borrowRetentionClient();
|
|
3891
|
-
try {
|
|
3892
|
-
// Batches are cut by candidate count, so rows spared by the anti-join only thin one
|
|
3893
|
-
// out; they are re-checked next round.
|
|
3894
|
-
const { rows } = await client.query(`SELECT retention_timestamp
|
|
3895
|
-
FROM "${this.schemaName}".${table}
|
|
3896
|
-
WHERE retention_timestamp < $1 AND retention_timestamp > $2
|
|
3897
|
-
ORDER BY retention_timestamp
|
|
3898
|
-
LIMIT 1 OFFSET ${batchSize - 1}`, [cutoff, watermark]);
|
|
3899
|
-
const step = rows.length > 0 ? Number(rows[0].retention_timestamp) : undefined;
|
|
3900
|
-
const params = [cutoff, watermark];
|
|
3901
|
-
// Timestamp ties may push the batch slightly over batchSize, but never split across two.
|
|
3902
|
-
const upperBound = step === undefined ? '' : `AND t.retention_timestamp <= $${params.push(step)} `;
|
|
3903
|
-
const result = await client.query(`DELETE FROM "${this.schemaName}".${table} t
|
|
3904
|
-
WHERE t.retention_timestamp < $1 AND t.retention_timestamp > $2 ${upperBound}AND ${orphaned}`, params);
|
|
3905
|
-
return { step, deleted: result.rowCount ?? 0 };
|
|
3906
|
-
}
|
|
3907
|
-
finally {
|
|
3908
|
-
this.#releaseRetentionClient(client);
|
|
3909
|
-
}
|
|
3910
|
-
});
|
|
3911
|
-
total += batch.deleted;
|
|
3912
|
-
if (batch.step === undefined) {
|
|
3913
|
-
return total;
|
|
3914
|
-
}
|
|
3915
|
-
watermark = batch.step;
|
|
3916
|
-
}
|
|
3917
|
-
}
|
|
3918
|
-
/**
|
|
3919
|
-
* Delete payload and step rows below the cutoff whose workflow is gone, returning the count
|
|
3920
|
-
* removed from each table. Runs after the status sweep, whose orphans all fall in range:
|
|
3921
|
-
* every payload is stamped no later than the completion that made the row collectable.
|
|
3922
|
-
*/
|
|
3923
|
-
async garbageCollectPayloads(cutoff, batchSize = exports.DEFAULT_GC_BATCH_SIZE) {
|
|
3924
|
-
// A NaN survives a bare `< 1` test and would only fail once it reached SQL.
|
|
3925
|
-
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
3926
|
-
throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
|
|
3927
|
-
}
|
|
3928
|
-
const tables = ['workflow_input', 'workflow_output', 'operation_outputs'];
|
|
3929
|
-
// To optimize performance, vacuum payload tables both before and after collecting them.
|
|
3930
|
-
await this.#vacuumTables(['workflow_status', ...tables]);
|
|
3931
|
-
// One connection per concurrent sweep, on top of the one the retention lock holds for the
|
|
3932
|
-
// whole round. A pool too small for all three runs them sequentially rather than leaving
|
|
3933
|
-
// sweeps waiting on a connection that only the round itself would free.
|
|
3934
|
-
const poolMax = this.pool.options.max ?? exports.DEFAULT_POOL_SIZE;
|
|
3935
|
-
const concurrency = Math.max(1, Math.min(tables.length, poolMax - 1));
|
|
3936
|
-
const deleted = new Array(tables.length).fill(0);
|
|
3937
|
-
const failures = [];
|
|
3938
|
-
let next = 0;
|
|
3939
|
-
const sweep = async () => {
|
|
3940
|
-
for (;;) {
|
|
3941
|
-
const i = next++;
|
|
3942
|
-
if (i >= tables.length)
|
|
3943
|
-
return;
|
|
3944
|
-
try {
|
|
3945
|
-
deleted[i] = await this.#garbageCollectTable(tables[i], cutoff, batchSize);
|
|
3946
|
-
}
|
|
3947
|
-
catch (e) {
|
|
3948
|
-
failures.push(e);
|
|
3949
|
-
}
|
|
3950
|
-
}
|
|
3951
|
-
};
|
|
3952
|
-
await Promise.all(Array.from({ length: concurrency }, () => sweep()));
|
|
3953
|
-
// Only the first can be thrown, so the rest would otherwise be lost.
|
|
3954
|
-
for (const extra of failures.slice(1)) {
|
|
3955
|
-
this.logger.warn(`Payload retention sweep also failed: ${extra instanceof Error ? extra.message : String(extra)}`);
|
|
3956
|
-
}
|
|
3957
|
-
if (failures.length > 0) {
|
|
3958
|
-
throw failures[0];
|
|
3959
|
-
}
|
|
3960
|
-
await this.#vacuumTables(tables);
|
|
3961
|
-
this.logger.debug(`Payload retention deleted ${deleted[0]} inputs, ${deleted[1]} outputs, and ${deleted[2]} steps`);
|
|
3962
|
-
return deleted;
|
|
3963
|
-
}
|
|
3964
|
-
/**
|
|
3965
|
-
* Rows garbage collection may delete. completed_at is set on every terminal transition and
|
|
3966
|
-
* cleared on resume, so one predicate covers eligibility: in-flight rows hold NULL and never
|
|
3967
|
-
* compare true. Unscoped by application: retention is system-wide.
|
|
3968
|
-
*/
|
|
3969
|
-
#gcFilter(cutoffEpochTimestampMs, params) {
|
|
3970
|
-
params.push(cutoffEpochTimestampMs);
|
|
3971
|
-
return `completed_at < $${params.length}`;
|
|
3972
|
-
}
|
|
3973
|
-
/**
|
|
3974
|
-
* Delete one batch, returning the watermark to resume from, or undefined once the last one ran.
|
|
3975
|
-
* The delete is its own transaction; it re-checks the filter, so it needs no snapshot shared
|
|
3976
|
-
* with the select that bounds it.
|
|
3977
|
-
*/
|
|
3978
|
-
async #garbageCollectBatch(cutoffEpochTimestampMs, batchSize, watermark) {
|
|
3979
|
-
// Borrowed rather than pool.query'd: that releases with the error, which discards the
|
|
3980
|
-
// connection on a deadlock, so the retry wrapping this would churn the pool per batch.
|
|
3981
|
-
const client = await this.#borrowRetentionClient();
|
|
3982
|
-
try {
|
|
3983
|
-
// The batchSize-th oldest eligible row above the watermark bounds this range
|
|
3984
|
-
const stepParams = [];
|
|
3985
|
-
const stepScope = this.#gcFilter(cutoffEpochTimestampMs, stepParams);
|
|
3986
|
-
stepParams.push(watermark);
|
|
3987
|
-
const stepResult = await client.query(`SELECT completed_at
|
|
3988
|
-
FROM "${this.schemaName}".workflow_status
|
|
3989
|
-
WHERE ${stepScope} AND completed_at > $${stepParams.length}
|
|
3990
|
-
ORDER BY completed_at
|
|
3991
|
-
LIMIT 1 OFFSET ${batchSize - 1}`, stepParams);
|
|
3992
|
-
// completed_at is a bigint, so node-postgres hands it back as a string.
|
|
3993
|
-
const step = stepResult.rows.length > 0 ? Number(stepResult.rows[0].completed_at) : undefined;
|
|
3994
|
-
const deleteParams = [];
|
|
3995
|
-
let deleteScope = this.#gcFilter(cutoffEpochTimestampMs, deleteParams);
|
|
3996
|
-
if (step !== undefined) {
|
|
3997
|
-
// Inclusive upper bound: completed_at ties may push a batch over batchSize, but never split across two.
|
|
3998
|
-
deleteParams.push(watermark, step);
|
|
3999
|
-
deleteScope = `${deleteScope} AND completed_at > $${deleteParams.length - 1} AND completed_at <= $${deleteParams.length}`;
|
|
4000
|
-
}
|
|
4001
|
-
// The final batch drops the watermark: unbounded, since an import can land a
|
|
4002
|
-
// completed_at below it mid-pass.
|
|
4003
|
-
await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE ${deleteScope}`, deleteParams);
|
|
4004
|
-
return step;
|
|
4005
|
-
}
|
|
4006
|
-
finally {
|
|
4007
|
-
this.#releaseRetentionClient(client);
|
|
4008
|
-
}
|
|
4009
|
-
}
|
|
4010
|
-
/**
|
|
4011
|
-
* Delete old terminal workflows throughout the system database, returning the cutoff
|
|
4012
|
-
* actually used, or undefined when there is nothing to collect.
|
|
4013
|
-
*
|
|
4014
|
-
* Conductor sends cleared retention thresholds as JSON null, so every param is nullish.
|
|
4015
|
-
*/
|
|
4016
|
-
async garbageCollect(cutoffEpochTimestampMs, rowsThreshold, options = {}) {
|
|
4017
|
-
const batchSize = options.batchSize ?? exports.DEFAULT_GC_BATCH_SIZE;
|
|
4018
|
-
// A NaN survives a bare `< 1` test and would only fail once it reached SQL, leaving GC half-applied.
|
|
4019
|
-
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
4020
|
-
throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
|
|
4021
|
-
}
|
|
4022
|
-
if (rowsThreshold !== undefined && rowsThreshold !== null) {
|
|
4023
|
-
// The completed_at of the rowsThreshold newest completed row
|
|
4024
|
-
const result = await retryOnSerializationError(() => this.pool.query(`SELECT completed_at
|
|
4025
|
-
FROM "${this.schemaName}".workflow_status
|
|
4026
|
-
WHERE completed_at IS NOT NULL
|
|
4027
|
-
ORDER BY completed_at DESC
|
|
4028
|
-
LIMIT 1 OFFSET $1`, [rowsThreshold - 1]));
|
|
4029
|
-
if (result.rows.length > 0) {
|
|
4030
|
-
const rowsBasedCutoff = Number(result.rows[0].completed_at);
|
|
4031
|
-
// Use the more restrictive cutoff (higher timestamp = more recent = more deletion)
|
|
4032
|
-
if (cutoffEpochTimestampMs === undefined ||
|
|
4033
|
-
cutoffEpochTimestampMs === null ||
|
|
4034
|
-
rowsBasedCutoff > cutoffEpochTimestampMs) {
|
|
4035
|
-
cutoffEpochTimestampMs = rowsBasedCutoff;
|
|
4036
|
-
}
|
|
4037
|
-
}
|
|
4038
|
-
}
|
|
4039
|
-
if (cutoffEpochTimestampMs === undefined || cutoffEpochTimestampMs === null) {
|
|
4040
|
-
return undefined;
|
|
4041
|
-
}
|
|
4042
|
-
// Narrowed to a constant so the closures below keep it.
|
|
4043
|
-
const cutoff = cutoffEpochTimestampMs;
|
|
4044
|
-
// Advance a completed_at watermark, one committed transaction per batch, so a long
|
|
4045
|
-
// history neither deletes in one transaction nor rescans what it already deleted.
|
|
4046
|
-
const oldest = await retryOnSerializationError(async () => {
|
|
4047
|
-
const params = [];
|
|
4048
|
-
const scope = this.#gcFilter(cutoff, params);
|
|
4049
|
-
const { rows } = await this.pool.query(`SELECT completed_at
|
|
4050
|
-
FROM "${this.schemaName}".workflow_status
|
|
4051
|
-
WHERE ${scope}
|
|
4052
|
-
ORDER BY completed_at
|
|
4053
|
-
LIMIT 1`, params);
|
|
4054
|
-
return rows.length > 0 ? Number(rows[0].completed_at) : undefined;
|
|
4055
|
-
});
|
|
4056
|
-
let watermark = oldest === undefined ? 0 : oldest - 1;
|
|
4057
|
-
for (;;) {
|
|
4058
|
-
const next = await retryOnSerializationError(() => this.#garbageCollectBatch(cutoff, batchSize, watermark));
|
|
4059
|
-
// Fewer than a full batch remained, so that delete took the rest.
|
|
4060
|
-
if (next === undefined)
|
|
4061
|
-
return cutoff;
|
|
4062
|
-
watermark = next;
|
|
4063
|
-
}
|
|
4064
|
-
}
|
|
4065
|
-
/**
|
|
4066
|
-
* IDs of this application's in-flight workflows created at or before the cutoff.
|
|
4067
|
-
* Claiming-scoped, so an upgrade still times out its own unclaimed workflows.
|
|
4068
|
-
*/
|
|
4069
|
-
async listTimedOutWorkflowIds(cutoffEpochTimestampMs) {
|
|
4070
|
-
const params = [
|
|
4071
|
-
cutoffEpochTimestampMs,
|
|
4072
|
-
workflow_1.StatusString.PENDING,
|
|
4073
|
-
workflow_1.StatusString.ENQUEUED,
|
|
4074
|
-
workflow_1.StatusString.DELAYED,
|
|
4075
|
-
];
|
|
4076
|
-
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
4077
|
-
const { rows } = await this.pool.query(`SELECT workflow_uuid
|
|
4078
|
-
FROM "${this.schemaName}".workflow_status
|
|
4079
|
-
WHERE created_at <= $1
|
|
4080
|
-
AND status IN ($2, $3, $4)
|
|
4081
|
-
AND ${scope}`, params);
|
|
4082
|
-
return rows.map((row) => row.workflow_uuid);
|
|
4083
|
-
}
|
|
4084
|
-
async getMetrics(startTime, endTime, applicationName) {
|
|
4085
|
-
const startEpochMs = new Date(startTime).getTime();
|
|
4086
|
-
const endEpochMs = new Date(endTime).getTime();
|
|
4087
|
-
const workflowParams = [startEpochMs, endEpochMs];
|
|
4088
|
-
const workflowScope = this.#observabilityFilter('application_name', applicationName, workflowParams);
|
|
4089
|
-
const stepParams = [startEpochMs, endEpochMs];
|
|
4090
|
-
const stepScope = this.#observabilityFilter('application_name', applicationName, stepParams);
|
|
4091
|
-
const [workflowResult, stepResult] = await this.observabilityQuery(async (client) => [
|
|
4092
|
-
await client.query(`SELECT name, COUNT(workflow_uuid) as count
|
|
4093
|
-
FROM "${this.schemaName}".workflow_status
|
|
4094
|
-
WHERE created_at >= $1 AND created_at < $2 AND ${workflowScope}
|
|
4095
|
-
GROUP BY name`, workflowParams),
|
|
4096
|
-
await client.query(`SELECT function_name, COUNT(*) as count
|
|
4097
|
-
FROM "${this.schemaName}".operation_outputs
|
|
4098
|
-
WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2 AND ${stepScope}
|
|
4099
|
-
GROUP BY function_name`, stepParams),
|
|
4100
|
-
]);
|
|
4101
|
-
const metrics = [];
|
|
4102
|
-
for (const row of workflowResult.rows) {
|
|
4103
|
-
metrics.push({
|
|
4104
|
-
metricType: 'workflow_count',
|
|
4105
|
-
metricName: row.name,
|
|
4106
|
-
value: Number(row.count),
|
|
4107
|
-
});
|
|
4108
|
-
}
|
|
4109
|
-
for (const row of stepResult.rows) {
|
|
4110
|
-
metrics.push({
|
|
4111
|
-
metricType: 'step_count',
|
|
4112
|
-
metricName: row.function_name,
|
|
4113
|
-
value: Number(row.count),
|
|
4114
|
-
});
|
|
4115
|
-
}
|
|
4116
|
-
return metrics;
|
|
4117
|
-
}
|
|
4118
3132
|
// ==================== Scheduling ====================
|
|
4119
3133
|
async createSchedule(schedule, client) {
|
|
4120
3134
|
const q = client ?? this.pool;
|
|
@@ -4180,7 +3194,7 @@ class SystemDatabase {
|
|
|
4180
3194
|
conditions.push(`(${likeClauses.join(' OR ')})`);
|
|
4181
3195
|
}
|
|
4182
3196
|
// Unset scopes to this application, as on every other observability query.
|
|
4183
|
-
conditions.push(this
|
|
3197
|
+
conditions.push(this.observabilityFilter('application_name', filters?.applicationName, params));
|
|
4184
3198
|
paramIdx = params.length + 1;
|
|
4185
3199
|
const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
|
|
4186
3200
|
const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
|
|
@@ -4247,7 +3261,7 @@ class SystemDatabase {
|
|
|
4247
3261
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE schedule_name = $2`, [lastFiredAt, name]);
|
|
4248
3262
|
}
|
|
4249
3263
|
async applySchedules(schedules) {
|
|
4250
|
-
const client = await this
|
|
3264
|
+
const client = await this.connect();
|
|
4251
3265
|
try {
|
|
4252
3266
|
await client.query('BEGIN');
|
|
4253
3267
|
for (const sched of schedules) {
|
|
@@ -4299,7 +3313,7 @@ class SystemDatabase {
|
|
|
4299
3313
|
*/
|
|
4300
3314
|
async createApplicationVersion(versionName, applicationName) {
|
|
4301
3315
|
const owner = applicationName ?? this.appName;
|
|
4302
|
-
const client = await this
|
|
3316
|
+
const client = await this.connect();
|
|
4303
3317
|
try {
|
|
4304
3318
|
await client.query('BEGIN');
|
|
4305
3319
|
// Claim a pre-upgrade row in place, so the version is not recreated or retimed.
|
|
@@ -4330,7 +3344,7 @@ class SystemDatabase {
|
|
|
4330
3344
|
*/
|
|
4331
3345
|
async updateApplicationVersionTimestamp(versionName, newTimestamp, applicationName) {
|
|
4332
3346
|
const owner = applicationName ?? this.appName;
|
|
4333
|
-
const client = await this
|
|
3347
|
+
const client = await this.connect();
|
|
4334
3348
|
try {
|
|
4335
3349
|
await client.query('BEGIN');
|
|
4336
3350
|
const resolved = await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
|
|
@@ -4354,7 +3368,7 @@ class SystemDatabase {
|
|
|
4354
3368
|
}
|
|
4355
3369
|
async listApplicationVersions() {
|
|
4356
3370
|
const params = [];
|
|
4357
|
-
const scope = this
|
|
3371
|
+
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
4358
3372
|
const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
|
|
4359
3373
|
FROM "${this.schemaName}".application_versions
|
|
4360
3374
|
WHERE ${scope}
|
|
@@ -4368,7 +3382,7 @@ class SystemDatabase {
|
|
|
4368
3382
|
async getLatestApplicationVersion(applicationName) {
|
|
4369
3383
|
const owner = applicationName ?? this.appName;
|
|
4370
3384
|
const params = [];
|
|
4371
|
-
const scope = this
|
|
3385
|
+
const scope = this.appNameFilter('application_name', owner, params);
|
|
4372
3386
|
const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
|
|
4373
3387
|
FROM "${this.schemaName}".application_versions
|
|
4374
3388
|
WHERE ${scope}
|
|
@@ -4392,7 +3406,7 @@ class SystemDatabase {
|
|
|
4392
3406
|
*/
|
|
4393
3407
|
async listQueues(applicationName) {
|
|
4394
3408
|
const params = [];
|
|
4395
|
-
const scope = this
|
|
3409
|
+
const scope = this.observabilityFilter('application_name', applicationName, params);
|
|
4396
3410
|
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
4397
3411
|
FROM "${this.schemaName}".queues
|
|
4398
3412
|
WHERE ${scope}`, params);
|
|
@@ -4440,7 +3454,7 @@ class SystemDatabase {
|
|
|
4440
3454
|
application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
|
|
4441
3455
|
: `ON CONFLICT (name) DO NOTHING`;
|
|
4442
3456
|
const owner = record.applicationName ?? this.appName;
|
|
4443
|
-
const client = await this
|
|
3457
|
+
const client = await this.connect();
|
|
4444
3458
|
try {
|
|
4445
3459
|
await client.query('BEGIN');
|
|
4446
3460
|
const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE name = $1`, [record.name]);
|
|
@@ -4565,7 +3579,7 @@ class SystemDatabase {
|
|
|
4565
3579
|
throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
|
|
4566
3580
|
}
|
|
4567
3581
|
// Never a merge: queue, schedule, and version names are globally unique whatever their owner, so this cannot collide.
|
|
4568
|
-
const client = await this
|
|
3582
|
+
const client = await this.connect();
|
|
4569
3583
|
let queues, schedules, versions, inFlight;
|
|
4570
3584
|
try {
|
|
4571
3585
|
await client.query('BEGIN');
|
|
@@ -4875,7 +3889,7 @@ class SystemDatabase {
|
|
|
4875
3889
|
const startTimeMs = Date.now();
|
|
4876
3890
|
// Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
|
|
4877
3891
|
const endTimeMs = startTimeMs + Math.ceil(durationMS);
|
|
4878
|
-
const client = await this
|
|
3892
|
+
const client = await this.connect();
|
|
4879
3893
|
try {
|
|
4880
3894
|
const res = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
4881
3895
|
if (res) {
|
|
@@ -4942,7 +3956,7 @@ class SystemDatabase {
|
|
|
4942
3956
|
};
|
|
4943
3957
|
let acquired = null;
|
|
4944
3958
|
try {
|
|
4945
|
-
const client = await this
|
|
3959
|
+
const client = await this.connect();
|
|
4946
3960
|
acquired = client;
|
|
4947
3961
|
if (this.#abandonIfStopped(client))
|
|
4948
3962
|
return;
|
|
@@ -5199,18 +4213,6 @@ __decorate([
|
|
|
5199
4213
|
__metadata("design:paramtypes", [String]),
|
|
5200
4214
|
__metadata("design:returntype", Promise)
|
|
5201
4215
|
], SystemDatabase.prototype, "getQueuePartitions", null);
|
|
5202
|
-
__decorate([
|
|
5203
|
-
dbRetry(),
|
|
5204
|
-
__metadata("design:type", Function),
|
|
5205
|
-
__metadata("design:paramtypes", [Number]),
|
|
5206
|
-
__metadata("design:returntype", Promise)
|
|
5207
|
-
], SystemDatabase.prototype, "listTimedOutWorkflowIds", null);
|
|
5208
|
-
__decorate([
|
|
5209
|
-
dbRetry(),
|
|
5210
|
-
__metadata("design:type", Function),
|
|
5211
|
-
__metadata("design:paramtypes", [String, String, Array]),
|
|
5212
|
-
__metadata("design:returntype", Promise)
|
|
5213
|
-
], SystemDatabase.prototype, "getMetrics", null);
|
|
5214
4216
|
__decorate([
|
|
5215
4217
|
dbRetry(),
|
|
5216
4218
|
__metadata("design:type", Function),
|