@dbos-inc/dbos-sdk 5.4.11-preview → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/schemas/system_db_schema.d.ts +7 -0
- package/dist/schemas/system_db_schema.d.ts.map +1 -1
- package/dist/schemas/system_db_schema.js.map +1 -1
- package/dist/src/client.d.ts +26 -2
- package/dist/src/client.d.ts.map +1 -1
- package/dist/src/client.js +65 -1
- package/dist/src/client.js.map +1 -1
- package/dist/src/config.d.ts.map +1 -1
- package/dist/src/config.js +3 -0
- package/dist/src/config.js.map +1 -1
- package/dist/src/dbos-executor.d.ts +26 -1
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js +50 -7
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts +74 -5
- package/dist/src/dbos.d.ts.map +1 -1
- package/dist/src/dbos.js +139 -22
- package/dist/src/dbos.js.map +1 -1
- package/dist/src/debugpoint.d.ts +1 -0
- package/dist/src/debugpoint.d.ts.map +1 -1
- package/dist/src/debugpoint.js +4 -2
- package/dist/src/debugpoint.js.map +1 -1
- package/dist/src/enterprise.d.ts +56 -1
- package/dist/src/enterprise.d.ts.map +1 -1
- package/dist/src/enterprise.js +26 -4
- package/dist/src/enterprise.js.map +1 -1
- package/dist/src/error.d.ts +42 -0
- package/dist/src/error.d.ts.map +1 -1
- package/dist/src/error.js +85 -1
- package/dist/src/error.js.map +1 -1
- package/dist/src/external_storage.d.ts +27 -0
- package/dist/src/external_storage.d.ts.map +1 -0
- package/dist/src/external_storage.js +28 -0
- package/dist/src/external_storage.js.map +1 -0
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/scheduler/scheduler.d.ts +78 -3
- package/dist/src/scheduler/scheduler.d.ts.map +1 -1
- package/dist/src/scheduler/scheduler.js +414 -173
- package/dist/src/scheduler/scheduler.js.map +1 -1
- package/dist/src/sysdb_migrations/internal/migrations.js +82 -0
- package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
- package/dist/src/system_database.d.ts +51 -10
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +418 -207
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/utils.d.ts +2 -0
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +9 -1
- package/dist/src/utils.js.map +1 -1
- package/dist/src/wfqueue.js +1 -1
- package/dist/src/wfqueue.js.map +1 -1
- package/dist/src/workflow.d.ts +3 -1
- package/dist/src/workflow.d.ts.map +1 -1
- package/dist/src/workflow.js +16 -3
- package/dist/src/workflow.js.map +1 -1
- package/dist/src/workflow_management.d.ts +16 -4
- package/dist/src/workflow_management.d.ts.map +1 -1
- package/dist/src/workflow_management.js +74 -32
- package/dist/src/workflow_management.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
|
@@ -32,7 +32,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
32
32
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
33
33
|
};
|
|
34
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
35
|
-
exports.SystemDatabase = exports.createSystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.qualify = exports.holdsShardedSchema = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
35
|
+
exports.SystemDatabase = exports.createSystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.qualify = exports.holdsShardedSchema = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.compareCodeUnits = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
36
36
|
const dbos_executor_1 = require("./dbos-executor");
|
|
37
37
|
const pg_1 = require("pg");
|
|
38
38
|
const error_1 = require("./error");
|
|
@@ -97,6 +97,11 @@ exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutM
|
|
|
97
97
|
exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = 60_000;
|
|
98
98
|
// How long a background job's lock may sit idle before Postgres ends its holder's session.
|
|
99
99
|
exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = 5000;
|
|
100
|
+
/** Orders strings by code unit, which matches Postgres's "C" collation outside surrogate pairs. */
|
|
101
|
+
function compareCodeUnits(a, b) {
|
|
102
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
103
|
+
}
|
|
104
|
+
exports.compareCodeUnits = compareCodeUnits;
|
|
100
105
|
/** The timeout is rendered into a `SET idle_in_transaction_session_timeout`, which takes only an in-range integer. */
|
|
101
106
|
function validateIdleTransactionTimeoutMs(value) {
|
|
102
107
|
if (value !== undefined && (!Number.isFinite(value) || value > MAX_STATEMENT_TIMEOUT_MS)) {
|
|
@@ -620,6 +625,15 @@ function dbRetry(options = {}) {
|
|
|
620
625
|
}
|
|
621
626
|
/** Build the system database these options describe. */
|
|
622
627
|
function createSystemDatabase(options) {
|
|
628
|
+
const sysdb = createBaseSystemDatabase(options);
|
|
629
|
+
if (options.suspendMode) {
|
|
630
|
+
const { Suspend } = enterprise.load(enterprise.SUSPEND.description);
|
|
631
|
+
sysdb.suspend = new Suspend(sysdb);
|
|
632
|
+
}
|
|
633
|
+
return sysdb;
|
|
634
|
+
}
|
|
635
|
+
exports.createSystemDatabase = createSystemDatabase;
|
|
636
|
+
function createBaseSystemDatabase(options) {
|
|
623
637
|
if (options.sharding) {
|
|
624
638
|
const { ShardedSystemDatabase } = enterprise.load(enterprise.SHARDING.description);
|
|
625
639
|
return new ShardedSystemDatabase({
|
|
@@ -637,7 +651,6 @@ function createSystemDatabase(options) {
|
|
|
637
651
|
}
|
|
638
652
|
return new SystemDatabase(options.systemDatabaseUrl, options.logger, options.serializer, options.sysDbPoolSize, options.systemDatabasePool, options.schemaName, options.useListenNotify, options.pollingConcurrency, options.notificationCoalesceMs, options.appName, options.observabilityQueryTimeoutMs, options.idleTransactionTimeoutMs);
|
|
639
653
|
}
|
|
640
|
-
exports.createSystemDatabase = createSystemDatabase;
|
|
641
654
|
/**
|
|
642
655
|
* General notes:
|
|
643
656
|
* The responsibilities of the `SystemDatabase` are to store data for workflows, and
|
|
@@ -681,8 +694,11 @@ class SystemDatabase {
|
|
|
681
694
|
dbPollingIntervalStreamMs = 1000;
|
|
682
695
|
dbPollingIntervalCancelMs = 1000;
|
|
683
696
|
shouldUseDBNotifications = true;
|
|
697
|
+
// How long a background job's lock may sit idle before Postgres ends its holder's session.
|
|
698
|
+
backgroundJobLockIdleTimeoutMs = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS;
|
|
684
699
|
notificationsMap = new NotificationMap();
|
|
685
700
|
workflowEventsMap = new NotificationMap();
|
|
701
|
+
suspend = undefined;
|
|
686
702
|
streamsMap = new NotificationMap();
|
|
687
703
|
customPool = false;
|
|
688
704
|
// Interval for coalescing LISTEN/NOTIFY notifications pushed off the write path (Postgres + L/N only).
|
|
@@ -902,6 +918,23 @@ class SystemDatabase {
|
|
|
902
918
|
claimGate(_queueName, _partitionKey, _params) {
|
|
903
919
|
return 'TRUE';
|
|
904
920
|
}
|
|
921
|
+
/** Statement whose `acquired` says whether a claim took its budget scope's transaction lock; no row means nothing to claim. */
|
|
922
|
+
claimLockQuery(queueName, partitionKey, params) {
|
|
923
|
+
const name = JSON.stringify(['dbos.queue_claim', this.schemaName, queueName, partitionKey ?? null]);
|
|
924
|
+
params.push(advisoryLockKey(name).toString());
|
|
925
|
+
return `SELECT pg_try_advisory_xact_lock($${params.length}) AS acquired`;
|
|
926
|
+
}
|
|
927
|
+
/** Statement whose `held` lists which of `partitionKeys` a sweep's transaction locked; no row means nothing to claim. */
|
|
928
|
+
partitionLockQuery(queueName, partitionKeys, params) {
|
|
929
|
+
const name = JSON.stringify(['dbos.partition_claim', this.schemaName, queueName]);
|
|
930
|
+
// Two-int keys: the queue, then the partition, hashed so every worker computes the same lock.
|
|
931
|
+
const lockIDs = partitionKeys.map((pk) => (0, crypto_1.createHash)('sha256').update(pk).digest().readInt32BE(0));
|
|
932
|
+
params.push(partitionKeys, lockIDs, Number(BigInt.asIntN(32, advisoryLockKey(name))));
|
|
933
|
+
const n = params.length;
|
|
934
|
+
return `SELECT coalesce(array_agg(p.pk), '{}') AS held
|
|
935
|
+
FROM unnest($${n - 2}::text[], $${n - 1}::int4[]) AS p(pk, lock_id)
|
|
936
|
+
WHERE pg_try_advisory_xact_lock($${n}::int4, p.lock_id)`;
|
|
937
|
+
}
|
|
905
938
|
/** Predicate confining a statement that names no single workflow to the current shard. */
|
|
906
939
|
scoped(_alias, _params) {
|
|
907
940
|
return 'TRUE';
|
|
@@ -958,24 +991,29 @@ class SystemDatabase {
|
|
|
958
991
|
}
|
|
959
992
|
return (0, shard_keys_1.bindShardKeys)([[workflowID, key]], callback);
|
|
960
993
|
}
|
|
961
|
-
/** Run `callback` under this application's lock for a background job, passing
|
|
994
|
+
/** Run `callback` under this application's lock for a background job, passing undefined if refused, else a call renewing it in place. */
|
|
962
995
|
async backgroundJobLock(jobName, callback) {
|
|
963
996
|
// CockroachDB grants every advisory lock, so each process runs every job.
|
|
964
997
|
if (await this.#isCockroach()) {
|
|
965
|
-
return await callback(
|
|
998
|
+
return await callback(() => Promise.resolve());
|
|
966
999
|
}
|
|
967
1000
|
// Leading 8 bytes of SHA-256, big-endian signed, as for the retention lock.
|
|
968
1001
|
const key = advisoryLockKey(`dbos.background_job.${this.schemaName}.${this.appName ?? ''}.${jobName}`);
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
1002
|
+
const timeout = `'${Math.trunc(this.backgroundJobLockIdleTimeoutMs)}'`;
|
|
1003
|
+
return await this.lockedBackgroundJob(jobName, `SELECT pg_try_advisory_xact_lock(${key}) FROM set_config('idle_in_transaction_session_timeout', ${timeout}, true)`,
|
|
1004
|
+
// Snapshot-free, so a concurrent index build never waits on it; fails if the session ended.
|
|
1005
|
+
async (client) => {
|
|
1006
|
+
await client.query(`SET LOCAL idle_in_transaction_session_timeout = ${timeout}`);
|
|
1007
|
+
}, callback);
|
|
1008
|
+
}
|
|
1009
|
+
/** Take a transaction-scoped lock with literal `lockQuery` (its first column says if taken), as a bound statement's portal keeps its snapshot (an xmin) while the session idles. */
|
|
1010
|
+
async lockedBackgroundJob(jobName, lockQuery, renew, callback) {
|
|
973
1011
|
const client = await this.connect();
|
|
974
1012
|
try {
|
|
975
|
-
await client.query('BEGIN');
|
|
1013
|
+
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
976
1014
|
// The local idle timeout ends a frozen holder's session, releasing its lock.
|
|
977
|
-
const { rows } = await client.query(lockQuery,
|
|
978
|
-
return await callback(rows[0]?.
|
|
1015
|
+
const { rows } = await client.query({ text: lockQuery, rowMode: 'array' });
|
|
1016
|
+
return await callback(rows[0]?.[0] === true ? () => renew(client) : undefined);
|
|
979
1017
|
}
|
|
980
1018
|
finally {
|
|
981
1019
|
try {
|
|
@@ -1280,7 +1318,7 @@ class SystemDatabase {
|
|
|
1280
1318
|
* Stamp created_at monotonic within each partition key so per-key order holds across batches.
|
|
1281
1319
|
* Unordered rows (no partition key) get wall-clock time and never touch the cursors.
|
|
1282
1320
|
*/
|
|
1283
|
-
async
|
|
1321
|
+
async assignBatchCreatedAt(statuses) {
|
|
1284
1322
|
const nowMS = Date.now();
|
|
1285
1323
|
const batchKeys = new Set();
|
|
1286
1324
|
for (const status of statuses) {
|
|
@@ -1325,9 +1363,8 @@ class SystemDatabase {
|
|
|
1325
1363
|
* shutdown. Callers retry this themselves.
|
|
1326
1364
|
*/
|
|
1327
1365
|
async enqueueWorkflows(statuses) {
|
|
1328
|
-
const inserted = new Set();
|
|
1329
1366
|
if (statuses.length === 0)
|
|
1330
|
-
return
|
|
1367
|
+
return new Set();
|
|
1331
1368
|
for (const status of statuses) {
|
|
1332
1369
|
if (status.status !== workflow_1.StatusString.ENQUEUED) {
|
|
1333
1370
|
throw new error_1.DBOSError(`enqueueWorkflows only accepts ${workflow_1.StatusString.ENQUEUED} workflows, but ${status.workflowUUID} is ${status.status}`);
|
|
@@ -1336,7 +1373,63 @@ class SystemDatabase {
|
|
|
1336
1373
|
throw new error_1.DBOSError(`enqueueWorkflows does not support deduplication IDs, but ${status.workflowUUID} has one`);
|
|
1337
1374
|
}
|
|
1338
1375
|
}
|
|
1339
|
-
|
|
1376
|
+
// Stamped before the transaction, as seeding a key's cursor reads on another connection.
|
|
1377
|
+
const createdAts = await this.assignBatchCreatedAt(statuses);
|
|
1378
|
+
const client = await this.connect();
|
|
1379
|
+
try {
|
|
1380
|
+
return await this.inTransaction(client, () => this.insertWorkflowBatch(client, statuses, createdAts));
|
|
1381
|
+
}
|
|
1382
|
+
finally {
|
|
1383
|
+
client.release();
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
/** Batch-enqueue scheduled runs, skipping existing IDs, stamping last_fired_at from `firedAt` (ID to tick) for those inserted, in one transaction. */
|
|
1387
|
+
async enqueueScheduledWorkflows(statuses, firedAt) {
|
|
1388
|
+
if (statuses.length === 0)
|
|
1389
|
+
return new Set();
|
|
1390
|
+
// Stamped before the transaction, as seeding a key's cursor reads on another connection.
|
|
1391
|
+
const createdAts = await this.assignBatchCreatedAt(statuses);
|
|
1392
|
+
const client = await this.connect();
|
|
1393
|
+
try {
|
|
1394
|
+
return await this.inTransaction(client, async () => {
|
|
1395
|
+
const inserted = await this.insertWorkflowBatch(client, statuses, createdAts);
|
|
1396
|
+
await this.stampLastFiredAt(client, statuses, firedAt, inserted);
|
|
1397
|
+
return inserted;
|
|
1398
|
+
});
|
|
1399
|
+
}
|
|
1400
|
+
finally {
|
|
1401
|
+
client.release();
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
/** Set each schedule's last_fired_at to its last inserted tick, `statuses` being in tick order. */
|
|
1405
|
+
async stampLastFiredAt(client, statuses, firedAt, inserted) {
|
|
1406
|
+
const latest = new Map();
|
|
1407
|
+
for (const status of statuses) {
|
|
1408
|
+
const fired = firedAt.get(status.workflowUUID);
|
|
1409
|
+
if (inserted.has(status.workflowUUID) && fired !== undefined && status.scheduleName !== undefined) {
|
|
1410
|
+
latest.set(status.scheduleName, fired);
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
if (latest.size === 0)
|
|
1414
|
+
return;
|
|
1415
|
+
// Sorted, like applySchedules, so concurrent writers lock schedule rows in one order.
|
|
1416
|
+
const names = Array.from(latest.keys()).sort(compareCodeUnits);
|
|
1417
|
+
const lockParams = [names];
|
|
1418
|
+
await client.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules
|
|
1419
|
+
WHERE ${this.global(undefined, lockParams)} AND schedule_name = ANY($1::text[])
|
|
1420
|
+
ORDER BY schedule_name COLLATE "C" FOR UPDATE`, lockParams);
|
|
1421
|
+
const params = [names];
|
|
1422
|
+
const cases = names.map((name) => `WHEN schedule_name = $${params.push(name)} THEN $${params.push(latest.get(name))}`);
|
|
1423
|
+
await client.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = CASE ${cases.join(' ')} END
|
|
1424
|
+
WHERE ${this.global(undefined, params)} AND schedule_name = ANY($1::text[])`, params);
|
|
1425
|
+
}
|
|
1426
|
+
/** Insert these ENQUEUED workflows' rows on `client` with `stamped` from assignBatchCreatedAt, skipping existing IDs. Returns the IDs inserted. */
|
|
1427
|
+
async insertWorkflowBatch(client, batch, stamped) {
|
|
1428
|
+
// Inserted in one global order, so batches overlapping across processes wait on each other instead of deadlocking.
|
|
1429
|
+
const order = batch.map((_, i) => i).sort((a, b) => compareCodeUnits(batch[a].workflowUUID, batch[b].workflowUUID));
|
|
1430
|
+
const statuses = order.map((i) => batch[i]);
|
|
1431
|
+
const createdAts = order.map((i) => stamped[i]);
|
|
1432
|
+
const inserted = new Set();
|
|
1340
1433
|
const columns = [
|
|
1341
1434
|
'workflow_uuid',
|
|
1342
1435
|
'status',
|
|
@@ -1367,76 +1460,64 @@ class SystemDatabase {
|
|
|
1367
1460
|
'schedule_name',
|
|
1368
1461
|
'application_name',
|
|
1369
1462
|
];
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
const
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
const rows = chunk.map((status, i) => {
|
|
1382
|
-
const createdAt = createdAts[start + i];
|
|
1383
|
-
return [
|
|
1384
|
-
...Object.values(keyed[i]),
|
|
1385
|
-
status.workflowUUID,
|
|
1386
|
-
status.status,
|
|
1387
|
-
status.workflowName,
|
|
1388
|
-
// For cross-language compatibility, these MUST be NULL in the database when not set
|
|
1389
|
-
status.workflowClassName === '' ? null : status.workflowClassName,
|
|
1390
|
-
status.workflowConfigName === '' ? null : status.workflowConfigName,
|
|
1391
|
-
status.queueName ?? null,
|
|
1392
|
-
status.authenticatedUser,
|
|
1393
|
-
status.assumedRole,
|
|
1394
|
-
JSON.stringify(status.authenticatedRoles),
|
|
1395
|
-
status.executorId,
|
|
1396
|
-
status.applicationVersion ?? null,
|
|
1397
|
-
status.applicationID,
|
|
1398
|
-
createdAt,
|
|
1399
|
-
0,
|
|
1400
|
-
createdAt,
|
|
1401
|
-
status.timeoutMS ?? null,
|
|
1402
|
-
status.deadlineEpochMS ?? null,
|
|
1403
|
-
// Legacy column: the payload lives in workflow_input.
|
|
1404
|
-
null,
|
|
1405
|
-
null,
|
|
1406
|
-
status.priority,
|
|
1407
|
-
status.queuePartitionKey ?? null,
|
|
1408
|
-
status.parentWorkflowID ?? null,
|
|
1409
|
-
status.serialization,
|
|
1410
|
-
null,
|
|
1411
|
-
status.delayUntilEpochMS ?? null,
|
|
1412
|
-
status.attributes ? JSON.stringify(status.attributes) : null,
|
|
1413
|
-
status.scheduleName ?? null,
|
|
1414
|
-
status.applicationName ?? null,
|
|
1415
|
-
];
|
|
1416
|
-
});
|
|
1417
|
-
const returned = await this.insertRows(client, 'workflow_status', [...Object.keys(keyed[0]), ...columns], rows, `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING RETURNING workflow_uuid`);
|
|
1418
|
-
for (const row of returned) {
|
|
1419
|
-
inserted.add(row.workflow_uuid);
|
|
1420
|
-
}
|
|
1421
|
-
}
|
|
1422
|
-
for (let start = 0; start < statuses.length; start += chunkSize) {
|
|
1423
|
-
const chunk = statuses.slice(start, start + chunkSize);
|
|
1424
|
-
const keyed = chunk.map((status) => this.wfValues(status.workflowUUID, status.shardKey));
|
|
1425
|
-
await this.insertRows(client, 'workflow_input', [...Object.keys(keyed[0]), 'workflow_uuid', 'inputs', 'retention_timestamp'], chunk.map((status, i) => [
|
|
1463
|
+
// Chunk to stay well under the bind-parameter limit.
|
|
1464
|
+
const chunkSize = 500;
|
|
1465
|
+
for (let start = 0; start < statuses.length; start += chunkSize) {
|
|
1466
|
+
const chunk = statuses.slice(start, start + chunkSize);
|
|
1467
|
+
const keyed = chunk.map((status) => ({
|
|
1468
|
+
...this.wfValues(status.workflowUUID, status.shardKey),
|
|
1469
|
+
...this.parentValues(status.parentWorkflowID),
|
|
1470
|
+
}));
|
|
1471
|
+
const rows = chunk.map((status, i) => {
|
|
1472
|
+
const createdAt = createdAts[start + i];
|
|
1473
|
+
return [
|
|
1426
1474
|
...Object.values(keyed[i]),
|
|
1427
1475
|
status.workflowUUID,
|
|
1428
|
-
status.
|
|
1429
|
-
|
|
1430
|
-
|
|
1476
|
+
status.status,
|
|
1477
|
+
status.workflowName,
|
|
1478
|
+
// For cross-language compatibility, these MUST be NULL in the database when not set
|
|
1479
|
+
status.workflowClassName === '' ? null : status.workflowClassName,
|
|
1480
|
+
status.workflowConfigName === '' ? null : status.workflowConfigName,
|
|
1481
|
+
status.queueName ?? null,
|
|
1482
|
+
status.authenticatedUser,
|
|
1483
|
+
status.assumedRole,
|
|
1484
|
+
JSON.stringify(status.authenticatedRoles),
|
|
1485
|
+
status.executorId,
|
|
1486
|
+
status.applicationVersion ?? null,
|
|
1487
|
+
status.applicationID,
|
|
1488
|
+
createdAt,
|
|
1489
|
+
0,
|
|
1490
|
+
createdAt,
|
|
1491
|
+
status.timeoutMS ?? null,
|
|
1492
|
+
status.deadlineEpochMS ?? null,
|
|
1493
|
+
// Legacy column: the payload lives in workflow_input.
|
|
1494
|
+
null,
|
|
1495
|
+
null,
|
|
1496
|
+
status.priority,
|
|
1497
|
+
status.queuePartitionKey ?? null,
|
|
1498
|
+
status.parentWorkflowID ?? null,
|
|
1499
|
+
status.serialization,
|
|
1500
|
+
null,
|
|
1501
|
+
status.delayUntilEpochMS ?? null,
|
|
1502
|
+
status.attributes ? JSON.stringify(status.attributes) : null,
|
|
1503
|
+
status.scheduleName ?? null,
|
|
1504
|
+
status.applicationName ?? null,
|
|
1505
|
+
];
|
|
1506
|
+
});
|
|
1507
|
+
const returned = await this.insertRows(client, 'workflow_status', [...Object.keys(keyed[0]), ...columns], rows, `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING RETURNING workflow_uuid`);
|
|
1508
|
+
for (const row of returned) {
|
|
1509
|
+
inserted.add(row.workflow_uuid);
|
|
1431
1510
|
}
|
|
1432
|
-
await client.query('COMMIT');
|
|
1433
1511
|
}
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1512
|
+
for (let start = 0; start < statuses.length; start += chunkSize) {
|
|
1513
|
+
const chunk = statuses.slice(start, start + chunkSize);
|
|
1514
|
+
const keyed = chunk.map((status) => this.wfValues(status.workflowUUID, status.shardKey));
|
|
1515
|
+
await this.insertRows(client, 'workflow_input', [...Object.keys(keyed[0]), 'workflow_uuid', 'inputs', 'retention_timestamp'], chunk.map((status, i) => [
|
|
1516
|
+
...Object.values(keyed[i]),
|
|
1517
|
+
status.workflowUUID,
|
|
1518
|
+
status.input,
|
|
1519
|
+
createdAts[start + i],
|
|
1520
|
+
]), `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`);
|
|
1440
1521
|
}
|
|
1441
1522
|
return inserted;
|
|
1442
1523
|
}
|
|
@@ -1756,13 +1837,20 @@ class SystemDatabase {
|
|
|
1756
1837
|
return { isPatched: true, hasEntry: checkpointName === patchName };
|
|
1757
1838
|
}
|
|
1758
1839
|
// Nondeprecated - skip matching entry, unpatched if nonmatching entry,
|
|
1759
|
-
// If
|
|
1840
|
+
// If the history ends before this point, we insert an entry that indicates it is patched, as an owner-checked checkpoint.
|
|
1841
|
+
// If the history has a checkpoint past this point but none here, the execution reached this point on pre-patch code (e.g., recv).
|
|
1842
|
+
// It is unpatched, and nothing is consumed.
|
|
1760
1843
|
const client = await this.connect();
|
|
1761
1844
|
try {
|
|
1762
1845
|
return await this.inTransaction(client, async () => {
|
|
1763
|
-
const
|
|
1764
|
-
|
|
1765
|
-
|
|
1846
|
+
const params = [functionID];
|
|
1847
|
+
const { rows } = await client.query(`SELECT function_id, function_name
|
|
1848
|
+
FROM "${this.schemaName}".operation_outputs
|
|
1849
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND function_id>=$1
|
|
1850
|
+
ORDER BY function_id
|
|
1851
|
+
LIMIT 1`, params);
|
|
1852
|
+
if (rows.length !== 0) {
|
|
1853
|
+
const matches = rows[0].function_id === functionID && rows[0].function_name === patchName;
|
|
1766
1854
|
return { isPatched: matches, hasEntry: matches };
|
|
1767
1855
|
}
|
|
1768
1856
|
const dn = Date.now();
|
|
@@ -1796,6 +1884,7 @@ class SystemDatabase {
|
|
|
1796
1884
|
const params = [workflow_1.StatusString.CANCELLED, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR];
|
|
1797
1885
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1798
1886
|
SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
1887
|
+
suspend_key = NULL,
|
|
1799
1888
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1800
1889
|
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1801
1890
|
WHERE ${this.wfIn(undefined, workflowIDs, params)}
|
|
@@ -1826,7 +1915,7 @@ class SystemDatabase {
|
|
|
1826
1915
|
await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1827
1916
|
SET status = $1, owner_xid = NULL, queue_name = $2, recovery_attempts = 0,
|
|
1828
1917
|
workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
|
|
1829
|
-
started_at_epoch_ms = NULL,
|
|
1918
|
+
started_at_epoch_ms = NULL, suspend_key = NULL,
|
|
1830
1919
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1831
1920
|
completed_at = NULL
|
|
1832
1921
|
WHERE ${this.wfIn(undefined, workflowIDs, updateParams)}
|
|
@@ -2043,7 +2132,7 @@ class SystemDatabase {
|
|
|
2043
2132
|
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
|
|
2044
2133
|
}
|
|
2045
2134
|
const status = statusRows[0].status;
|
|
2046
|
-
if (
|
|
2135
|
+
if ((0, workflow_1.isWorkflowActive)(status)) {
|
|
2047
2136
|
throw new error_1.DBOSError(`Cannot rewind ${workflowID} (${status}): only a workflow in a terminal state can be rewound, so cancel it first`);
|
|
2048
2137
|
}
|
|
2049
2138
|
// Roll workflow_events back to the last value published before the cut, using
|
|
@@ -2101,7 +2190,7 @@ class SystemDatabase {
|
|
|
2101
2190
|
const { rowCount } = await client.query(`UPDATE "${schema}".workflow_status
|
|
2102
2191
|
SET status = $1, owner_xid = NULL, queue_name = $2, queue_partition_key = $3, recovery_attempts = 0,
|
|
2103
2192
|
workflow_deadline_epoch_ms = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
2104
|
-
completed_at = NULL, output = NULL, error = NULL,
|
|
2193
|
+
suspend_key = NULL, completed_at = NULL, output = NULL, error = NULL,
|
|
2105
2194
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint${setVersion}
|
|
2106
2195
|
WHERE ${this.wf(undefined, workflowID, params)} AND status = $4`, params);
|
|
2107
2196
|
if (rowCount !== 1) {
|
|
@@ -2608,7 +2697,7 @@ class SystemDatabase {
|
|
|
2608
2697
|
const ids = this.wfIn(undefined, workflowIds, params);
|
|
2609
2698
|
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
2610
2699
|
WHERE ${ids}
|
|
2611
|
-
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
|
|
2700
|
+
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')
|
|
2612
2701
|
LIMIT 1`, params));
|
|
2613
2702
|
return rows[0]?.workflow_uuid;
|
|
2614
2703
|
}
|
|
@@ -2634,7 +2723,7 @@ class SystemDatabase {
|
|
|
2634
2723
|
const ids = this.wfIn(undefined, workflowIds, params);
|
|
2635
2724
|
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
2636
2725
|
WHERE ${ids}
|
|
2637
|
-
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, params));
|
|
2726
|
+
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')`, params));
|
|
2638
2727
|
return rows.map((row) => row.workflow_uuid);
|
|
2639
2728
|
}
|
|
2640
2729
|
// ==================== Sleep ====================
|
|
@@ -2708,6 +2797,16 @@ class SystemDatabase {
|
|
|
2708
2797
|
return await this.sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2709
2798
|
}
|
|
2710
2799
|
async sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2800
|
+
if (db === this.pool) {
|
|
2801
|
+
// All sends need a transaction now because they wake up suspended workflows unconditionally.
|
|
2802
|
+
const client = await this.connect();
|
|
2803
|
+
try {
|
|
2804
|
+
return await this.inTransaction(client, () => this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey));
|
|
2805
|
+
}
|
|
2806
|
+
finally {
|
|
2807
|
+
client.release();
|
|
2808
|
+
}
|
|
2809
|
+
}
|
|
2711
2810
|
topic = topic ?? this.nullTopic;
|
|
2712
2811
|
// An idempotency key is scoped per destination: suffix it with the recipient's workflow ID.
|
|
2713
2812
|
const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
|
|
@@ -2729,6 +2828,23 @@ class SystemDatabase {
|
|
|
2729
2828
|
}
|
|
2730
2829
|
throw err;
|
|
2731
2830
|
}
|
|
2831
|
+
await this.wakeReceivers(db, [destinationID], [topic]);
|
|
2832
|
+
}
|
|
2833
|
+
/**
|
|
2834
|
+
* Make SUSPENDED receivers waiting on one of `topics` due, for the background pass to re-enqueue.
|
|
2835
|
+
* Runs on every send, whether or not this process uses suspend mode, so any sender wakes a workflow
|
|
2836
|
+
* another process parked. A primary-key probe per destination; it matches nothing in the common case.
|
|
2837
|
+
*
|
|
2838
|
+
* The match is destinations x topics, so a batch sending A on topic X and B on topic Y also wakes A
|
|
2839
|
+
* if it waits on Y. The spurious wake-up is harmless: the replayed recv finds no message and parks again.
|
|
2840
|
+
*/
|
|
2841
|
+
async wakeReceivers(db, destinations, topics) {
|
|
2842
|
+
const params = [topics];
|
|
2843
|
+
await db.query(
|
|
2844
|
+
// Status is tested in SET, not WHERE, so the planner cannot scan idx_workflow_status_suspended.
|
|
2845
|
+
`UPDATE "${this.schemaName}".workflow_status
|
|
2846
|
+
SET delay_until_epoch_ms = CASE WHEN status = '${workflow_1.StatusString.SUSPENDED}' THEN 0 ELSE delay_until_epoch_ms END
|
|
2847
|
+
WHERE ${this.wfIn(undefined, destinations, params)} AND suspend_key = ANY($1::text[])`, params);
|
|
2732
2848
|
}
|
|
2733
2849
|
async recv(workflowID, functionID, timeoutFunctionID, topic, timeoutSeconds = dbos_executor_1.DBOSExecutor.defaultNotificationTimeoutSec, pollingIntervalMs) {
|
|
2734
2850
|
topic = topic ?? this.nullTopic;
|
|
@@ -2782,7 +2898,24 @@ class SystemDatabase {
|
|
|
2782
2898
|
this.notificationsMap.deregisterCallback(cbr);
|
|
2783
2899
|
}
|
|
2784
2900
|
}
|
|
2785
|
-
await this.
|
|
2901
|
+
return await this.recvConsume(workflowID, functionID, topic, startTime);
|
|
2902
|
+
}
|
|
2903
|
+
/**
|
|
2904
|
+
* Consume and return the oldest message on `topic`, or null if there is none, recording it as the recv at `functionID`. Idempotent on `functionID`.
|
|
2905
|
+
*/
|
|
2906
|
+
async recvConsume(workflowID, functionID, topic, startTime) {
|
|
2907
|
+
topic = topic ?? this.nullTopic;
|
|
2908
|
+
const res = await this.getOperationResultAndThrowIfCancelled(workflowID, functionID);
|
|
2909
|
+
if (res) {
|
|
2910
|
+
if (res.functionName !== exports.DBOS_FUNCNAME_RECV) {
|
|
2911
|
+
throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_RECV, res.functionName);
|
|
2912
|
+
}
|
|
2913
|
+
const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
|
|
2914
|
+
if (ownerXid !== undefined && (await this.getWorkflowOwner(workflowID)) !== ownerXid) {
|
|
2915
|
+
throw new error_1.DBOSWorkflowConflictError(workflowID);
|
|
2916
|
+
}
|
|
2917
|
+
return { serializedValue: res.output, serialization: res.serialization ?? null };
|
|
2918
|
+
}
|
|
2786
2919
|
// Transactionally consume and return the message if it's in the DB, otherwise return null.
|
|
2787
2920
|
let message = null;
|
|
2788
2921
|
let serialization = null;
|
|
@@ -3148,15 +3281,16 @@ class SystemDatabase {
|
|
|
3148
3281
|
async cancelTimedOutWorkflows(limit) {
|
|
3149
3282
|
const params = [workflow_1.StatusString.CANCELLED, Date.now(), limit];
|
|
3150
3283
|
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
3151
|
-
// Literal statuses let the planner match
|
|
3284
|
+
// Literal statuses let the planner match idx_workflow_status_deadline_v2 under any plan mode.
|
|
3152
3285
|
// SKIP LOCKED leaves a row a dequeue or a peer's sweep holds for the next sweep.
|
|
3153
3286
|
const { rows } = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
3154
3287
|
SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
3288
|
+
suspend_key = NULL,
|
|
3155
3289
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
3156
3290
|
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
3157
3291
|
WHERE workflow_uuid IN (
|
|
3158
3292
|
SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
3159
|
-
WHERE status IN ('${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.DELAYED}')
|
|
3293
|
+
WHERE status IN ('${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')
|
|
3160
3294
|
AND workflow_deadline_epoch_ms IS NOT NULL
|
|
3161
3295
|
AND workflow_deadline_epoch_ms <= $2
|
|
3162
3296
|
AND ${scope}
|
|
@@ -3210,17 +3344,49 @@ class SystemDatabase {
|
|
|
3210
3344
|
queue.partitionConcurrency !== undefined ||
|
|
3211
3345
|
queue.rateLimit !== undefined ||
|
|
3212
3346
|
queue.partitionRateLimit !== undefined;
|
|
3213
|
-
// Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
|
|
3347
|
+
// Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it: lock the whole queue.
|
|
3214
3348
|
const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
|
|
3349
|
+
// Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
|
|
3350
|
+
let maxTasks = Infinity;
|
|
3351
|
+
// The first limit found with no room, which the caller reads as why nothing was claimed.
|
|
3352
|
+
let stopReason = undefined;
|
|
3353
|
+
const limit = (room, reason) => {
|
|
3354
|
+
maxTasks = Math.min(maxTasks, room);
|
|
3355
|
+
if (maxTasks <= 0)
|
|
3356
|
+
stopReason ??= reason;
|
|
3357
|
+
};
|
|
3358
|
+
// This worker's own limits need no database, so a full worker returns before taking the claim lock a peer could use.
|
|
3359
|
+
if (queue.workerConcurrency !== undefined) {
|
|
3360
|
+
// Use the in-memory registry for this worker's running count — avoids a DB round trip.
|
|
3361
|
+
limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
|
|
3362
|
+
}
|
|
3363
|
+
if (queue.partitionWorkerConcurrency !== undefined) {
|
|
3364
|
+
limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
|
|
3365
|
+
}
|
|
3366
|
+
if (maxTasks <= 0) {
|
|
3367
|
+
return { claimedIDs, stopReason };
|
|
3368
|
+
}
|
|
3215
3369
|
// A sharded database keeps the version row under another key, which the claim's transaction cannot span.
|
|
3216
3370
|
if (isLatestVersion === undefined && this.sharded) {
|
|
3217
3371
|
isLatestVersion = await this.isLatestVersion(appVersion);
|
|
3218
3372
|
}
|
|
3373
|
+
// Row locks only collide when claimers see the same queue head, so serialize claims on the budget's scope; CockroachDB has no advisory locks before 26.3.
|
|
3374
|
+
const useClaimLock = hasSharedBudget && (this.sharded || !(await this.#isCockroach()));
|
|
3219
3375
|
const client = await this.connect();
|
|
3220
3376
|
try {
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3377
|
+
if (useClaimLock) {
|
|
3378
|
+
await client.query('BEGIN');
|
|
3379
|
+
const lockParams = [];
|
|
3380
|
+
const { rows } = await client.query(this.claimLockQuery(queue.name, hasWriteSkew ? undefined : queuePartitionKey, lockParams), lockParams);
|
|
3381
|
+
if (rows[0]?.acquired !== true) {
|
|
3382
|
+
await client.query('COMMIT');
|
|
3383
|
+
// Another worker is claiming from this scope, so it still has work; the next poll retries.
|
|
3384
|
+
return rows.length > 0 ? { claimedIDs, stopReason: 'claim_lock' } : { claimedIDs };
|
|
3385
|
+
}
|
|
3386
|
+
}
|
|
3387
|
+
else if (hasSharedBudget) {
|
|
3388
|
+
// CockroachDB, without advisory locks before 26.3: SERIALIZABLE aborts claimers that spend the same budget.
|
|
3389
|
+
await client.query('BEGIN ISOLATION LEVEL SERIALIZABLE');
|
|
3224
3390
|
}
|
|
3225
3391
|
else {
|
|
3226
3392
|
await client.query('BEGIN');
|
|
@@ -3254,26 +3420,6 @@ class SystemDatabase {
|
|
|
3254
3420
|
WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
|
|
3255
3421
|
return Number(rows[0]?.count ?? 0);
|
|
3256
3422
|
};
|
|
3257
|
-
// Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
|
|
3258
|
-
let maxTasks = Infinity;
|
|
3259
|
-
// The first limit found with no room, which the caller reads as why nothing was claimed.
|
|
3260
|
-
let stopReason = undefined;
|
|
3261
|
-
const limit = (room, reason) => {
|
|
3262
|
-
maxTasks = Math.min(maxTasks, room);
|
|
3263
|
-
if (maxTasks <= 0)
|
|
3264
|
-
stopReason ??= reason;
|
|
3265
|
-
};
|
|
3266
|
-
if (queue.workerConcurrency !== undefined) {
|
|
3267
|
-
// Use the in-memory registry for this worker's running count — avoids a DB round trip.
|
|
3268
|
-
limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
|
|
3269
|
-
}
|
|
3270
|
-
if (queue.partitionWorkerConcurrency !== undefined) {
|
|
3271
|
-
limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
|
|
3272
|
-
}
|
|
3273
|
-
if (maxTasks <= 0) {
|
|
3274
|
-
await client.query('COMMIT');
|
|
3275
|
-
return { claimedIDs, stopReason };
|
|
3276
|
-
}
|
|
3277
3423
|
if (queue.rateLimit !== undefined) {
|
|
3278
3424
|
// Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
|
|
3279
3425
|
limit(await rateLimitRemaining(queue.rateLimit, false), 'limiter');
|
|
@@ -3404,7 +3550,9 @@ class SystemDatabase {
|
|
|
3404
3550
|
}
|
|
3405
3551
|
/** Max heads admitted per sweep: bounds dispatch, not the partition walk; an unconstrained sweep takes the lowest keys, so higher keys can wait under sustained load. */
|
|
3406
3552
|
partitionedDequeueSweepCap = 8192;
|
|
3407
|
-
/**
|
|
3553
|
+
/** Max partitions a sweep claims per transaction, which bounds the locks it holds at once and for how long. */
|
|
3554
|
+
partitionedDequeueBatchSize = 64;
|
|
3555
|
+
/** Dequeue each partition's head-of-line workflow, at most {@link partitionedDequeueSweepCap} per sweep in transactions of {@link partitionedDequeueBatchSize}; only valid for partition-concurrency-1 queues with no queue-wide limit. */
|
|
3408
3556
|
async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity,
|
|
3409
3557
|
// Written to every claimed row: the dispatches of this claim own their executions.
|
|
3410
3558
|
ownerXid = null, isLatestVersion) {
|
|
@@ -3421,13 +3569,31 @@ class SystemDatabase {
|
|
|
3421
3569
|
if (isLatestVersion === undefined && this.sharded) {
|
|
3422
3570
|
isLatestVersion = await this.isLatestVersion(appVersion);
|
|
3423
3571
|
}
|
|
3572
|
+
// Sweeps whose heads differ lock disjoint rows, so a claim also takes its partition's lock; CockroachDB, without advisory locks before 26.3, relies on its SERIALIZABLE default.
|
|
3573
|
+
const usePartitionLocks = this.sharded || !(await this.#isCockroach());
|
|
3424
3574
|
const client = await this.connect();
|
|
3425
3575
|
try {
|
|
3426
|
-
await client.query('BEGIN');
|
|
3427
3576
|
isLatestVersion ??= await this.isLatestVersion(appVersion, client);
|
|
3428
3577
|
const versionClause = (n) => isLatestVersion
|
|
3429
3578
|
? `(application_version = $${n} OR application_version IS NULL)`
|
|
3430
3579
|
: `application_version = $${n}`;
|
|
3580
|
+
// The partition's head: $1 queue, $2 ENQUEUED, `version` the app version, `pk` the partition key expression.
|
|
3581
|
+
const headOf = (pk, version, scope) => `SELECT workflow_uuid
|
|
3582
|
+
FROM "${this.schemaName}".workflow_status
|
|
3583
|
+
WHERE queue_name = $1 AND status = $2
|
|
3584
|
+
AND queue_partition_key = ${pk}
|
|
3585
|
+
AND ${versionClause(version)}
|
|
3586
|
+
AND ${scope}
|
|
3587
|
+
-- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
|
|
3588
|
+
ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
|
|
3589
|
+
LIMIT 1`;
|
|
3590
|
+
// No application filter by design: a mutual-exclusion probe must block on any owner's row.
|
|
3591
|
+
const noPending = (pk, pending, scope) => `NOT EXISTS (
|
|
3592
|
+
SELECT 1
|
|
3593
|
+
FROM "${this.schemaName}".workflow_status
|
|
3594
|
+
WHERE ${scope} AND queue_name = $1 AND status = $${pending}
|
|
3595
|
+
AND queue_partition_key IS NOT NULL AND queue_partition_key = ${pk}
|
|
3596
|
+
)`;
|
|
3431
3597
|
// This worker's own budget bounds the sweep alongside the cap.
|
|
3432
3598
|
const sweepLimit = Math.min(this.partitionedDequeueSweepCap, maxTasks);
|
|
3433
3599
|
// When the worker's budget is the binding constraint, probe partitions in random order to prevent starvation.
|
|
@@ -3442,6 +3608,7 @@ class SystemDatabase {
|
|
|
3442
3608
|
const candidateScope = `${this.scoped(undefined, candidateParams)} AND ${this.appNameFilter('application_name', this.appName, candidateParams)}`;
|
|
3443
3609
|
const probeScope = this.scoped(undefined, candidateParams);
|
|
3444
3610
|
// Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
|
|
3611
|
+
// The walk takes no lock: each batch below picks its heads again under its partitions' locks.
|
|
3445
3612
|
const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
|
|
3446
3613
|
(SELECT MIN(queue_partition_key) AS pk
|
|
3447
3614
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -3457,38 +3624,17 @@ class SystemDatabase {
|
|
|
3457
3624
|
SELECT partitions.pk
|
|
3458
3625
|
FROM partitions
|
|
3459
3626
|
WHERE partitions.pk IS NOT NULL
|
|
3460
|
-
|
|
3461
|
-
AND NOT EXISTS (
|
|
3462
|
-
SELECT 1
|
|
3463
|
-
FROM "${this.schemaName}".workflow_status
|
|
3464
|
-
WHERE ${probeScope} AND queue_name = $1 AND status = $4
|
|
3465
|
-
AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
|
|
3466
|
-
)
|
|
3627
|
+
AND ${noPending('partitions.pk', 4, probeScope)}
|
|
3467
3628
|
ORDER BY ${sweepOrder}
|
|
3468
3629
|
LIMIT $5
|
|
3469
3630
|
)
|
|
3470
|
-
SELECT
|
|
3631
|
+
SELECT chosen.pk
|
|
3471
3632
|
FROM chosen
|
|
3472
|
-
--
|
|
3473
|
-
JOIN LATERAL (
|
|
3474
|
-
|
|
3475
|
-
FROM "${this.schemaName}".workflow_status
|
|
3476
|
-
WHERE queue_name = $1 AND status = $2
|
|
3477
|
-
AND queue_partition_key = chosen.pk
|
|
3478
|
-
AND ${versionClause(3)}
|
|
3479
|
-
AND ${candidateScope}
|
|
3480
|
-
-- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
|
|
3481
|
-
ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
|
|
3482
|
-
LIMIT 1
|
|
3483
|
-
) head ON TRUE
|
|
3484
|
-
-- Which partitions were chosen is settled above; order the result so the claim, and the dispatch it feeds, are deterministic.
|
|
3633
|
+
-- Only partitions with a head this worker may run; LATERAL plans as a tight nested loop.
|
|
3634
|
+
JOIN LATERAL (${headOf('chosen.pk', 3, candidateScope)}) head ON TRUE
|
|
3635
|
+
-- Which partitions were chosen is settled above; order them so the claim, and the dispatch it feeds, are deterministic.
|
|
3485
3636
|
ORDER BY chosen.pk ASC`, candidateParams);
|
|
3486
|
-
const
|
|
3487
|
-
if (candidateIDs.length === 0) {
|
|
3488
|
-
await client.query('COMMIT');
|
|
3489
|
-
return [];
|
|
3490
|
-
}
|
|
3491
|
-
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
|
|
3637
|
+
const partitionKeys = candidateResult.rows.map((row) => row.pk);
|
|
3492
3638
|
// Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
|
|
3493
3639
|
const claimGuard = (ids, status, name, version, scope) => `${ids}
|
|
3494
3640
|
AND status = $${status}
|
|
@@ -3496,68 +3642,108 @@ class SystemDatabase {
|
|
|
3496
3642
|
AND queue_partition_key IS NOT NULL
|
|
3497
3643
|
AND ${versionClause(version)}
|
|
3498
3644
|
AND ${scope}`;
|
|
3499
|
-
|
|
3500
|
-
const
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3645
|
+
/** Claim the heads of these partitions inside the open transaction, returning the IDs it marked PENDING. */
|
|
3646
|
+
const claimBatch = async (batch) => {
|
|
3647
|
+
let heldPks = batch;
|
|
3648
|
+
if (usePartitionLocks) {
|
|
3649
|
+
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_BEFORE_LOCKS);
|
|
3650
|
+
const lockParams = [];
|
|
3651
|
+
const { rows } = await client.query(this.partitionLockQuery(queue.name, batch, lockParams), lockParams);
|
|
3652
|
+
// Partitions another sweep holds are left to it, or to the next poll.
|
|
3653
|
+
heldPks = rows[0]?.held ?? [];
|
|
3654
|
+
if (heldPks.length === 0)
|
|
3655
|
+
return [];
|
|
3656
|
+
}
|
|
3657
|
+
// Pick the heads again under the locks, so a previous holder's claims and any newly visible head are seen.
|
|
3658
|
+
const headParams = [queue.name, workflow_1.StatusString.ENQUEUED, appVersion, workflow_1.StatusString.PENDING];
|
|
3659
|
+
const headScope = `${this.scoped(undefined, headParams)} AND ${this.appNameFilter('application_name', this.appName, headParams)}`;
|
|
3660
|
+
const headProbeScope = this.scoped(undefined, headParams);
|
|
3661
|
+
headParams.push(heldPks);
|
|
3662
|
+
const headResult = await client.query(`SELECT head.workflow_uuid
|
|
3663
|
+
FROM unnest($${headParams.length}::text[]) AS chosen(pk)
|
|
3664
|
+
JOIN LATERAL (${headOf('chosen.pk', 3, headScope)}) head ON TRUE
|
|
3665
|
+
WHERE ${noPending('chosen.pk', 4, headProbeScope)}
|
|
3666
|
+
ORDER BY chosen.pk ASC`, headParams);
|
|
3667
|
+
const candidateIDs = headResult.rows.map((row) => row.workflow_uuid);
|
|
3668
|
+
if (candidateIDs.length === 0)
|
|
3669
|
+
return [];
|
|
3670
|
+
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
|
|
3671
|
+
const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
|
|
3672
|
+
const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
|
|
3673
|
+
const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
|
|
3674
|
+
// Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
|
|
3675
|
+
const lockedResult = await client.query(`SELECT workflow_uuid
|
|
3676
|
+
FROM "${this.schemaName}".workflow_status
|
|
3677
|
+
WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
|
|
3678
|
+
FOR UPDATE SKIP LOCKED`, lockedParams);
|
|
3679
|
+
const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
|
|
3680
|
+
// Preserve partition order for submission.
|
|
3681
|
+
const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
|
|
3682
|
+
if (claimIDs.length === 0)
|
|
3683
|
+
return [];
|
|
3684
|
+
const flipParams = [
|
|
3685
|
+
workflow_1.StatusString.PENDING,
|
|
3686
|
+
executorID,
|
|
3687
|
+
workflow_1.StatusString.ENQUEUED,
|
|
3688
|
+
queue.name,
|
|
3689
|
+
appVersion,
|
|
3690
|
+
// Claim the row, as the unpartitioned dequeue does.
|
|
3691
|
+
this.appName ?? null,
|
|
3692
|
+
ownerXid,
|
|
3693
|
+
startTimeMs,
|
|
3694
|
+
];
|
|
3695
|
+
const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
|
|
3696
|
+
const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
|
|
3697
|
+
// Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
|
|
3698
|
+
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
3699
|
+
SET status = $1,
|
|
3700
|
+
owner_xid = $7,
|
|
3701
|
+
executor_id = $2,
|
|
3702
|
+
application_version = $5,
|
|
3703
|
+
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
3704
|
+
rate_limited = FALSE,
|
|
3705
|
+
application_name = COALESCE(application_name, $6),
|
|
3706
|
+
recovery_attempts = recovery_attempts + 1,
|
|
3707
|
+
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
3708
|
+
workflow_deadline_epoch_ms = CASE
|
|
3709
|
+
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
3710
|
+
CASE
|
|
3711
|
+
WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
|
|
3712
|
+
THEN 9223372036854775807
|
|
3713
|
+
ELSE $8::bigint + workflow_timeout_ms
|
|
3714
|
+
END
|
|
3715
|
+
ELSE workflow_deadline_epoch_ms
|
|
3716
|
+
END
|
|
3717
|
+
WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
|
|
3718
|
+
RETURNING workflow_uuid`, flipParams);
|
|
3719
|
+
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
3720
|
+
return claimIDs.filter((id) => flippedIDs.has(id));
|
|
3721
|
+
};
|
|
3722
|
+
// One short transaction per batch, so a sweep holds at most a batch's locks, and only while it claims that batch.
|
|
3723
|
+
const claimedIDs = [];
|
|
3724
|
+
for (let start = 0; start < partitionKeys.length; start += this.partitionedDequeueBatchSize) {
|
|
3725
|
+
const batch = partitionKeys.slice(start, start + this.partitionedDequeueBatchSize);
|
|
3726
|
+
try {
|
|
3727
|
+
await client.query('BEGIN');
|
|
3728
|
+
const batchIDs = await claimBatch(batch);
|
|
3729
|
+
await client.query('COMMIT');
|
|
3730
|
+
claimedIDs.push(...batchIDs);
|
|
3731
|
+
}
|
|
3732
|
+
catch (error) {
|
|
3733
|
+
// Swallowed: a rollback failure must not mask why we are rolling back.
|
|
3734
|
+
await client.query('ROLLBACK').catch(() => undefined);
|
|
3735
|
+
// Earlier batches committed their claims, which must still reach dispatch; the next poll retries the rest.
|
|
3736
|
+
if (claimedIDs.length === 0)
|
|
3737
|
+
throw error;
|
|
3738
|
+
this.logger.warn(`[${queue.name}] stopping a partitioned sweep after ${claimedIDs.length} claim(s): ${String(error)}`);
|
|
3739
|
+
break;
|
|
3740
|
+
}
|
|
3513
3741
|
}
|
|
3514
|
-
const flipParams = [
|
|
3515
|
-
workflow_1.StatusString.PENDING,
|
|
3516
|
-
executorID,
|
|
3517
|
-
workflow_1.StatusString.ENQUEUED,
|
|
3518
|
-
queue.name,
|
|
3519
|
-
appVersion,
|
|
3520
|
-
// Claim the row, as the unpartitioned dequeue does.
|
|
3521
|
-
this.appName ?? null,
|
|
3522
|
-
ownerXid,
|
|
3523
|
-
startTimeMs,
|
|
3524
|
-
];
|
|
3525
|
-
const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
|
|
3526
|
-
const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
|
|
3527
|
-
// Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
|
|
3528
|
-
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
3529
|
-
SET status = $1,
|
|
3530
|
-
owner_xid = $7,
|
|
3531
|
-
executor_id = $2,
|
|
3532
|
-
application_version = $5,
|
|
3533
|
-
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
3534
|
-
rate_limited = FALSE,
|
|
3535
|
-
application_name = COALESCE(application_name, $6),
|
|
3536
|
-
recovery_attempts = recovery_attempts + 1,
|
|
3537
|
-
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
3538
|
-
workflow_deadline_epoch_ms = CASE
|
|
3539
|
-
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
3540
|
-
CASE
|
|
3541
|
-
WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
|
|
3542
|
-
THEN 9223372036854775807
|
|
3543
|
-
ELSE $8::bigint + workflow_timeout_ms
|
|
3544
|
-
END
|
|
3545
|
-
ELSE workflow_deadline_epoch_ms
|
|
3546
|
-
END
|
|
3547
|
-
WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
|
|
3548
|
-
RETURNING workflow_uuid`, flipParams);
|
|
3549
|
-
await client.query('COMMIT');
|
|
3550
|
-
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
3551
|
-
const claimedIDs = claimIDs.filter((id) => flippedIDs.has(id));
|
|
3552
3742
|
if (claimedIDs.length > 0) {
|
|
3553
3743
|
this.logger.debug(`[${queue.name}] dequeueing ${claimedIDs.length} task(s)`);
|
|
3554
3744
|
}
|
|
3555
3745
|
return claimedIDs;
|
|
3556
3746
|
}
|
|
3557
|
-
catch (error) {
|
|
3558
|
-
await client.query('ROLLBACK');
|
|
3559
|
-
throw error;
|
|
3560
|
-
}
|
|
3561
3747
|
finally {
|
|
3562
3748
|
client.release();
|
|
3563
3749
|
}
|
|
@@ -3655,9 +3841,9 @@ class SystemDatabase {
|
|
|
3655
3841
|
// If queuesOnly, filter for queued workflows
|
|
3656
3842
|
if (input.queuesOnly) {
|
|
3657
3843
|
whereClauses.push(`queue_name IS NOT NULL`);
|
|
3658
|
-
whereClauses.push(`status IN ($${paramCounter}, $${paramCounter + 1}, $${paramCounter + 2})`);
|
|
3659
|
-
params.push(workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING, workflow_1.StatusString.DELAYED);
|
|
3660
|
-
paramCounter +=
|
|
3844
|
+
whereClauses.push(`status IN ($${paramCounter}, $${paramCounter + 1}, $${paramCounter + 2}, $${paramCounter + 3})`);
|
|
3845
|
+
params.push(workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING, workflow_1.StatusString.DELAYED, workflow_1.StatusString.SUSPENDED);
|
|
3846
|
+
paramCounter += 4;
|
|
3661
3847
|
}
|
|
3662
3848
|
addFilter('name', input.workflowName);
|
|
3663
3849
|
addFilter('queue_name', input.queueName);
|
|
@@ -3930,7 +4116,9 @@ class SystemDatabase {
|
|
|
3930
4116
|
const client = await this.connect();
|
|
3931
4117
|
try {
|
|
3932
4118
|
await client.query('BEGIN');
|
|
3933
|
-
|
|
4119
|
+
// Sorted, so concurrent writers lock schedule rows in one order.
|
|
4120
|
+
const sorted = [...schedules].sort((a, b) => compareCodeUnits(a.scheduleName, b.scheduleName));
|
|
4121
|
+
for (const sched of sorted) {
|
|
3934
4122
|
const owner = await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
3935
4123
|
const scheduleKeys = this.scheduleValues(sched);
|
|
3936
4124
|
const values = {
|
|
@@ -4273,7 +4461,12 @@ class SystemDatabase {
|
|
|
4273
4461
|
queues = await move('queues');
|
|
4274
4462
|
schedules = await move('workflow_schedules');
|
|
4275
4463
|
versions = await move('application_versions');
|
|
4276
|
-
inFlight = await move('workflow_status', [
|
|
4464
|
+
inFlight = await move('workflow_status', [
|
|
4465
|
+
workflow_1.StatusString.PENDING,
|
|
4466
|
+
workflow_1.StatusString.ENQUEUED,
|
|
4467
|
+
workflow_1.StatusString.DELAYED,
|
|
4468
|
+
workflow_1.StatusString.SUSPENDED,
|
|
4469
|
+
]);
|
|
4277
4470
|
await client.query('COMMIT');
|
|
4278
4471
|
}
|
|
4279
4472
|
catch (e) {
|
|
@@ -4724,6 +4917,12 @@ __decorate([
|
|
|
4724
4917
|
__metadata("design:paramtypes", [Array, Number]),
|
|
4725
4918
|
__metadata("design:returntype", Promise)
|
|
4726
4919
|
], SystemDatabase.prototype, "deadLetterWorkflows", null);
|
|
4920
|
+
__decorate([
|
|
4921
|
+
dbRetry(),
|
|
4922
|
+
__metadata("design:type", Function),
|
|
4923
|
+
__metadata("design:paramtypes", [Array, Map]),
|
|
4924
|
+
__metadata("design:returntype", Promise)
|
|
4925
|
+
], SystemDatabase.prototype, "enqueueScheduledWorkflows", null);
|
|
4727
4926
|
__decorate([
|
|
4728
4927
|
dbRetry(),
|
|
4729
4928
|
__metadata("design:type", Function),
|
|
@@ -4868,6 +5067,12 @@ __decorate([
|
|
|
4868
5067
|
__metadata("design:paramtypes", [String, Number, Number, String, Number, Number]),
|
|
4869
5068
|
__metadata("design:returntype", Promise)
|
|
4870
5069
|
], SystemDatabase.prototype, "recv", null);
|
|
5070
|
+
__decorate([
|
|
5071
|
+
dbRetry(),
|
|
5072
|
+
__metadata("design:type", Function),
|
|
5073
|
+
__metadata("design:paramtypes", [String, Number, Object, Number]),
|
|
5074
|
+
__metadata("design:returntype", Promise)
|
|
5075
|
+
], SystemDatabase.prototype, "recvConsume", null);
|
|
4871
5076
|
__decorate([
|
|
4872
5077
|
dbRetry(),
|
|
4873
5078
|
__metadata("design:type", Function),
|
|
@@ -4934,4 +5139,10 @@ __decorate([
|
|
|
4934
5139
|
__metadata("design:paramtypes", [String]),
|
|
4935
5140
|
__metadata("design:returntype", Promise)
|
|
4936
5141
|
], SystemDatabase.prototype, "getWorkflowOwner", null);
|
|
5142
|
+
__decorate([
|
|
5143
|
+
dbRetry(),
|
|
5144
|
+
__metadata("design:type", Function),
|
|
5145
|
+
__metadata("design:paramtypes", [String, Number, Number, Boolean]),
|
|
5146
|
+
__metadata("design:returntype", Promise)
|
|
5147
|
+
], SystemDatabase.prototype, "durableSleepInternal", null);
|
|
4937
5148
|
//# sourceMappingURL=system_database.js.map
|