@dbos-inc/dbos-sdk 5.4.10-preview → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/schemas/system_db_schema.d.ts +7 -0
- package/dist/schemas/system_db_schema.d.ts.map +1 -1
- package/dist/schemas/system_db_schema.js.map +1 -1
- package/dist/src/cli/cli.js +56 -13
- package/dist/src/cli/cli.js.map +1 -1
- package/dist/src/client.d.ts +44 -10
- package/dist/src/client.d.ts.map +1 -1
- package/dist/src/client.js +140 -44
- package/dist/src/client.js.map +1 -1
- package/dist/src/config.d.ts +1 -0
- package/dist/src/config.d.ts.map +1 -1
- package/dist/src/config.js +4 -0
- package/dist/src/config.js.map +1 -1
- package/dist/src/context.d.ts +6 -0
- package/dist/src/context.d.ts.map +1 -1
- package/dist/src/context.js +15 -1
- package/dist/src/context.js.map +1 -1
- package/dist/src/datasource.d.ts +2 -0
- package/dist/src/datasource.d.ts.map +1 -1
- package/dist/src/datasource.js +7 -1
- package/dist/src/datasource.js.map +1 -1
- package/dist/src/dbos-executor.d.ts +32 -4
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js +102 -33
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts +104 -15
- package/dist/src/dbos.d.ts.map +1 -1
- package/dist/src/dbos.js +275 -86
- package/dist/src/dbos.js.map +1 -1
- package/dist/src/debugpoint.d.ts +1 -0
- package/dist/src/debugpoint.d.ts.map +1 -1
- package/dist/src/debugpoint.js +4 -2
- package/dist/src/debugpoint.js.map +1 -1
- package/dist/src/enqueue_options.d.ts +2 -0
- package/dist/src/enqueue_options.d.ts.map +1 -1
- package/dist/src/enqueue_options.js +3 -0
- package/dist/src/enqueue_options.js.map +1 -1
- package/dist/src/enqueue_workflow.d.ts.map +1 -1
- package/dist/src/enqueue_workflow.js +11 -5
- package/dist/src/enqueue_workflow.js.map +1 -1
- package/dist/src/enterprise.d.ts +78 -3
- package/dist/src/enterprise.d.ts.map +1 -1
- package/dist/src/enterprise.js +104 -45
- package/dist/src/enterprise.js.map +1 -1
- package/dist/src/error.d.ts +42 -0
- package/dist/src/error.d.ts.map +1 -1
- package/dist/src/error.js +85 -1
- package/dist/src/error.js.map +1 -1
- package/dist/src/eventreceiver.d.ts +2 -0
- package/dist/src/eventreceiver.d.ts.map +1 -1
- package/dist/src/eventreceiver.js +7 -1
- package/dist/src/eventreceiver.js.map +1 -1
- package/dist/src/external_storage.d.ts +27 -0
- package/dist/src/external_storage.d.ts.map +1 -0
- package/dist/src/external_storage.js +28 -0
- package/dist/src/external_storage.js.map +1 -0
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/scheduler/scheduler.d.ts +83 -3
- package/dist/src/scheduler/scheduler.d.ts.map +1 -1
- package/dist/src/scheduler/scheduler.js +422 -171
- package/dist/src/scheduler/scheduler.js.map +1 -1
- package/dist/src/shard_keys.d.ts +13 -0
- package/dist/src/shard_keys.d.ts.map +1 -0
- package/dist/src/shard_keys.js +55 -0
- package/dist/src/shard_keys.js.map +1 -0
- package/dist/src/streams.d.ts +2 -0
- package/dist/src/streams.d.ts.map +1 -1
- package/dist/src/streams.js +5 -2
- package/dist/src/streams.js.map +1 -1
- package/dist/src/sysdb_migrations/internal/migrations.js +82 -0
- package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
- package/dist/src/system_database.d.ts +185 -10
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +1429 -626
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/utils.d.ts +2 -0
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +9 -1
- package/dist/src/utils.js.map +1 -1
- package/dist/src/wfqueue.d.ts +1 -2
- package/dist/src/wfqueue.d.ts.map +1 -1
- package/dist/src/wfqueue.js +83 -63
- package/dist/src/wfqueue.js.map +1 -1
- package/dist/src/workflow.d.ts +15 -3
- package/dist/src/workflow.d.ts.map +1 -1
- package/dist/src/workflow.js +30 -12
- package/dist/src/workflow.js.map +1 -1
- package/dist/src/workflow_management.d.ts +17 -3
- package/dist/src/workflow_management.d.ts.map +1 -1
- package/dist/src/workflow_management.js +93 -20
- package/dist/src/workflow_management.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
|
@@ -1,15 +1,38 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
2
18
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
19
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
20
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
21
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
22
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
23
|
};
|
|
24
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
25
|
+
if (mod && mod.__esModule) return mod;
|
|
26
|
+
var result = {};
|
|
27
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
28
|
+
__setModuleDefault(result, mod);
|
|
29
|
+
return result;
|
|
30
|
+
};
|
|
8
31
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
32
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
33
|
};
|
|
11
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.SystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
35
|
+
exports.SystemDatabase = exports.createSystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.qualify = exports.holdsShardedSchema = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.compareCodeUnits = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
13
36
|
const dbos_executor_1 = require("./dbos-executor");
|
|
14
37
|
const pg_1 = require("pg");
|
|
15
38
|
const error_1 = require("./error");
|
|
@@ -18,6 +41,8 @@ const utils_1 = require("./utils");
|
|
|
18
41
|
const context_1 = require("./context");
|
|
19
42
|
const async_hooks_1 = require("async_hooks");
|
|
20
43
|
const crypto_1 = require("crypto");
|
|
44
|
+
const shard_keys_1 = require("./shard_keys");
|
|
45
|
+
const enterprise = __importStar(require("./enterprise"));
|
|
21
46
|
const utils_2 = require("./utils");
|
|
22
47
|
const database_utils_1 = require("./database_utils");
|
|
23
48
|
const migration_runner_1 = require("./sysdb_migrations/migration_runner");
|
|
@@ -70,6 +95,13 @@ function validateObservabilityQueryTimeoutMs(value) {
|
|
|
70
95
|
exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutMs;
|
|
71
96
|
// Bounds how long a frozen or unreachable client can hold system database locks.
|
|
72
97
|
exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = 60_000;
|
|
98
|
+
// How long a background job's lock may sit idle before Postgres ends its holder's session.
|
|
99
|
+
exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = 5000;
|
|
100
|
+
/** Orders strings by code unit, which matches Postgres's "C" collation outside surrogate pairs. */
|
|
101
|
+
function compareCodeUnits(a, b) {
|
|
102
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
103
|
+
}
|
|
104
|
+
exports.compareCodeUnits = compareCodeUnits;
|
|
73
105
|
/** The timeout is rendered into a `SET idle_in_transaction_session_timeout`, which takes only an in-range integer. */
|
|
74
106
|
function validateIdleTransactionTimeoutMs(value) {
|
|
75
107
|
if (value !== undefined && (!Number.isFinite(value) || value > MAX_STATEMENT_TIMEOUT_MS)) {
|
|
@@ -157,6 +189,7 @@ async function grantDbosSchemaPermissions(databaseUrl, roleName, logger, schemaN
|
|
|
157
189
|
await client.end();
|
|
158
190
|
}
|
|
159
191
|
}
|
|
192
|
+
exports.grantDbosSchemaPermissions = grantDbosSchemaPermissions;
|
|
160
193
|
/**
|
|
161
194
|
* Check out a connection carrying our own error handler, so a socket death while we hold it is
|
|
162
195
|
* logged instead of thrown at an emitter with no listener. `release()` takes the handler back off,
|
|
@@ -219,6 +252,7 @@ async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = '
|
|
|
219
252
|
}
|
|
220
253
|
const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
|
|
221
254
|
try {
|
|
255
|
+
await refuseShardedSchema(client, schemaName);
|
|
222
256
|
const isCockroach = await isCockroachDB(client);
|
|
223
257
|
await (0, migration_runner_1.runSysMigrationsPg)(client, (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }), schemaName, {
|
|
224
258
|
onWarn: (e) => logger.info(e),
|
|
@@ -242,6 +276,7 @@ exports.migrateSystemDatabase = migrateSystemDatabase;
|
|
|
242
276
|
async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
|
|
243
277
|
const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
|
|
244
278
|
try {
|
|
279
|
+
await refuseShardedSchema(client, schemaName);
|
|
245
280
|
const isCockroach = await isCockroachDB(client);
|
|
246
281
|
const requiredVersion = (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }).length;
|
|
247
282
|
const currentVersion = await (0, migration_runner_1.getCurrentSysDBVersion)(client, schemaName);
|
|
@@ -259,6 +294,41 @@ async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = '
|
|
|
259
294
|
}
|
|
260
295
|
}
|
|
261
296
|
exports.verifySystemDatabase = verifySystemDatabase;
|
|
297
|
+
/** Throw if this schema is sharded: an unsharded handle's statements would span every tenant's rows. */
|
|
298
|
+
async function refuseShardedSchema(client, schemaName) {
|
|
299
|
+
const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
|
|
300
|
+
WHERE table_schema = $1 AND table_name = 'workflow_status' AND column_name = 'shard_key'`, [schemaName]);
|
|
301
|
+
if (rows.length > 0) {
|
|
302
|
+
throw new error_1.DBOSError(`The DBOS system schema ${schemaName} is sharded. Enable sharding to use it: ` +
|
|
303
|
+
'set sharding in your DBOS configuration, or pass sharding: true to DBOSClient.');
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
/** Whether this database holds a sharded DBOS system schema (this one, if named); false if it can't be reached. */
|
|
307
|
+
async function holdsShardedSchema(databaseUrl, schemaName) {
|
|
308
|
+
let client;
|
|
309
|
+
try {
|
|
310
|
+
// Bounded, so an unreachable host fails as fast as the caller's own connection would.
|
|
311
|
+
client = new pg_1.Client({ ...(0, utils_2.getClientConfig)(databaseUrl), connectionTimeoutMillis: 10_000 });
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
// A malformed URL, for the caller's own connection to report.
|
|
315
|
+
return false;
|
|
316
|
+
}
|
|
317
|
+
client.on('error', () => { });
|
|
318
|
+
try {
|
|
319
|
+
await client.connect();
|
|
320
|
+
const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
|
|
321
|
+
WHERE table_name = 'workflow_status' AND column_name = 'shard_key'${schemaName !== undefined ? ' AND table_schema = $1' : ''} LIMIT 1`, schemaName !== undefined ? [schemaName] : []);
|
|
322
|
+
return rows.length > 0;
|
|
323
|
+
}
|
|
324
|
+
catch {
|
|
325
|
+
return false;
|
|
326
|
+
}
|
|
327
|
+
finally {
|
|
328
|
+
await client.end().catch(() => { });
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
exports.holdsShardedSchema = holdsShardedSchema;
|
|
262
332
|
class NotificationMap {
|
|
263
333
|
map = new Map();
|
|
264
334
|
curCK = 0;
|
|
@@ -300,6 +370,23 @@ function mapVersionInfo(row) {
|
|
|
300
370
|
};
|
|
301
371
|
}
|
|
302
372
|
const SCHEDULE_COLUMNS = 'schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name';
|
|
373
|
+
/** A schedule's own columns, as a new row stores them. */
|
|
374
|
+
function scheduleRow(schedule, owner) {
|
|
375
|
+
return {
|
|
376
|
+
schedule_id: schedule.scheduleId,
|
|
377
|
+
schedule_name: schedule.scheduleName,
|
|
378
|
+
workflow_name: schedule.workflowName,
|
|
379
|
+
workflow_class_name: schedule.workflowClassName,
|
|
380
|
+
schedule: schedule.schedule,
|
|
381
|
+
status: schedule.status,
|
|
382
|
+
context: schedule.context,
|
|
383
|
+
last_fired_at: schedule.lastFiredAt,
|
|
384
|
+
automatic_backfill: schedule.automaticBackfill,
|
|
385
|
+
cron_timezone: schedule.cronTimezone,
|
|
386
|
+
queue_name: schedule.queueName,
|
|
387
|
+
application_name: owner ?? null,
|
|
388
|
+
};
|
|
389
|
+
}
|
|
303
390
|
function mapWorkflowSchedule(row) {
|
|
304
391
|
return {
|
|
305
392
|
scheduleId: row.schedule_id,
|
|
@@ -314,6 +401,7 @@ function mapWorkflowSchedule(row) {
|
|
|
314
401
|
cronTimezone: row.cron_timezone ?? null,
|
|
315
402
|
queueName: row.queue_name ?? null,
|
|
316
403
|
applicationName: row.application_name ?? undefined,
|
|
404
|
+
shardKey: row.workflow_shard_key ?? undefined,
|
|
317
405
|
};
|
|
318
406
|
}
|
|
319
407
|
function mapWorkflowStatus(row) {
|
|
@@ -356,8 +444,16 @@ function mapWorkflowStatus(row) {
|
|
|
356
444
|
debounceDeadlineEpochMS: row.debounce_deadline_epoch_ms ? Number(row.debounce_deadline_epoch_ms) : undefined,
|
|
357
445
|
isDebounced: row.is_debounced ?? false,
|
|
358
446
|
applicationName: row.application_name ?? undefined,
|
|
447
|
+
// Read back only from a sharded system database, whose rows carry them.
|
|
448
|
+
shardKey: row.shard_key ?? undefined,
|
|
449
|
+
parentShardKey: row.parent_shard_key ?? undefined,
|
|
359
450
|
};
|
|
360
451
|
}
|
|
452
|
+
/** `column`, qualified by `alias` if one is given. */
|
|
453
|
+
function qualify(alias, column) {
|
|
454
|
+
return alias ? `${alias}.${column}` : column;
|
|
455
|
+
}
|
|
456
|
+
exports.qualify = qualify;
|
|
361
457
|
// SQLSTATE classes/codes that are generally safe to retry
|
|
362
458
|
// https://www.postgresql.org/docs/current/errcodes-appendix.html
|
|
363
459
|
const RETRY_SQLSTATE_PREFIXES = new Set([
|
|
@@ -527,6 +623,34 @@ function dbRetry(options = {}) {
|
|
|
527
623
|
return descriptor;
|
|
528
624
|
};
|
|
529
625
|
}
|
|
626
|
+
/** Build the system database these options describe. */
|
|
627
|
+
function createSystemDatabase(options) {
|
|
628
|
+
const sysdb = createBaseSystemDatabase(options);
|
|
629
|
+
if (options.suspendMode) {
|
|
630
|
+
const { Suspend } = enterprise.load(enterprise.SUSPEND.description);
|
|
631
|
+
sysdb.suspend = new Suspend(sysdb);
|
|
632
|
+
}
|
|
633
|
+
return sysdb;
|
|
634
|
+
}
|
|
635
|
+
exports.createSystemDatabase = createSystemDatabase;
|
|
636
|
+
function createBaseSystemDatabase(options) {
|
|
637
|
+
if (options.sharding) {
|
|
638
|
+
const { ShardedSystemDatabase } = enterprise.load(enterprise.SHARDING.description);
|
|
639
|
+
return new ShardedSystemDatabase({
|
|
640
|
+
systemDatabaseUrl: options.systemDatabaseUrl,
|
|
641
|
+
logger: options.logger,
|
|
642
|
+
serializer: options.serializer,
|
|
643
|
+
sysDbPoolSize: options.sysDbPoolSize,
|
|
644
|
+
systemDatabasePool: options.systemDatabasePool,
|
|
645
|
+
schemaName: options.schemaName,
|
|
646
|
+
pollingConcurrency: options.pollingConcurrency,
|
|
647
|
+
appName: options.appName,
|
|
648
|
+
observabilityQueryTimeoutMs: options.observabilityQueryTimeoutMs,
|
|
649
|
+
idleTransactionTimeoutMs: options.idleTransactionTimeoutMs,
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
return new SystemDatabase(options.systemDatabaseUrl, options.logger, options.serializer, options.sysDbPoolSize, options.systemDatabasePool, options.schemaName, options.useListenNotify, options.pollingConcurrency, options.notificationCoalesceMs, options.appName, options.observabilityQueryTimeoutMs, options.idleTransactionTimeoutMs);
|
|
653
|
+
}
|
|
530
654
|
/**
|
|
531
655
|
* General notes:
|
|
532
656
|
* The responsibilities of the `SystemDatabase` are to store data for workflows, and
|
|
@@ -570,8 +694,11 @@ class SystemDatabase {
|
|
|
570
694
|
dbPollingIntervalStreamMs = 1000;
|
|
571
695
|
dbPollingIntervalCancelMs = 1000;
|
|
572
696
|
shouldUseDBNotifications = true;
|
|
697
|
+
// How long a background job's lock may sit idle before Postgres ends its holder's session.
|
|
698
|
+
backgroundJobLockIdleTimeoutMs = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS;
|
|
573
699
|
notificationsMap = new NotificationMap();
|
|
574
700
|
workflowEventsMap = new NotificationMap();
|
|
701
|
+
suspend = undefined;
|
|
575
702
|
streamsMap = new NotificationMap();
|
|
576
703
|
customPool = false;
|
|
577
704
|
// Interval for coalescing LISTEN/NOTIFY notifications pushed off the write path (Postgres + L/N only).
|
|
@@ -595,6 +722,7 @@ class SystemDatabase {
|
|
|
595
722
|
pollLimiter;
|
|
596
723
|
/** Executions running in this process, by workflow ID; a re-dispatch runs alongside its stale execution, which parks at its next write. */
|
|
597
724
|
runningWorkflowMap = new Map();
|
|
725
|
+
// Keyed by waiter key, as one ID can live under several shard keys.
|
|
598
726
|
#cancelWatchers = new Map();
|
|
599
727
|
#cancelPollerLoop = undefined;
|
|
600
728
|
#cancelPollerWake = null;
|
|
@@ -727,6 +855,214 @@ class SystemDatabase {
|
|
|
727
855
|
observabilityFilter(column, value, params) {
|
|
728
856
|
return this.appNameFilter(column, value ?? this.appName, params);
|
|
729
857
|
}
|
|
858
|
+
// ==================== Shard Keys ====================
|
|
859
|
+
// Statements locate rows through these, so a sharded subclass adds its key; each appends to `params` and returns SQL.
|
|
860
|
+
/** Whether this handle routes every statement by shard key. */
|
|
861
|
+
get sharded() {
|
|
862
|
+
return false;
|
|
863
|
+
}
|
|
864
|
+
#rejectShardKey(workflowID, shardKey) {
|
|
865
|
+
if ((shardKey !== undefined && shardKey !== null) || (0, shard_keys_1.boundShardKey)(workflowID) !== undefined) {
|
|
866
|
+
throw new error_1.DBOSError('A shard key was given, but this system database is not sharded');
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
/** Predicate selecting one workflow's rows: those whose `column` is `workflowID`. */
|
|
870
|
+
wf(alias, workflowID, params, column = 'workflow_uuid', shardKey) {
|
|
871
|
+
this.#rejectShardKey(workflowID, shardKey);
|
|
872
|
+
params.push(workflowID);
|
|
873
|
+
return `${qualify(alias, column)} = $${params.length}`;
|
|
874
|
+
}
|
|
875
|
+
/** Predicate selecting several workflows' rows. */
|
|
876
|
+
wfIn(alias, workflowIDs, params) {
|
|
877
|
+
for (const workflowID of workflowIDs) {
|
|
878
|
+
this.#rejectShardKey(workflowID, undefined);
|
|
879
|
+
}
|
|
880
|
+
params.push(workflowIDs);
|
|
881
|
+
return `${qualify(alias, 'workflow_uuid')} = ANY($${params.length}::text[])`;
|
|
882
|
+
}
|
|
883
|
+
/** Condition joining two tables' rows of the same workflow. */
|
|
884
|
+
wfJoin(left, right) {
|
|
885
|
+
return `${left}.workflow_uuid = ${right}.workflow_uuid`;
|
|
886
|
+
}
|
|
887
|
+
/** Extra column values for a new row of this workflow. */
|
|
888
|
+
wfValues(workflowID, shardKey) {
|
|
889
|
+
this.#rejectShardKey(workflowID, shardKey);
|
|
890
|
+
return {};
|
|
891
|
+
}
|
|
892
|
+
/** Extra column values storing the shard key of a new workflow's parent. */
|
|
893
|
+
parentValues(_parentWorkflowID) {
|
|
894
|
+
return {};
|
|
895
|
+
}
|
|
896
|
+
/** Extra column values storing the shard key of the workflow a step refers to. */
|
|
897
|
+
childValues(_childWorkflowID) {
|
|
898
|
+
return {};
|
|
899
|
+
}
|
|
900
|
+
/** Extra (column, expression) pairs for an INSERT ... SELECT of this workflow's rows. */
|
|
901
|
+
wfColumns(_workflowID, _params) {
|
|
902
|
+
return [];
|
|
903
|
+
}
|
|
904
|
+
/** Conflict target for a unique key of a workflow table. */
|
|
905
|
+
keyConflict(...columns) {
|
|
906
|
+
return columns.join(', ');
|
|
907
|
+
}
|
|
908
|
+
/** Insert rows into `table` in one statement, returning the RETURNING rows; a sharded subclass inserts row by row. */
|
|
909
|
+
async insertRows(client, table, columns, rows, suffix = '') {
|
|
910
|
+
if (rows.length === 0)
|
|
911
|
+
return [];
|
|
912
|
+
const params = [];
|
|
913
|
+
const tuples = rows.map((row) => `(${row.map((value) => `$${params.push(value)}`).join(', ')})`);
|
|
914
|
+
const { rows: returned } = await client.query(`INSERT INTO "${this.schemaName}".${table} (${columns.join(', ')}) VALUES ${tuples.join(', ')} ${suffix}`, params);
|
|
915
|
+
return returned;
|
|
916
|
+
}
|
|
917
|
+
/** Predicate a queue claim must also meet, evaluated once before it scans; sharded, it caps claimers per key. */
|
|
918
|
+
claimGate(_queueName, _partitionKey, _params) {
|
|
919
|
+
return 'TRUE';
|
|
920
|
+
}
|
|
921
|
+
/** Statement whose `acquired` says whether a claim took its budget scope's transaction lock; no row means nothing to claim. */
|
|
922
|
+
claimLockQuery(queueName, partitionKey, params) {
|
|
923
|
+
const name = JSON.stringify(['dbos.queue_claim', this.schemaName, queueName, partitionKey ?? null]);
|
|
924
|
+
params.push(advisoryLockKey(name).toString());
|
|
925
|
+
return `SELECT pg_try_advisory_xact_lock($${params.length}) AS acquired`;
|
|
926
|
+
}
|
|
927
|
+
/** Statement whose `held` lists which of `partitionKeys` a sweep's transaction locked; no row means nothing to claim. */
|
|
928
|
+
partitionLockQuery(queueName, partitionKeys, params) {
|
|
929
|
+
const name = JSON.stringify(['dbos.partition_claim', this.schemaName, queueName]);
|
|
930
|
+
// Two-int keys: the queue, then the partition, hashed so every worker computes the same lock.
|
|
931
|
+
const lockIDs = partitionKeys.map((pk) => (0, crypto_1.createHash)('sha256').update(pk).digest().readInt32BE(0));
|
|
932
|
+
params.push(partitionKeys, lockIDs, Number(BigInt.asIntN(32, advisoryLockKey(name))));
|
|
933
|
+
const n = params.length;
|
|
934
|
+
return `SELECT coalesce(array_agg(p.pk), '{}') AS held
|
|
935
|
+
FROM unnest($${n - 2}::text[], $${n - 1}::int4[]) AS p(pk, lock_id)
|
|
936
|
+
WHERE pg_try_advisory_xact_lock($${n}::int4, p.lock_id)`;
|
|
937
|
+
}
|
|
938
|
+
/** Predicate confining a statement that names no single workflow to the current shard. */
|
|
939
|
+
scoped(_alias, _params) {
|
|
940
|
+
return 'TRUE';
|
|
941
|
+
}
|
|
942
|
+
/** Extra (column, expression) pairs for an INSERT ... SELECT confined to the current shard. */
|
|
943
|
+
scopedColumns(_params) {
|
|
944
|
+
return [];
|
|
945
|
+
}
|
|
946
|
+
/** Predicate selecting a global table's rows (queues, schedules, versions). */
|
|
947
|
+
global(_alias, _params) {
|
|
948
|
+
return 'TRUE';
|
|
949
|
+
}
|
|
950
|
+
/** Extra column values for a new row of a global table. */
|
|
951
|
+
globalValues() {
|
|
952
|
+
return {};
|
|
953
|
+
}
|
|
954
|
+
/** Predicate selecting rows whose `column` holds one of these shard keys. */
|
|
955
|
+
keyIn(_alias, _shardKeys, _params, _column = 'shard_key') {
|
|
956
|
+
throw new error_1.DBOSError('Filtering by shard key requires a sharded system database');
|
|
957
|
+
}
|
|
958
|
+
/** A shard key column to read back with a row, if this handle has one. */
|
|
959
|
+
keyColumn(_alias, _column = 'shard_key') {
|
|
960
|
+
return undefined;
|
|
961
|
+
}
|
|
962
|
+
/** Extra column values storing the shard key a schedule's workflows run under. */
|
|
963
|
+
scheduleValues(schedule) {
|
|
964
|
+
if (schedule.shardKey !== undefined && schedule.shardKey !== null) {
|
|
965
|
+
throw new error_1.DBOSError('Scheduling under a shard key requires a sharded system database');
|
|
966
|
+
}
|
|
967
|
+
return {};
|
|
968
|
+
}
|
|
969
|
+
/** The column holding the shard key a schedule's workflows run under, if any. */
|
|
970
|
+
scheduleKeyColumn() {
|
|
971
|
+
return undefined;
|
|
972
|
+
}
|
|
973
|
+
/** This workflow's shard key, resolved now for a statement or waiter that needs it later. */
|
|
974
|
+
keyHint(workflowID) {
|
|
975
|
+
this.#rejectShardKey(workflowID, undefined);
|
|
976
|
+
return undefined;
|
|
977
|
+
}
|
|
978
|
+
/** The notification-map key for a workflow's topic or key, distinct per shard key as IDs repeat across keys. */
|
|
979
|
+
waiter(workflowID, name) {
|
|
980
|
+
const key = this.keyHint(workflowID);
|
|
981
|
+
const payload = `${workflowID}::${name}`;
|
|
982
|
+
return key === undefined ? payload : `${[...key].length}:${key}:${payload}`;
|
|
983
|
+
}
|
|
984
|
+
/** Bind a workflow to `key`, else to the running workflow's own key even where a same-ID target is bound. */
|
|
985
|
+
ownKey(workflowID, key, callback) {
|
|
986
|
+
if (key === undefined) {
|
|
987
|
+
const ctx = (0, context_1.getCurrentContextStore)();
|
|
988
|
+
if (ctx !== undefined && ctx.workflowId === workflowID) {
|
|
989
|
+
key = ctx.shardKey;
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
return (0, shard_keys_1.bindShardKeys)([[workflowID, key]], callback);
|
|
993
|
+
}
|
|
994
|
+
/** Run `callback` under this application's lock for a background job, passing undefined if refused, else a call renewing it in place. */
|
|
995
|
+
async backgroundJobLock(jobName, callback) {
|
|
996
|
+
// CockroachDB grants every advisory lock, so each process runs every job.
|
|
997
|
+
if (await this.#isCockroach()) {
|
|
998
|
+
return await callback(() => Promise.resolve());
|
|
999
|
+
}
|
|
1000
|
+
// Leading 8 bytes of SHA-256, big-endian signed, as for the retention lock.
|
|
1001
|
+
const key = advisoryLockKey(`dbos.background_job.${this.schemaName}.${this.appName ?? ''}.${jobName}`);
|
|
1002
|
+
const timeout = `'${Math.trunc(this.backgroundJobLockIdleTimeoutMs)}'`;
|
|
1003
|
+
return await this.lockedBackgroundJob(jobName, `SELECT pg_try_advisory_xact_lock(${key}) FROM set_config('idle_in_transaction_session_timeout', ${timeout}, true)`,
|
|
1004
|
+
// Snapshot-free, so a concurrent index build never waits on it; fails if the session ended.
|
|
1005
|
+
async (client) => {
|
|
1006
|
+
await client.query(`SET LOCAL idle_in_transaction_session_timeout = ${timeout}`);
|
|
1007
|
+
}, callback);
|
|
1008
|
+
}
|
|
1009
|
+
/** Take a transaction-scoped lock with literal `lockQuery` (its first column says if taken), as a bound statement's portal keeps its snapshot (an xmin) while the session idles. */
|
|
1010
|
+
async lockedBackgroundJob(jobName, lockQuery, renew, callback) {
|
|
1011
|
+
const client = await this.connect();
|
|
1012
|
+
try {
|
|
1013
|
+
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1014
|
+
// The local idle timeout ends a frozen holder's session, releasing its lock.
|
|
1015
|
+
const { rows } = await client.query({ text: lockQuery, rowMode: 'array' });
|
|
1016
|
+
return await callback(rows[0]?.[0] === true ? () => renew(client) : undefined);
|
|
1017
|
+
}
|
|
1018
|
+
finally {
|
|
1019
|
+
try {
|
|
1020
|
+
await client.query('ROLLBACK');
|
|
1021
|
+
client.release();
|
|
1022
|
+
}
|
|
1023
|
+
catch (e) {
|
|
1024
|
+
// A session the idle timeout ended has released the lock already.
|
|
1025
|
+
this.logger.debug(`Could not release the ${jobName} lock: ${String(e)}`);
|
|
1026
|
+
client.release(e instanceof Error ? e : true);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
#cockroach = undefined;
|
|
1031
|
+
/** Whether this handle's database is CockroachDB, read on first use. */
|
|
1032
|
+
#isCockroach() {
|
|
1033
|
+
this.#cockroach ??= (async () => {
|
|
1034
|
+
const client = await this.connect();
|
|
1035
|
+
try {
|
|
1036
|
+
return await isCockroachDB(client);
|
|
1037
|
+
}
|
|
1038
|
+
finally {
|
|
1039
|
+
client.release();
|
|
1040
|
+
}
|
|
1041
|
+
})();
|
|
1042
|
+
// A failed read is retried on next use, rather than cached.
|
|
1043
|
+
this.#cockroach.catch(() => {
|
|
1044
|
+
this.#cockroach = undefined;
|
|
1045
|
+
});
|
|
1046
|
+
return this.#cockroach;
|
|
1047
|
+
}
|
|
1048
|
+
/** Scopes a queue's workers dequeue from in turn; one unkeyed scope unless sharded. */
|
|
1049
|
+
dequeueScopes(_queueName) {
|
|
1050
|
+
return Promise.resolve([undefined]);
|
|
1051
|
+
}
|
|
1052
|
+
/** Whether a dequeue failed because another worker holds the rows it needs. */
|
|
1053
|
+
isLockConflict(e) {
|
|
1054
|
+
return isPgDatabaseError(e) && e.code === '55P03';
|
|
1055
|
+
}
|
|
1056
|
+
/** Whether a dequeue failed on a serialization conflict with a concurrent one. */
|
|
1057
|
+
isSerializationConflict(e) {
|
|
1058
|
+
return isPgDatabaseError(e) && e.code === '40001';
|
|
1059
|
+
}
|
|
1060
|
+
/** Whether `appVersion` is this application's latest registered version, or none is registered. */
|
|
1061
|
+
async isLatestVersion(appVersion, client) {
|
|
1062
|
+
// Own plus unclaimed: a named peer's deploy must not demote this one.
|
|
1063
|
+
const latest = await this.#latestApplicationVersionName(client ?? this.pool);
|
|
1064
|
+
return latest === undefined || latest === appVersion;
|
|
1065
|
+
}
|
|
730
1066
|
/**
|
|
731
1067
|
* The version name a dequeue treats as latest: this application's own plus unclaimed
|
|
732
1068
|
* ones, so a named peer's deploy does not demote this one.
|
|
@@ -737,15 +1073,17 @@ class SystemDatabase {
|
|
|
737
1073
|
async #latestApplicationVersion(client, applicationName) {
|
|
738
1074
|
const columns = 'version_id, version_name, version_timestamp, created_at, application_name';
|
|
739
1075
|
const table = `"${this.schemaName}".application_versions`;
|
|
1076
|
+
const params = applicationName ? [applicationName] : [];
|
|
1077
|
+
const scope = this.global(undefined, params);
|
|
740
1078
|
// Probe each eligible owner separately; an OR filter can scan an unbounded history of newer peer versions.
|
|
741
1079
|
// Keep the constant owner in the ordering so the IS NULL probe uses the composite index too.
|
|
742
1080
|
const query = applicationName
|
|
743
|
-
? `(SELECT ${columns} FROM ${table} WHERE application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
1081
|
+
? `(SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
744
1082
|
UNION ALL
|
|
745
|
-
(SELECT ${columns} FROM ${table} WHERE application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
1083
|
+
(SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
746
1084
|
ORDER BY version_timestamp DESC LIMIT 1`
|
|
747
|
-
: `SELECT ${columns} FROM ${table} ORDER BY version_timestamp DESC LIMIT 1`;
|
|
748
|
-
const { rows } = await client.query(query,
|
|
1085
|
+
: `SELECT ${columns} FROM ${table} WHERE ${scope} ORDER BY version_timestamp DESC LIMIT 1`;
|
|
1086
|
+
const { rows } = await client.query(query, params);
|
|
749
1087
|
return rows[0];
|
|
750
1088
|
}
|
|
751
1089
|
/**
|
|
@@ -753,7 +1091,8 @@ class SystemDatabase {
|
|
|
753
1091
|
* leaves the owner intact; a named one collides only with a different name.
|
|
754
1092
|
*/
|
|
755
1093
|
async #resolveRowOwner(client, table, keyColumn, name, owner, kind) {
|
|
756
|
-
const
|
|
1094
|
+
const params = [name];
|
|
1095
|
+
const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${this.global(undefined, params)} AND ${keyColumn} = $1`, params);
|
|
757
1096
|
const current = rows[0]?.application_name ?? null;
|
|
758
1097
|
if (current === null)
|
|
759
1098
|
return owner;
|
|
@@ -770,8 +1109,12 @@ class SystemDatabase {
|
|
|
770
1109
|
}
|
|
771
1110
|
/** Migrates the system database, or, when `runMigrations` is false, verifies it is already migrated. */
|
|
772
1111
|
async init(runMigrations = true) {
|
|
773
|
-
|
|
774
|
-
|
|
1112
|
+
if (runMigrations) {
|
|
1113
|
+
await this.runMigrations();
|
|
1114
|
+
}
|
|
1115
|
+
else {
|
|
1116
|
+
await this.verifyMigrations();
|
|
1117
|
+
}
|
|
775
1118
|
if (this.shouldUseDBNotifications) {
|
|
776
1119
|
await this.#listenForNotifications();
|
|
777
1120
|
// Push coalesced stream and event notifications off the write path.
|
|
@@ -779,6 +1122,32 @@ class SystemDatabase {
|
|
|
779
1122
|
this.#notifierLoop = this.#runNotifier();
|
|
780
1123
|
}
|
|
781
1124
|
}
|
|
1125
|
+
/** Create or migrate the system database. */
|
|
1126
|
+
async runMigrations() {
|
|
1127
|
+
await ensureSystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
|
|
1128
|
+
}
|
|
1129
|
+
/** Throw if this unsharded handle's schema is sharded; an unreachable database is left for its first statement. */
|
|
1130
|
+
async checkShardedSchema() {
|
|
1131
|
+
if (this.sharded)
|
|
1132
|
+
return;
|
|
1133
|
+
let client;
|
|
1134
|
+
try {
|
|
1135
|
+
client = await this.connect();
|
|
1136
|
+
}
|
|
1137
|
+
catch {
|
|
1138
|
+
return;
|
|
1139
|
+
}
|
|
1140
|
+
try {
|
|
1141
|
+
await refuseShardedSchema(client, this.schemaName);
|
|
1142
|
+
}
|
|
1143
|
+
finally {
|
|
1144
|
+
client.release();
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
/** Check the system database is migrated, creating and changing nothing. */
|
|
1148
|
+
async verifyMigrations() {
|
|
1149
|
+
await verifySystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
|
|
1150
|
+
}
|
|
782
1151
|
async destroy() {
|
|
783
1152
|
// Set synchronously, before any await, so no reconnect is scheduled or published after this point.
|
|
784
1153
|
this.#notificationsStopped = true;
|
|
@@ -824,7 +1193,7 @@ class SystemDatabase {
|
|
|
824
1193
|
/** Runs on `client` if given, joining its transaction; otherwise in its own retried transaction. */
|
|
825
1194
|
async initWorkflowStatus(initStatus, creatorXid, client, reusePolicy = 'return-existing') {
|
|
826
1195
|
if (client !== undefined) {
|
|
827
|
-
return await this
|
|
1196
|
+
return await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
828
1197
|
}
|
|
829
1198
|
return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
|
|
830
1199
|
}
|
|
@@ -833,7 +1202,7 @@ class SystemDatabase {
|
|
|
833
1202
|
let shouldCommit = false;
|
|
834
1203
|
try {
|
|
835
1204
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
836
|
-
const result = await this
|
|
1205
|
+
const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
837
1206
|
// If there is an existing DB record and we aren't here to recover it, leave it be.
|
|
838
1207
|
shouldCommit = result.shouldExecuteOnThisExecutor;
|
|
839
1208
|
return result;
|
|
@@ -861,10 +1230,12 @@ class SystemDatabase {
|
|
|
861
1230
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
862
1231
|
const parentOwnerXid = (0, context_1.currentOwnerXid)(parentWorkflowID);
|
|
863
1232
|
if (parentOwnerXid !== undefined) {
|
|
864
|
-
await this
|
|
1233
|
+
await this.checkOwner(client, parentWorkflowID, parentOwnerXid);
|
|
865
1234
|
}
|
|
866
|
-
const result = await this
|
|
867
|
-
await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs,
|
|
1235
|
+
const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
1236
|
+
await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs,
|
|
1237
|
+
// Unkeyed, as only a sharded database records the child's key, through its own override.
|
|
1238
|
+
{ childWorkflowID: initStatus.workflowUUID, ownerChecked: true, childValues: {} });
|
|
868
1239
|
shouldCommit = true;
|
|
869
1240
|
return result;
|
|
870
1241
|
}
|
|
@@ -883,7 +1254,7 @@ class SystemDatabase {
|
|
|
883
1254
|
}
|
|
884
1255
|
}
|
|
885
1256
|
}
|
|
886
|
-
async
|
|
1257
|
+
async initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy) {
|
|
887
1258
|
const resRow = await this.insertWorkflowStatus(client, initStatus, creatorXid);
|
|
888
1259
|
if (reusePolicy === 'reject' && (creatorXid === null || resRow.creator_xid !== creatorXid)) {
|
|
889
1260
|
throw new error_1.DBOSWorkflowIDInUseError(initStatus.workflowUUID, resRow.status, resRow.name);
|
|
@@ -915,6 +1286,7 @@ class SystemDatabase {
|
|
|
915
1286
|
async deadLetterWorkflows(workflowIDs, minRecoveryAttempts) {
|
|
916
1287
|
if (workflowIDs.length === 0)
|
|
917
1288
|
return;
|
|
1289
|
+
const params = [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflow_1.StatusString.PENDING, minRecoveryAttempts];
|
|
918
1290
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
919
1291
|
SET status = $1,
|
|
920
1292
|
owner_xid = NULL,
|
|
@@ -923,17 +1295,18 @@ class SystemDatabase {
|
|
|
923
1295
|
queue_name = NULL,
|
|
924
1296
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
925
1297
|
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
926
|
-
WHERE
|
|
1298
|
+
WHERE ${this.wfIn(undefined, workflowIDs, params)} AND status = $2 AND recovery_attempts >= $3`, params);
|
|
927
1299
|
}
|
|
928
1300
|
/** Highest created_at among still-active rows per partition key, used to seed the in-memory cursor. */
|
|
929
1301
|
async #maxPartitionKeyCreatedAt(keys) {
|
|
930
1302
|
const maxima = new Map();
|
|
931
1303
|
if (keys.length === 0)
|
|
932
1304
|
return maxima;
|
|
1305
|
+
const params = [keys, [workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING]];
|
|
933
1306
|
const { rows } = await this.pool.query(`SELECT queue_partition_key, MAX(created_at) AS max_created_at
|
|
934
1307
|
FROM "${this.schemaName}".workflow_status
|
|
935
|
-
WHERE queue_partition_key = ANY($1) AND status = ANY($2)
|
|
936
|
-
GROUP BY queue_partition_key`,
|
|
1308
|
+
WHERE ${this.scoped(undefined, params)} AND queue_partition_key = ANY($1) AND status = ANY($2)
|
|
1309
|
+
GROUP BY queue_partition_key`, params);
|
|
937
1310
|
for (const row of rows) {
|
|
938
1311
|
if (row.max_created_at !== null) {
|
|
939
1312
|
maxima.set(row.queue_partition_key, Number(row.max_created_at));
|
|
@@ -945,7 +1318,7 @@ class SystemDatabase {
|
|
|
945
1318
|
* Stamp created_at monotonic within each partition key so per-key order holds across batches.
|
|
946
1319
|
* Unordered rows (no partition key) get wall-clock time and never touch the cursors.
|
|
947
1320
|
*/
|
|
948
|
-
async
|
|
1321
|
+
async assignBatchCreatedAt(statuses) {
|
|
949
1322
|
const nowMS = Date.now();
|
|
950
1323
|
const batchKeys = new Set();
|
|
951
1324
|
for (const status of statuses) {
|
|
@@ -990,9 +1363,8 @@ class SystemDatabase {
|
|
|
990
1363
|
* shutdown. Callers retry this themselves.
|
|
991
1364
|
*/
|
|
992
1365
|
async enqueueWorkflows(statuses) {
|
|
993
|
-
const inserted = new Set();
|
|
994
1366
|
if (statuses.length === 0)
|
|
995
|
-
return
|
|
1367
|
+
return new Set();
|
|
996
1368
|
for (const status of statuses) {
|
|
997
1369
|
if (status.status !== workflow_1.StatusString.ENQUEUED) {
|
|
998
1370
|
throw new error_1.DBOSError(`enqueueWorkflows only accepts ${workflow_1.StatusString.ENQUEUED} workflows, but ${status.workflowUUID} is ${status.status}`);
|
|
@@ -1001,7 +1373,63 @@ class SystemDatabase {
|
|
|
1001
1373
|
throw new error_1.DBOSError(`enqueueWorkflows does not support deduplication IDs, but ${status.workflowUUID} has one`);
|
|
1002
1374
|
}
|
|
1003
1375
|
}
|
|
1004
|
-
|
|
1376
|
+
// Stamped before the transaction, as seeding a key's cursor reads on another connection.
|
|
1377
|
+
const createdAts = await this.assignBatchCreatedAt(statuses);
|
|
1378
|
+
const client = await this.connect();
|
|
1379
|
+
try {
|
|
1380
|
+
return await this.inTransaction(client, () => this.insertWorkflowBatch(client, statuses, createdAts));
|
|
1381
|
+
}
|
|
1382
|
+
finally {
|
|
1383
|
+
client.release();
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
/** Batch-enqueue scheduled runs, skipping existing IDs, stamping last_fired_at from `firedAt` (ID to tick) for those inserted, in one transaction. */
|
|
1387
|
+
async enqueueScheduledWorkflows(statuses, firedAt) {
|
|
1388
|
+
if (statuses.length === 0)
|
|
1389
|
+
return new Set();
|
|
1390
|
+
// Stamped before the transaction, as seeding a key's cursor reads on another connection.
|
|
1391
|
+
const createdAts = await this.assignBatchCreatedAt(statuses);
|
|
1392
|
+
const client = await this.connect();
|
|
1393
|
+
try {
|
|
1394
|
+
return await this.inTransaction(client, async () => {
|
|
1395
|
+
const inserted = await this.insertWorkflowBatch(client, statuses, createdAts);
|
|
1396
|
+
await this.stampLastFiredAt(client, statuses, firedAt, inserted);
|
|
1397
|
+
return inserted;
|
|
1398
|
+
});
|
|
1399
|
+
}
|
|
1400
|
+
finally {
|
|
1401
|
+
client.release();
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
/** Set each schedule's last_fired_at to its last inserted tick, `statuses` being in tick order. */
|
|
1405
|
+
async stampLastFiredAt(client, statuses, firedAt, inserted) {
|
|
1406
|
+
const latest = new Map();
|
|
1407
|
+
for (const status of statuses) {
|
|
1408
|
+
const fired = firedAt.get(status.workflowUUID);
|
|
1409
|
+
if (inserted.has(status.workflowUUID) && fired !== undefined && status.scheduleName !== undefined) {
|
|
1410
|
+
latest.set(status.scheduleName, fired);
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
if (latest.size === 0)
|
|
1414
|
+
return;
|
|
1415
|
+
// Sorted, like applySchedules, so concurrent writers lock schedule rows in one order.
|
|
1416
|
+
const names = Array.from(latest.keys()).sort(compareCodeUnits);
|
|
1417
|
+
const lockParams = [names];
|
|
1418
|
+
await client.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules
|
|
1419
|
+
WHERE ${this.global(undefined, lockParams)} AND schedule_name = ANY($1::text[])
|
|
1420
|
+
ORDER BY schedule_name COLLATE "C" FOR UPDATE`, lockParams);
|
|
1421
|
+
const params = [names];
|
|
1422
|
+
const cases = names.map((name) => `WHEN schedule_name = $${params.push(name)} THEN $${params.push(latest.get(name))}`);
|
|
1423
|
+
await client.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = CASE ${cases.join(' ')} END
|
|
1424
|
+
WHERE ${this.global(undefined, params)} AND schedule_name = ANY($1::text[])`, params);
|
|
1425
|
+
}
|
|
1426
|
+
/** Insert these ENQUEUED workflows' rows on `client` with `stamped` from assignBatchCreatedAt, skipping existing IDs. Returns the IDs inserted. */
|
|
1427
|
+
async insertWorkflowBatch(client, batch, stamped) {
|
|
1428
|
+
// Inserted in one global order, so batches overlapping across processes wait on each other instead of deadlocking.
|
|
1429
|
+
const order = batch.map((_, i) => i).sort((a, b) => compareCodeUnits(batch[a].workflowUUID, batch[b].workflowUUID));
|
|
1430
|
+
const statuses = order.map((i) => batch[i]);
|
|
1431
|
+
const createdAts = order.map((i) => stamped[i]);
|
|
1432
|
+
const inserted = new Set();
|
|
1005
1433
|
const columns = [
|
|
1006
1434
|
'workflow_uuid',
|
|
1007
1435
|
'status',
|
|
@@ -1032,75 +1460,72 @@ class SystemDatabase {
|
|
|
1032
1460
|
'schedule_name',
|
|
1033
1461
|
'application_name',
|
|
1034
1462
|
];
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
const
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1463
|
+
// Chunk to stay well under the bind-parameter limit.
|
|
1464
|
+
const chunkSize = 500;
|
|
1465
|
+
for (let start = 0; start < statuses.length; start += chunkSize) {
|
|
1466
|
+
const chunk = statuses.slice(start, start + chunkSize);
|
|
1467
|
+
const keyed = chunk.map((status) => ({
|
|
1468
|
+
...this.wfValues(status.workflowUUID, status.shardKey),
|
|
1469
|
+
...this.parentValues(status.parentWorkflowID),
|
|
1470
|
+
}));
|
|
1471
|
+
const rows = chunk.map((status, i) => {
|
|
1472
|
+
const createdAt = createdAts[start + i];
|
|
1473
|
+
return [
|
|
1474
|
+
...Object.values(keyed[i]),
|
|
1475
|
+
status.workflowUUID,
|
|
1476
|
+
status.status,
|
|
1477
|
+
status.workflowName,
|
|
1050
1478
|
// For cross-language compatibility, these MUST be NULL in the database when not set
|
|
1051
|
-
status.workflowClassName === '' ? null : status.workflowClassName,
|
|
1479
|
+
status.workflowClassName === '' ? null : status.workflowClassName,
|
|
1480
|
+
status.workflowConfigName === '' ? null : status.workflowConfigName,
|
|
1481
|
+
status.queueName ?? null,
|
|
1482
|
+
status.authenticatedUser,
|
|
1483
|
+
status.assumedRole,
|
|
1484
|
+
JSON.stringify(status.authenticatedRoles),
|
|
1485
|
+
status.executorId,
|
|
1486
|
+
status.applicationVersion ?? null,
|
|
1487
|
+
status.applicationID,
|
|
1488
|
+
createdAt,
|
|
1489
|
+
0,
|
|
1490
|
+
createdAt,
|
|
1491
|
+
status.timeoutMS ?? null,
|
|
1492
|
+
status.deadlineEpochMS ?? null,
|
|
1052
1493
|
// Legacy column: the payload lives in workflow_input.
|
|
1053
|
-
null,
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
tuples.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
|
|
1070
|
-
params.push(chunk[i].workflowUUID, chunk[i].input, createdAts[start + i]);
|
|
1071
|
-
}
|
|
1072
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
|
|
1073
|
-
VALUES ${tuples.join(', ')}
|
|
1074
|
-
ON CONFLICT (workflow_uuid) DO NOTHING`, params);
|
|
1494
|
+
null,
|
|
1495
|
+
null,
|
|
1496
|
+
status.priority,
|
|
1497
|
+
status.queuePartitionKey ?? null,
|
|
1498
|
+
status.parentWorkflowID ?? null,
|
|
1499
|
+
status.serialization,
|
|
1500
|
+
null,
|
|
1501
|
+
status.delayUntilEpochMS ?? null,
|
|
1502
|
+
status.attributes ? JSON.stringify(status.attributes) : null,
|
|
1503
|
+
status.scheduleName ?? null,
|
|
1504
|
+
status.applicationName ?? null,
|
|
1505
|
+
];
|
|
1506
|
+
});
|
|
1507
|
+
const returned = await this.insertRows(client, 'workflow_status', [...Object.keys(keyed[0]), ...columns], rows, `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING RETURNING workflow_uuid`);
|
|
1508
|
+
for (const row of returned) {
|
|
1509
|
+
inserted.add(row.workflow_uuid);
|
|
1075
1510
|
}
|
|
1076
|
-
await client.query('COMMIT');
|
|
1077
1511
|
}
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1512
|
+
for (let start = 0; start < statuses.length; start += chunkSize) {
|
|
1513
|
+
const chunk = statuses.slice(start, start + chunkSize);
|
|
1514
|
+
const keyed = chunk.map((status) => this.wfValues(status.workflowUUID, status.shardKey));
|
|
1515
|
+
await this.insertRows(client, 'workflow_input', [...Object.keys(keyed[0]), 'workflow_uuid', 'inputs', 'retention_timestamp'], chunk.map((status, i) => [
|
|
1516
|
+
...Object.values(keyed[i]),
|
|
1517
|
+
status.workflowUUID,
|
|
1518
|
+
status.input,
|
|
1519
|
+
createdAts[start + i],
|
|
1520
|
+
]), `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`);
|
|
1084
1521
|
}
|
|
1085
1522
|
return inserted;
|
|
1086
1523
|
}
|
|
1087
1524
|
async recordWorkflowOutput(workflowID, status, ownerXid) {
|
|
1088
|
-
|
|
1089
|
-
try {
|
|
1090
|
-
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
|
|
1091
|
-
}
|
|
1092
|
-
finally {
|
|
1093
|
-
client.release();
|
|
1094
|
-
}
|
|
1525
|
+
return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
|
|
1095
1526
|
}
|
|
1096
1527
|
async recordWorkflowError(workflowID, status, ownerXid) {
|
|
1097
|
-
|
|
1098
|
-
try {
|
|
1099
|
-
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
|
|
1100
|
-
}
|
|
1101
|
-
finally {
|
|
1102
|
-
client.release();
|
|
1103
|
-
}
|
|
1528
|
+
return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
|
|
1104
1529
|
}
|
|
1105
1530
|
// Record a workflow's terminal outcome (SUCCESS or ERROR), reporting whether
|
|
1106
1531
|
// the write landed. The write applies only to a PENDING row still owned by
|
|
@@ -1111,8 +1536,49 @@ class SystemDatabase {
|
|
|
1111
1536
|
// terminal, handed to another execution (e.g. by a concurrent resume or
|
|
1112
1537
|
// recovery), or deleted; the caller resolves which by awaiting the recorded
|
|
1113
1538
|
// outcome.
|
|
1114
|
-
async #recordWorkflowOutcome(
|
|
1539
|
+
async #recordWorkflowOutcome(workflowID, status, outcome, ownerXid) {
|
|
1115
1540
|
ownerXid ??= (0, context_1.currentOwnerXid)(workflowID);
|
|
1541
|
+
if (!(await this.outcomeInOneStatement())) {
|
|
1542
|
+
return await this.#recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid);
|
|
1543
|
+
}
|
|
1544
|
+
// One autocommit round trip; the upsert reads the UPDATE's rows, so the output lands only with the status.
|
|
1545
|
+
const params = [status, workflow_1.StatusString.PENDING, outcome.output ?? null, outcome.error ?? null];
|
|
1546
|
+
const target = this.wf(undefined, workflowID, params);
|
|
1547
|
+
const owned = ownerXid !== undefined ? `AND owner_xid = $${params.push(ownerXid)}` : '';
|
|
1548
|
+
const keys = this.wfValues(workflowID);
|
|
1549
|
+
const keyColumns = Object.keys(keys);
|
|
1550
|
+
const keyExprs = Object.values(keys).map((value) => `$${params.push(value)}::text`);
|
|
1551
|
+
const workflowParam = `$${params.push(workflowID)}::text`;
|
|
1552
|
+
const { rows } = await this.pool.query(`WITH updated AS (
|
|
1553
|
+
UPDATE "${this.schemaName}".workflow_status
|
|
1554
|
+
SET status = $1,
|
|
1555
|
+
-- As the workflow is complete, remove its deduplication ID; a finished workflow has no owner.
|
|
1556
|
+
deduplication_id = NULL,
|
|
1557
|
+
owner_xid = NULL,
|
|
1558
|
+
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1559
|
+
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1560
|
+
WHERE ${target} AND status = $2 ${owned}
|
|
1561
|
+
RETURNING 1
|
|
1562
|
+
), stored AS (
|
|
1563
|
+
INSERT INTO "${this.schemaName}".workflow_output (${[...keyColumns, 'workflow_uuid', 'output', 'error'].join(', ')})
|
|
1564
|
+
SELECT ${[...keyExprs, workflowParam, '$3::text', '$4::text'].join(', ')}
|
|
1565
|
+
WHERE EXISTS (SELECT 1 FROM updated)
|
|
1566
|
+
-- EXCLUDED is the row this INSERT proposed, so these are the values above.
|
|
1567
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error
|
|
1568
|
+
-- CockroachDB refuses a data-modifying CTE without RETURNING.
|
|
1569
|
+
RETURNING 1
|
|
1570
|
+
)
|
|
1571
|
+
-- Selecting from the CTEs rather than inserting at the top level lets a router see the shard key.
|
|
1572
|
+
SELECT count(*) AS count FROM updated`, params);
|
|
1573
|
+
return Number(rows[0]?.count ?? 0) > 0;
|
|
1574
|
+
}
|
|
1575
|
+
/** Whether a workflow's outcome is written in one statement, rather than a transaction of two. */
|
|
1576
|
+
outcomeInOneStatement() {
|
|
1577
|
+
return Promise.resolve(true);
|
|
1578
|
+
}
|
|
1579
|
+
/** The outcome write as a transaction of two statements, for a backend that cannot run it as one. */
|
|
1580
|
+
async #recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid) {
|
|
1581
|
+
const client = await this.connect();
|
|
1116
1582
|
let committed = false;
|
|
1117
1583
|
try {
|
|
1118
1584
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
@@ -1126,19 +1592,31 @@ class SystemDatabase {
|
|
|
1126
1592
|
// The outcome was not ours to write, so leave no orphan payload.
|
|
1127
1593
|
return false;
|
|
1128
1594
|
}
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1595
|
+
const values = {
|
|
1596
|
+
...this.wfValues(workflowID),
|
|
1597
|
+
workflow_uuid: workflowID,
|
|
1598
|
+
output: outcome.output ?? null,
|
|
1599
|
+
error: outcome.error ?? null,
|
|
1600
|
+
};
|
|
1601
|
+
const columns = Object.keys(values);
|
|
1602
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_output (${columns.join(', ')})
|
|
1603
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
1604
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error`, Object.values(values));
|
|
1132
1605
|
committed = true;
|
|
1133
1606
|
return true;
|
|
1134
1607
|
}
|
|
1135
1608
|
finally {
|
|
1136
|
-
|
|
1137
|
-
|
|
1609
|
+
try {
|
|
1610
|
+
if (committed) {
|
|
1611
|
+
await client.query('COMMIT');
|
|
1612
|
+
}
|
|
1613
|
+
else {
|
|
1614
|
+
// Swallowed: a rollback failure must not mask why we are rolling back.
|
|
1615
|
+
await client.query('ROLLBACK').catch(() => undefined);
|
|
1616
|
+
}
|
|
1138
1617
|
}
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
await client.query('ROLLBACK').catch(() => undefined);
|
|
1618
|
+
finally {
|
|
1619
|
+
client.release();
|
|
1142
1620
|
}
|
|
1143
1621
|
}
|
|
1144
1622
|
}
|
|
@@ -1147,6 +1625,7 @@ class SystemDatabase {
|
|
|
1147
1625
|
const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
|
|
1148
1626
|
// executor_id defaults to "local", so it collides across applications.
|
|
1149
1627
|
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
1628
|
+
const keyColumn = this.keyColumn(undefined);
|
|
1150
1629
|
const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1151
1630
|
SET started_at_epoch_ms = NULL,
|
|
1152
1631
|
status = $1,
|
|
@@ -1157,27 +1636,31 @@ class SystemDatabase {
|
|
|
1157
1636
|
AND executor_id = $4
|
|
1158
1637
|
AND application_version = $5
|
|
1159
1638
|
AND ${scope}
|
|
1160
|
-
RETURNING workflow_uuid`, params);
|
|
1161
|
-
return result.rows.map((row) => row.workflow_uuid);
|
|
1639
|
+
RETURNING workflow_uuid${keyColumn !== undefined ? `, ${keyColumn} AS shard_key` : ''}`, params);
|
|
1640
|
+
return result.rows.map((row) => ({ workflowID: row.workflow_uuid, shardKey: row.shard_key ?? undefined }));
|
|
1162
1641
|
}
|
|
1163
1642
|
async getWorkflowStatus(workflowID, callerID, callerFN) {
|
|
1643
|
+
// Resolved before binding the caller's own key, which a same-ID target would share.
|
|
1644
|
+
const shardKey = this.keyHint(workflowID);
|
|
1164
1645
|
// Inside a workflow this runs on the transaction's client: taking a second pool connection while
|
|
1165
1646
|
// holding the first deadlocks the pool once every connection is held by a waiting caller.
|
|
1166
1647
|
const funcGetStatus = async (client) => {
|
|
1167
|
-
const statuses = await this.#listWorkflows({ workflowIDs: [workflowID] }, client);
|
|
1648
|
+
const statuses = await this.#listWorkflows({ workflowIDs: [workflowID], shardKey }, {}, client);
|
|
1168
1649
|
const status = statuses.find((s) => s.workflowUUID === workflowID);
|
|
1169
1650
|
return status ? JSON.stringify(status) : null;
|
|
1170
1651
|
};
|
|
1171
1652
|
if (callerID && callerFN !== undefined) {
|
|
1172
|
-
const
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1653
|
+
const json = await this.ownKey(callerID, undefined, async () => {
|
|
1654
|
+
const client = await this.connect();
|
|
1655
|
+
try {
|
|
1656
|
+
// Check if the operation has been done before for OAOO (only do this inside a workflow).
|
|
1657
|
+
return await this.inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, () => funcGetStatus(client)));
|
|
1658
|
+
}
|
|
1659
|
+
finally {
|
|
1660
|
+
client.release();
|
|
1661
|
+
}
|
|
1662
|
+
});
|
|
1663
|
+
return parseStatus(json);
|
|
1181
1664
|
}
|
|
1182
1665
|
else {
|
|
1183
1666
|
const json = await funcGetStatus();
|
|
@@ -1191,7 +1674,7 @@ class SystemDatabase {
|
|
|
1191
1674
|
statusFetchChunkSize = 500;
|
|
1192
1675
|
// Retried per chunk so a reconnect refetches one chunk, not every chunk before it.
|
|
1193
1676
|
async fetchWorkflowStatusChunk(workflowIDs) {
|
|
1194
|
-
return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false });
|
|
1677
|
+
return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false }, { keyedIDs: true });
|
|
1195
1678
|
}
|
|
1196
1679
|
/** Fetch many statuses in as few round trips as possible. IDs with no row are omitted. */
|
|
1197
1680
|
async getWorkflowStatuses(workflowIDs) {
|
|
@@ -1222,23 +1705,26 @@ class SystemDatabase {
|
|
|
1222
1705
|
}
|
|
1223
1706
|
}
|
|
1224
1707
|
// ==================== Step Results ====================
|
|
1225
|
-
async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1708
|
+
async getOperationResultAndThrowIfCancelled(workflowID, functionID, ownerKey) {
|
|
1709
|
+
return await this.ownKey(workflowID, ownerKey, async () => {
|
|
1710
|
+
const client = await this.connect();
|
|
1711
|
+
try {
|
|
1712
|
+
return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
1713
|
+
}
|
|
1714
|
+
finally {
|
|
1715
|
+
client.release();
|
|
1716
|
+
}
|
|
1717
|
+
});
|
|
1233
1718
|
}
|
|
1234
1719
|
async getAllOperationResults(workflowID, limit, offset, loadOutput = true) {
|
|
1235
1720
|
// Skip the output/error payload columns when the caller does not need them,
|
|
1236
1721
|
// mirroring how listWorkflows skips its payload joins when loadOutput is false.
|
|
1722
|
+
const childKey = this.keyColumn(undefined, 'child_shard_key');
|
|
1237
1723
|
const columns = loadOutput
|
|
1238
1724
|
? '*'
|
|
1239
|
-
:
|
|
1240
|
-
|
|
1241
|
-
|
|
1725
|
+
: `workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? `, ${childKey}` : ''}`;
|
|
1726
|
+
const params = [];
|
|
1727
|
+
let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE ${this.wf(undefined, workflowID, params)} ORDER BY function_id`;
|
|
1242
1728
|
if (limit !== undefined) {
|
|
1243
1729
|
params.push(limit);
|
|
1244
1730
|
query += ` LIMIT $${params.length}`;
|
|
@@ -1251,14 +1737,18 @@ class SystemDatabase {
|
|
|
1251
1737
|
return rows;
|
|
1252
1738
|
}
|
|
1253
1739
|
async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1740
|
+
// Resolved before binding the caller's own key, which a same-ID child would share.
|
|
1741
|
+
const childValues = this.childValues(options.childWorkflowID);
|
|
1742
|
+
await this.ownKey(workflowID, options.ownerKey, async () => {
|
|
1743
|
+
const client = await this.connect();
|
|
1744
|
+
try {
|
|
1745
|
+
await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, { ...options, childValues }));
|
|
1746
|
+
}
|
|
1747
|
+
finally {
|
|
1748
|
+
client.release();
|
|
1749
|
+
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
|
|
1750
|
+
}
|
|
1751
|
+
});
|
|
1262
1752
|
}
|
|
1263
1753
|
/** Checkpoint an operation's failure so a replay re-throws it instead of re-running. */
|
|
1264
1754
|
async recordOperationError(workflowID, functionID, functionName, startTimeEpochMs, error, childWorkflowID) {
|
|
@@ -1332,9 +1822,10 @@ class SystemDatabase {
|
|
|
1332
1822
|
throw new TypeError('functionID must be defined');
|
|
1333
1823
|
patchName = `DBOS.patch-${patchName}`;
|
|
1334
1824
|
const readCheckpointName = async (queryable) => {
|
|
1825
|
+
const params = [functionID];
|
|
1335
1826
|
const { rows } = await queryable.query(`SELECT function_name
|
|
1336
1827
|
FROM "${this.schemaName}".operation_outputs
|
|
1337
|
-
WHERE
|
|
1828
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
|
|
1338
1829
|
return rows.length === 0 ? undefined : rows[0].function_name;
|
|
1339
1830
|
};
|
|
1340
1831
|
if (deprecated) {
|
|
@@ -1346,13 +1837,20 @@ class SystemDatabase {
|
|
|
1346
1837
|
return { isPatched: true, hasEntry: checkpointName === patchName };
|
|
1347
1838
|
}
|
|
1348
1839
|
// Nondeprecated - skip matching entry, unpatched if nonmatching entry,
|
|
1349
|
-
// If
|
|
1840
|
+
// If the history ends before this point, we insert an entry that indicates it is patched, as an owner-checked checkpoint.
|
|
1841
|
+
// If the history has a checkpoint past this point but none here, the execution reached this point on pre-patch code (e.g., recv).
|
|
1842
|
+
// It is unpatched, and nothing is consumed.
|
|
1350
1843
|
const client = await this.connect();
|
|
1351
1844
|
try {
|
|
1352
|
-
return await this
|
|
1353
|
-
const
|
|
1354
|
-
|
|
1355
|
-
|
|
1845
|
+
return await this.inTransaction(client, async () => {
|
|
1846
|
+
const params = [functionID];
|
|
1847
|
+
const { rows } = await client.query(`SELECT function_id, function_name
|
|
1848
|
+
FROM "${this.schemaName}".operation_outputs
|
|
1849
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND function_id>=$1
|
|
1850
|
+
ORDER BY function_id
|
|
1851
|
+
LIMIT 1`, params);
|
|
1852
|
+
if (rows.length !== 0) {
|
|
1853
|
+
const matches = rows[0].function_id === functionID && rows[0].function_name === patchName;
|
|
1356
1854
|
return { isPatched: matches, hasEntry: matches };
|
|
1357
1855
|
}
|
|
1358
1856
|
const dn = Date.now();
|
|
@@ -1375,7 +1873,7 @@ class SystemDatabase {
|
|
|
1375
1873
|
let frontier = workflowIDs;
|
|
1376
1874
|
while (frontier.length > 0) {
|
|
1377
1875
|
await this.#cancelWorkflows(frontier);
|
|
1378
|
-
const children = await this
|
|
1876
|
+
const children = await this.getDirectChildren(frontier);
|
|
1379
1877
|
frontier = children.filter((id) => !visited.has(id));
|
|
1380
1878
|
for (const id of frontier) {
|
|
1381
1879
|
visited.add(id);
|
|
@@ -1383,41 +1881,45 @@ class SystemDatabase {
|
|
|
1383
1881
|
}
|
|
1384
1882
|
}
|
|
1385
1883
|
async #cancelWorkflows(workflowIDs) {
|
|
1884
|
+
const params = [workflow_1.StatusString.CANCELLED, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR];
|
|
1386
1885
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1387
1886
|
SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
1887
|
+
suspend_key = NULL,
|
|
1388
1888
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1389
1889
|
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1390
|
-
WHERE
|
|
1391
|
-
AND status NOT IN ($
|
|
1890
|
+
WHERE ${this.wfIn(undefined, workflowIDs, params)}
|
|
1891
|
+
AND status NOT IN ($2, $3)`, params);
|
|
1392
1892
|
}
|
|
1393
1893
|
async checkIfCanceled(workflowID) {
|
|
1394
|
-
|
|
1894
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
1895
|
+
await this.ownKey(workflowID, undefined, () => this.#checkIfCanceled(this.pool, workflowID));
|
|
1395
1896
|
}
|
|
1396
1897
|
async resumeWorkflows(workflowIDs, queueName) {
|
|
1397
1898
|
const client = await this.connect();
|
|
1398
1899
|
try {
|
|
1399
1900
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1400
1901
|
// Check existence separately: a zero-row update also means "already complete", a legal no-op.
|
|
1401
|
-
const
|
|
1902
|
+
const existingParams = [];
|
|
1903
|
+
const { rows } = await client.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${this.wfIn(undefined, workflowIDs, existingParams)}`, existingParams);
|
|
1402
1904
|
const existing = new Set(rows.map((r) => r.workflow_uuid));
|
|
1403
1905
|
const missing = workflowIDs.filter((id) => !existing.has(id));
|
|
1404
1906
|
if (missing.length > 0) {
|
|
1405
1907
|
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${missing.join(', ')} does not exist`);
|
|
1406
1908
|
}
|
|
1909
|
+
const updateParams = [
|
|
1910
|
+
workflow_1.StatusString.ENQUEUED,
|
|
1911
|
+
queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1912
|
+
workflow_1.StatusString.SUCCESS,
|
|
1913
|
+
workflow_1.StatusString.ERROR,
|
|
1914
|
+
];
|
|
1407
1915
|
await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1408
1916
|
SET status = $1, owner_xid = NULL, queue_name = $2, recovery_attempts = 0,
|
|
1409
1917
|
workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
|
|
1410
|
-
started_at_epoch_ms = NULL,
|
|
1918
|
+
started_at_epoch_ms = NULL, suspend_key = NULL,
|
|
1411
1919
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1412
1920
|
completed_at = NULL
|
|
1413
|
-
WHERE
|
|
1414
|
-
AND status NOT IN ($
|
|
1415
|
-
workflow_1.StatusString.ENQUEUED,
|
|
1416
|
-
queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1417
|
-
workflowIDs,
|
|
1418
|
-
workflow_1.StatusString.SUCCESS,
|
|
1419
|
-
workflow_1.StatusString.ERROR,
|
|
1420
|
-
]);
|
|
1921
|
+
WHERE ${this.wfIn(undefined, workflowIDs, updateParams)}
|
|
1922
|
+
AND status NOT IN ($3, $4)`, updateParams);
|
|
1421
1923
|
await client.query('COMMIT');
|
|
1422
1924
|
}
|
|
1423
1925
|
catch (e) {
|
|
@@ -1430,16 +1932,18 @@ class SystemDatabase {
|
|
|
1430
1932
|
}
|
|
1431
1933
|
}
|
|
1432
1934
|
async setWorkflowPriority(workflowID, priority) {
|
|
1935
|
+
const params = [priority, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED];
|
|
1433
1936
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1434
1937
|
SET priority = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1435
|
-
WHERE
|
|
1436
|
-
AND status IN ($
|
|
1938
|
+
WHERE ${this.wf(undefined, workflowID, params)}
|
|
1939
|
+
AND status IN ($2, $3)`, params);
|
|
1437
1940
|
}
|
|
1438
1941
|
async setWorkflowDelay(workflowID, delayUntilEpochMS) {
|
|
1942
|
+
const params = [delayUntilEpochMS, workflow_1.StatusString.DELAYED];
|
|
1439
1943
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1440
1944
|
SET delay_until_epoch_ms = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1441
|
-
WHERE
|
|
1442
|
-
AND status = $
|
|
1945
|
+
WHERE ${this.wf(undefined, workflowID, params)}
|
|
1946
|
+
AND status = $2`, params);
|
|
1443
1947
|
}
|
|
1444
1948
|
/**
|
|
1445
1949
|
* Extend an existing debounced DELAYED workflow's delay and update its inputs, atomically.
|
|
@@ -1505,7 +2009,7 @@ class SystemDatabase {
|
|
|
1505
2009
|
if (updated.rows.length > 0) {
|
|
1506
2010
|
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
|
|
1507
2011
|
VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
|
|
1508
|
-
ON CONFLICT (workflow_uuid) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
|
|
2012
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
|
|
1509
2013
|
return {
|
|
1510
2014
|
bouncedWorkflowID: updated.rows[0].workflow_uuid,
|
|
1511
2015
|
holderWorkflowID: null,
|
|
@@ -1515,10 +2019,11 @@ class SystemDatabase {
|
|
|
1515
2019
|
holderApplicationName: null,
|
|
1516
2020
|
};
|
|
1517
2021
|
}
|
|
1518
|
-
//
|
|
2022
|
+
// No application filter, so a holder that blocked the update above is reportable.
|
|
2023
|
+
const holderParams = [params.queueName, params.deduplicationID];
|
|
1519
2024
|
const holder = await client.query(`SELECT workflow_uuid, is_debounced, name, class_name, application_name
|
|
1520
2025
|
FROM "${this.schemaName}".workflow_status
|
|
1521
|
-
WHERE queue_name = $1 AND deduplication_id = $2`,
|
|
2026
|
+
WHERE ${this.scoped(undefined, holderParams)} AND queue_name = $1 AND deduplication_id = $2`, holderParams);
|
|
1522
2027
|
if (holder.rows.length === 0) {
|
|
1523
2028
|
return {
|
|
1524
2029
|
bouncedWorkflowID: null,
|
|
@@ -1539,7 +2044,7 @@ class SystemDatabase {
|
|
|
1539
2044
|
};
|
|
1540
2045
|
}
|
|
1541
2046
|
// Get the immediate (one-level) child workflow IDs for a set of workflows.
|
|
1542
|
-
async
|
|
2047
|
+
async getDirectChildren(workflowIDs) {
|
|
1543
2048
|
if (workflowIDs.length === 0) {
|
|
1544
2049
|
return [];
|
|
1545
2050
|
}
|
|
@@ -1553,7 +2058,7 @@ class SystemDatabase {
|
|
|
1553
2058
|
const descendants = new Set();
|
|
1554
2059
|
let frontier = [workflowID];
|
|
1555
2060
|
while (frontier.length > 0) {
|
|
1556
|
-
const children = await this
|
|
2061
|
+
const children = await this.getDirectChildren(frontier);
|
|
1557
2062
|
frontier = children.filter((id) => !descendants.has(id));
|
|
1558
2063
|
for (const id of frontier) {
|
|
1559
2064
|
descendants.add(id);
|
|
@@ -1561,6 +2066,7 @@ class SystemDatabase {
|
|
|
1561
2066
|
}
|
|
1562
2067
|
return [...descendants];
|
|
1563
2068
|
}
|
|
2069
|
+
/** Delete workflows and all associated data, and with deleteChildren every workflow they started, recursively. */
|
|
1564
2070
|
async deleteWorkflows(workflowIDs, deleteChildren = false) {
|
|
1565
2071
|
const allIds = [...workflowIDs];
|
|
1566
2072
|
if (deleteChildren) {
|
|
@@ -1568,15 +2074,18 @@ class SystemDatabase {
|
|
|
1568
2074
|
allIds.push(...(await this.getWorkflowChildren(wfid)));
|
|
1569
2075
|
}
|
|
1570
2076
|
}
|
|
2077
|
+
await this.deleteWorkflowsInternal(allIds);
|
|
2078
|
+
}
|
|
2079
|
+
async deleteWorkflowsInternal(allIds) {
|
|
1571
2080
|
const client = await this.connect();
|
|
1572
2081
|
try {
|
|
1573
2082
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1574
2083
|
// The payload tables carry no foreign key, so the status delete does not cascade
|
|
1575
2084
|
// into them; without this they survive as orphans.
|
|
1576
|
-
for (const table of ['workflow_input', 'workflow_output', 'operation_outputs']) {
|
|
1577
|
-
|
|
2085
|
+
for (const table of ['workflow_input', 'workflow_output', 'operation_outputs', 'workflow_status']) {
|
|
2086
|
+
const params = [];
|
|
2087
|
+
await client.query(`DELETE FROM "${this.schemaName}".${table} WHERE ${this.wfIn(undefined, allIds, params)}`, params);
|
|
1578
2088
|
}
|
|
1579
|
-
await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [allIds]);
|
|
1580
2089
|
await client.query('COMMIT');
|
|
1581
2090
|
}
|
|
1582
2091
|
catch (e) {
|
|
@@ -1617,67 +2126,73 @@ class SystemDatabase {
|
|
|
1617
2126
|
const client = await this.connect();
|
|
1618
2127
|
try {
|
|
1619
2128
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1620
|
-
const
|
|
2129
|
+
const statusParams = [];
|
|
2130
|
+
const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE ${this.wf(undefined, workflowID, statusParams)}`, statusParams);
|
|
1621
2131
|
if (statusRows.length === 0) {
|
|
1622
2132
|
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
|
|
1623
2133
|
}
|
|
1624
2134
|
const status = statusRows[0].status;
|
|
1625
|
-
if (
|
|
2135
|
+
if ((0, workflow_1.isWorkflowActive)(status)) {
|
|
1626
2136
|
throw new error_1.DBOSError(`Cannot rewind ${workflowID} (${status}): only a workflow in a terminal state can be rewound, so cancel it first`);
|
|
1627
2137
|
}
|
|
1628
2138
|
// Roll workflow_events back to the last value published before the cut, using
|
|
1629
2139
|
// workflow_events_history as an undo log, then rewind that history too.
|
|
1630
2140
|
// `publishedPastCut` is the correlated EXISTS both halves key off.
|
|
1631
|
-
const publishedPastCut = `EXISTS (
|
|
2141
|
+
const publishedPastCut = (key, params) => `EXISTS (
|
|
1632
2142
|
SELECT 1 FROM "${schema}".workflow_events_history discarded
|
|
1633
|
-
WHERE discarded
|
|
2143
|
+
WHERE ${this.wf('discarded', workflowID, params)} AND discarded.key = ${key} AND discarded.function_id >= $1
|
|
1634
2144
|
)`;
|
|
1635
2145
|
// First unpublish everything the discarded run touched.
|
|
2146
|
+
const unpublishParams = [startStep];
|
|
1636
2147
|
await client.query(`DELETE FROM "${schema}".workflow_events
|
|
1637
|
-
WHERE
|
|
2148
|
+
WHERE ${this.wf(undefined, workflowID, unpublishParams)}
|
|
2149
|
+
AND ${publishedPastCut('"' + schema + '".workflow_events.key', unpublishParams)}`, unpublishParams);
|
|
1638
2150
|
// Then restore those keys, if any, to the value published below the cut.
|
|
1639
|
-
|
|
1640
|
-
|
|
2151
|
+
const restoreParams = [startStep];
|
|
2152
|
+
const keyColumns = this.wfColumns(workflowID, restoreParams);
|
|
2153
|
+
await client.query(`INSERT INTO "${schema}".workflow_events (${[...keyColumns.map(([c]) => c), 'workflow_uuid', 'key', 'value', 'serialization'].join(', ')})
|
|
2154
|
+
SELECT ${[...keyColumns.map(([, e]) => e), 'surviving.workflow_uuid', 'surviving.key', 'surviving.value', 'surviving.serialization'].join(', ')}
|
|
1641
2155
|
FROM (
|
|
1642
2156
|
SELECT weh.workflow_uuid, weh.key, weh.value, weh.serialization,
|
|
1643
2157
|
ROW_NUMBER() OVER (PARTITION BY weh.key ORDER BY weh.function_id DESC) AS rn
|
|
1644
2158
|
FROM "${schema}".workflow_events_history weh
|
|
1645
|
-
WHERE weh
|
|
1646
|
-
AND ${publishedPastCut
|
|
2159
|
+
WHERE ${this.wf('weh', workflowID, restoreParams)} AND weh.function_id < $1
|
|
2160
|
+
AND ${publishedPastCut('weh.key', restoreParams)}
|
|
1647
2161
|
) surviving
|
|
1648
|
-
WHERE surviving.rn = 1`,
|
|
2162
|
+
WHERE surviving.rn = 1`, restoreParams);
|
|
1649
2163
|
// Clear the streams' close sentinels so the replay can append again.
|
|
2164
|
+
const sentinelParams = [startStep, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name()];
|
|
1650
2165
|
await client.query(`DELETE FROM "${schema}".streams
|
|
1651
|
-
WHERE
|
|
2166
|
+
WHERE ${this.wf(undefined, workflowID, sentinelParams)} AND function_id >= $1 AND value = $2 AND serialization = $3`, sentinelParams);
|
|
1652
2167
|
// Discard the steps and the events history past the cut.
|
|
1653
2168
|
for (const table of ['operation_outputs', 'workflow_events_history']) {
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
startStep,
|
|
1657
|
-
]);
|
|
2169
|
+
const params = [startStep];
|
|
2170
|
+
await client.query(`DELETE FROM "${schema}".${table} WHERE ${this.wf(undefined, workflowID, params)} AND function_id >= $1`, params);
|
|
1658
2171
|
}
|
|
1659
2172
|
// Delete messages consumed or received after the rewind point
|
|
1660
|
-
|
|
1661
|
-
await client.query(`DELETE FROM "${schema}".
|
|
2173
|
+
const messageParams = [startStep];
|
|
2174
|
+
await client.query(`DELETE FROM "${schema}".notifications
|
|
2175
|
+
WHERE ${this.wf(undefined, workflowID, messageParams, 'destination_uuid')}
|
|
2176
|
+
AND (consumed_by_function_id >= $1 OR consumed = false)`, messageParams);
|
|
2177
|
+
const outputParams = [];
|
|
2178
|
+
await client.query(`DELETE FROM "${schema}".workflow_output WHERE ${this.wf(undefined, workflowID, outputParams)}`, outputParams);
|
|
1662
2179
|
// Re-enqueue the workflow. Re-asserting the status we read keeps a workflow that
|
|
1663
2180
|
// moved on underneath us from being resurrected.
|
|
1664
|
-
const setVersion = options.applicationVersion !== undefined ? ', application_version = $6' : '';
|
|
1665
2181
|
const params = [
|
|
1666
2182
|
workflow_1.StatusString.ENQUEUED,
|
|
1667
2183
|
options.queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1668
2184
|
options.queuePartitionKey ?? null,
|
|
1669
|
-
workflowID,
|
|
1670
2185
|
status,
|
|
1671
2186
|
];
|
|
1672
|
-
|
|
1673
|
-
params.push(options.applicationVersion)
|
|
1674
|
-
|
|
2187
|
+
const setVersion = options.applicationVersion !== undefined
|
|
2188
|
+
? `, application_version = $${params.push(options.applicationVersion)}`
|
|
2189
|
+
: '';
|
|
1675
2190
|
const { rowCount } = await client.query(`UPDATE "${schema}".workflow_status
|
|
1676
2191
|
SET status = $1, owner_xid = NULL, queue_name = $2, queue_partition_key = $3, recovery_attempts = 0,
|
|
1677
2192
|
workflow_deadline_epoch_ms = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
1678
|
-
completed_at = NULL, output = NULL, error = NULL,
|
|
2193
|
+
suspend_key = NULL, completed_at = NULL, output = NULL, error = NULL,
|
|
1679
2194
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint${setVersion}
|
|
1680
|
-
WHERE
|
|
2195
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND status = $4`, params);
|
|
1681
2196
|
if (rowCount !== 1) {
|
|
1682
2197
|
throw new error_1.DBOSError(`Workflow ${workflowID} changed status while being rewound; retry the rewind`);
|
|
1683
2198
|
}
|
|
@@ -1707,13 +2222,14 @@ class SystemDatabase {
|
|
|
1707
2222
|
try {
|
|
1708
2223
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1709
2224
|
// Fetch the status of all original workflows inside the transaction.
|
|
2225
|
+
const statusParams = [];
|
|
1710
2226
|
const { rows: statusRows } = await client.query(`SELECT ws.workflow_uuid, ws.name, ws.class_name, ws.config_name, ws.application_id,
|
|
1711
2227
|
ws.authenticated_user, ws.authenticated_roles, ws.assumed_role,
|
|
1712
2228
|
COALESCE(wi.inputs, ws.inputs) AS inputs, ws.serialization,
|
|
1713
2229
|
ws.application_version, ws.attributes, ws.application_name
|
|
1714
2230
|
FROM "${this.schemaName}".workflow_status ws
|
|
1715
|
-
LEFT JOIN "${this.schemaName}".workflow_input wi ON
|
|
1716
|
-
WHERE ws
|
|
2231
|
+
LEFT JOIN "${this.schemaName}".workflow_input wi ON ${this.wfJoin('ws', 'wi')}
|
|
2232
|
+
WHERE ${this.wfIn('ws', originalWorkflowIDs, statusParams)}`, statusParams);
|
|
1717
2233
|
const statusByID = new Map(statusRows.map((r) => [r.workflow_uuid, r]));
|
|
1718
2234
|
for (const wid of originalWorkflowIDs) {
|
|
1719
2235
|
if (!statusByID.has(wid)) {
|
|
@@ -1749,35 +2265,42 @@ class SystemDatabase {
|
|
|
1749
2265
|
forkedWorkflowIDs[i],
|
|
1750
2266
|
statusByID.get(origID).application_name ?? this.appName ?? null,
|
|
1751
2267
|
]));
|
|
1752
|
-
const
|
|
1753
|
-
|
|
1754
|
-
let paramIdx = 1;
|
|
1755
|
-
for (let i = 0; i < originalWorkflowIDs.length; i++) {
|
|
1756
|
-
const origID = originalWorkflowIDs[i];
|
|
2268
|
+
const forkKeys = forkedWorkflowIDs.map((forkID) => this.wfValues(forkID));
|
|
2269
|
+
await this.insertRows(client, 'workflow_status', [...Object.keys(forkKeys[0]), ...insertCols], originalWorkflowIDs.map((origID, i) => {
|
|
1757
2270
|
const forkID = forkedWorkflowIDs[i];
|
|
1758
2271
|
const ws = statusByID.get(origID);
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
2272
|
+
return [
|
|
2273
|
+
...Object.values(forkKeys[i]),
|
|
2274
|
+
forkID,
|
|
2275
|
+
workflow_1.StatusString.ENQUEUED,
|
|
2276
|
+
ws.name,
|
|
2277
|
+
ws.class_name ?? null,
|
|
2278
|
+
ws.config_name ?? null,
|
|
2279
|
+
queueName,
|
|
2280
|
+
ws.authenticated_user,
|
|
2281
|
+
ws.assumed_role,
|
|
2282
|
+
ws.authenticated_roles,
|
|
2283
|
+
options.applicationVersion ?? ws.application_version ?? null,
|
|
2284
|
+
ws.application_id,
|
|
2285
|
+
// Legacy column: the payload lives in workflow_input.
|
|
2286
|
+
null,
|
|
2287
|
+
options.queuePartitionKey ?? null,
|
|
2288
|
+
origID,
|
|
2289
|
+
ws.serialization,
|
|
2290
|
+
ws.attributes ? JSON.stringify(ws.attributes) : null,
|
|
2291
|
+
forkOwners.get(forkID) ?? null,
|
|
2292
|
+
...(options.timeoutMS !== undefined ? [options.timeoutMS] : []),
|
|
2293
|
+
];
|
|
2294
|
+
}));
|
|
2295
|
+
await this.insertRows(client, 'workflow_input', [...Object.keys(forkKeys[0]), 'workflow_uuid', 'inputs'], originalWorkflowIDs.map((origID, i) => [
|
|
2296
|
+
...Object.values(forkKeys[i]),
|
|
2297
|
+
forkedWorkflowIDs[i],
|
|
2298
|
+
statusByID.get(origID).inputs,
|
|
2299
|
+
]));
|
|
1779
2300
|
// Mark all original workflows as having been forked from.
|
|
1780
|
-
|
|
2301
|
+
const forkedFromParams = [];
|
|
2302
|
+
await client.query(`UPDATE "${this.schemaName}".workflow_status SET was_forked_from = TRUE
|
|
2303
|
+
WHERE ${this.wfIn(undefined, originalWorkflowIDs, forkedFromParams)}`, forkedFromParams);
|
|
1781
2304
|
// For workflows with start_step > 0, copy checkpoints/events/streams.
|
|
1782
2305
|
// Build a mapping CTE so each copy is a single SQL statement.
|
|
1783
2306
|
const forkMappings = originalWorkflowIDs
|
|
@@ -1794,55 +2317,85 @@ class SystemDatabase {
|
|
|
1794
2317
|
mIdx += 4;
|
|
1795
2318
|
}
|
|
1796
2319
|
const mappingCTE = `WITH mapping(orig_id, fork_id, start_step, owner) AS (VALUES ${mappingValues.join(', ')})`;
|
|
2320
|
+
// The forks live under their originals' shard key, read from the scope; a copy's params extend the mapping's.
|
|
2321
|
+
const copyParams = () => [...mappingParams];
|
|
2322
|
+
const keyed = (params) => {
|
|
2323
|
+
const columns = this.scopedColumns(params);
|
|
2324
|
+
return { names: columns.map(([c]) => `${c}, `).join(''), exprs: columns.map(([, e]) => `${e}, `).join('') };
|
|
2325
|
+
};
|
|
2326
|
+
// A replacement child is a fork, so it shares the replaced child's key.
|
|
2327
|
+
const childKey = this.keyColumn('oo', 'child_shard_key');
|
|
2328
|
+
if (options.replacementChildren && childKey !== undefined) {
|
|
2329
|
+
// A replacement names a child by ID, which can't pick one of several keys.
|
|
2330
|
+
const ambiguityParams = copyParams();
|
|
2331
|
+
const replaced = `$${ambiguityParams.push(Object.keys(options.replacementChildren))}::text[]`;
|
|
2332
|
+
const { rows: ambiguous } = await client.query(`${mappingCTE}
|
|
2333
|
+
SELECT oo.child_workflow_id
|
|
2334
|
+
FROM mapping m
|
|
2335
|
+
JOIN "${this.schemaName}".operation_outputs oo
|
|
2336
|
+
ON ${this.scoped('oo', ambiguityParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step
|
|
2337
|
+
WHERE oo.child_workflow_id = ANY(${replaced})
|
|
2338
|
+
GROUP BY oo.child_workflow_id
|
|
2339
|
+
HAVING COUNT(DISTINCT ${childKey}) > 1
|
|
2340
|
+
LIMIT 1`, ambiguityParams);
|
|
2341
|
+
if (ambiguous.length > 0) {
|
|
2342
|
+
throw new error_1.DBOSError(`Can't replace child workflow ${ambiguous[0].child_workflow_id}: the forked steps refer to children with that ID under more than one shard key`);
|
|
2343
|
+
}
|
|
2344
|
+
}
|
|
1797
2345
|
// Build the child_workflow_id expression, applying replacements if provided.
|
|
2346
|
+
const ooParams = copyParams();
|
|
1798
2347
|
let childWfExpr = 'oo.child_workflow_id';
|
|
1799
|
-
const ooParams = [...mappingParams];
|
|
1800
2348
|
if (options.replacementChildren && Object.keys(options.replacementChildren).length > 0) {
|
|
1801
2349
|
const whenClauses = [];
|
|
1802
2350
|
for (const [oldId, newId] of Object.entries(options.replacementChildren)) {
|
|
1803
|
-
whenClauses.push(`WHEN oo.child_workflow_id = $${
|
|
1804
|
-
ooParams.push(oldId, newId);
|
|
1805
|
-
mIdx += 2;
|
|
2351
|
+
whenClauses.push(`WHEN oo.child_workflow_id = $${ooParams.push(oldId)} THEN $${ooParams.push(newId)}::text`);
|
|
1806
2352
|
}
|
|
1807
2353
|
childWfExpr = `CASE ${whenClauses.join(' ')} ELSE oo.child_workflow_id END`;
|
|
1808
2354
|
}
|
|
2355
|
+
const ooKeys = keyed(ooParams);
|
|
1809
2356
|
// Copy operation outputs
|
|
1810
2357
|
await client.query(`${mappingCTE}
|
|
1811
2358
|
INSERT INTO "${this.schemaName}".operation_outputs
|
|
1812
|
-
(workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp)
|
|
1813
|
-
SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
2359
|
+
(${ooKeys.names}workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? ', child_shard_key' : ''})
|
|
2360
|
+
SELECT ${ooKeys.exprs}m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint${childKey !== undefined ? `, ${childKey}` : ''}
|
|
1814
2361
|
FROM mapping m
|
|
1815
2362
|
JOIN "${this.schemaName}".operation_outputs oo
|
|
1816
|
-
ON oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
|
|
2363
|
+
ON ${this.scoped('oo', ooParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
|
|
1817
2364
|
// Copy streams
|
|
2365
|
+
const streamParams = copyParams();
|
|
2366
|
+
const streamKeys = keyed(streamParams);
|
|
1818
2367
|
await client.query(`${mappingCTE}
|
|
1819
2368
|
INSERT INTO "${this.schemaName}".streams
|
|
1820
|
-
(workflow_uuid, key, value, serialization, "offset", function_id)
|
|
1821
|
-
SELECT m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
|
|
2369
|
+
(${streamKeys.names}workflow_uuid, key, value, serialization, "offset", function_id)
|
|
2370
|
+
SELECT ${streamKeys.exprs}m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
|
|
1822
2371
|
FROM mapping m
|
|
1823
2372
|
JOIN "${this.schemaName}".streams s
|
|
1824
|
-
ON s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`,
|
|
2373
|
+
ON ${this.scoped('s', streamParams)} AND s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`, streamParams);
|
|
1825
2374
|
// Copy events history
|
|
2375
|
+
const historyParams = copyParams();
|
|
2376
|
+
const historyKeys = keyed(historyParams);
|
|
1826
2377
|
await client.query(`${mappingCTE}
|
|
1827
2378
|
INSERT INTO "${this.schemaName}".workflow_events_history
|
|
1828
|
-
(workflow_uuid, function_id, key, value, serialization)
|
|
1829
|
-
SELECT m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
|
|
2379
|
+
(${historyKeys.names}workflow_uuid, function_id, key, value, serialization)
|
|
2380
|
+
SELECT ${historyKeys.exprs}m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
|
|
1830
2381
|
FROM mapping m
|
|
1831
2382
|
JOIN "${this.schemaName}".workflow_events_history weh
|
|
1832
|
-
ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`,
|
|
2383
|
+
ON ${this.scoped('weh', historyParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`, historyParams);
|
|
1833
2384
|
// Copy only the latest version of each event using a window function
|
|
2385
|
+
const eventParams = copyParams();
|
|
2386
|
+
const eventKeys = keyed(eventParams);
|
|
1834
2387
|
await client.query(`${mappingCTE}
|
|
1835
2388
|
INSERT INTO "${this.schemaName}".workflow_events
|
|
1836
|
-
(workflow_uuid, key, value, serialization)
|
|
1837
|
-
SELECT ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
|
|
2389
|
+
(${eventKeys.names}workflow_uuid, key, value, serialization)
|
|
2390
|
+
SELECT ${eventKeys.exprs}ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
|
|
1838
2391
|
FROM (
|
|
1839
2392
|
SELECT m.fork_id AS workflow_uuid, weh.key, weh.value, weh.serialization,
|
|
1840
|
-
ROW_NUMBER() OVER (PARTITION BY
|
|
2393
|
+
ROW_NUMBER() OVER (PARTITION BY m.fork_id, weh.key ORDER BY weh.function_id DESC) AS rn
|
|
1841
2394
|
FROM mapping m
|
|
1842
2395
|
JOIN "${this.schemaName}".workflow_events_history weh
|
|
1843
|
-
ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
|
|
2396
|
+
ON ${this.scoped('weh', eventParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
|
|
1844
2397
|
) ranked
|
|
1845
|
-
WHERE ranked.rn = 1`,
|
|
2398
|
+
WHERE ranked.rn = 1`, eventParams);
|
|
1846
2399
|
}
|
|
1847
2400
|
await client.query('COMMIT');
|
|
1848
2401
|
return forkedWorkflowIDs;
|
|
@@ -1965,27 +2518,36 @@ class SystemDatabase {
|
|
|
1965
2518
|
}
|
|
1966
2519
|
/** Cancellation check for polling waits: goes through the limiter so readers cannot starve the pool. */
|
|
1967
2520
|
async checkIfCanceledLimited(workflowID) {
|
|
1968
|
-
|
|
2521
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
2522
|
+
const params = [];
|
|
2523
|
+
const target = this.ownKey(workflowID, undefined, () => this.wf(undefined, workflowID, params));
|
|
2524
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${target}`, params));
|
|
2525
|
+
if (rows[0]?.status === workflow_1.StatusString.CANCELLED) {
|
|
2526
|
+
throw new error_1.DBOSWorkflowCancelledError(workflowID);
|
|
2527
|
+
}
|
|
1969
2528
|
}
|
|
1970
2529
|
watchForCancellation(workflowID) {
|
|
1971
2530
|
const controller = new AbortController();
|
|
1972
2531
|
let state = 'idle';
|
|
2532
|
+
// The running workflow's own key, resolved now for the poller that reads it later outside this context.
|
|
2533
|
+
const shardKey = this.ownKey(workflowID, undefined, () => this.keyHint(workflowID));
|
|
2534
|
+
const waiterKey = this.ownKey(workflowID, undefined, () => this.waiter(workflowID, ''));
|
|
1973
2535
|
const start = () => {
|
|
1974
2536
|
state = 'watching';
|
|
1975
2537
|
if (this.#destroyed)
|
|
1976
2538
|
return;
|
|
1977
|
-
let watchers = this.#cancelWatchers.get(
|
|
2539
|
+
let watchers = this.#cancelWatchers.get(waiterKey);
|
|
1978
2540
|
if (watchers === undefined) {
|
|
1979
|
-
watchers = new Set();
|
|
1980
|
-
this.#cancelWatchers.set(
|
|
2541
|
+
watchers = { workflowID, shardKey, controllers: new Set() };
|
|
2542
|
+
this.#cancelWatchers.set(waiterKey, watchers);
|
|
1981
2543
|
}
|
|
1982
|
-
watchers.add(controller);
|
|
1983
|
-
this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => this.#runCancelPoller());
|
|
2544
|
+
watchers.controllers.add(controller);
|
|
2545
|
+
this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => (0, shard_keys_1.withClearedShardKeyState)(() => this.#runCancelPoller()));
|
|
1984
2546
|
};
|
|
1985
2547
|
const stop = () => {
|
|
1986
|
-
const watchers = this.#cancelWatchers.get(
|
|
1987
|
-
if (watchers?.delete(controller) && watchers.size === 0) {
|
|
1988
|
-
this.#cancelWatchers.delete(
|
|
2548
|
+
const watchers = this.#cancelWatchers.get(waiterKey);
|
|
2549
|
+
if (watchers?.controllers.delete(controller) && watchers.controllers.size === 0) {
|
|
2550
|
+
this.#cancelWatchers.delete(waiterKey);
|
|
1989
2551
|
if (this.#cancelWatchers.size === 0)
|
|
1990
2552
|
this.#cancelPollerWake?.();
|
|
1991
2553
|
}
|
|
@@ -2024,15 +2586,36 @@ class SystemDatabase {
|
|
|
2024
2586
|
this.#cancelPollerLoop = undefined;
|
|
2025
2587
|
}
|
|
2026
2588
|
async #abortCancelledWatchers() {
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2589
|
+
// One read per shard key, each bound to its own workflows.
|
|
2590
|
+
const byKey = new Map();
|
|
2591
|
+
for (const { workflowID, shardKey } of this.#cancelWatchers.values()) {
|
|
2592
|
+
const ids = byKey.get(shardKey) ?? [];
|
|
2593
|
+
ids.push(workflowID);
|
|
2594
|
+
byKey.set(shardKey, ids);
|
|
2595
|
+
}
|
|
2596
|
+
for (const [shardKey, workflowIDs] of byKey) {
|
|
2597
|
+
// Each key's read on its own, so one key's failure doesn't hide another key's cancellations.
|
|
2598
|
+
let rows;
|
|
2599
|
+
try {
|
|
2600
|
+
({ rows } = await (0, shard_keys_1.bindShardKeys)(workflowIDs.map((id) => [id, shardKey]), () => {
|
|
2601
|
+
const params = [workflow_1.StatusString.CANCELLED];
|
|
2602
|
+
const ids = this.wfIn(undefined, workflowIDs, params);
|
|
2603
|
+
return this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${ids} AND status = $1`, params));
|
|
2604
|
+
}));
|
|
2605
|
+
}
|
|
2606
|
+
catch (e) {
|
|
2607
|
+
this.logger.warn(`Error polling status for cancellable steps: ${String(e)}`);
|
|
2031
2608
|
continue;
|
|
2032
|
-
|
|
2033
|
-
const
|
|
2034
|
-
for (const
|
|
2035
|
-
|
|
2609
|
+
}
|
|
2610
|
+
const cancelled = new Set(rows.map((row) => row.workflow_uuid));
|
|
2611
|
+
for (const [waiterKey, watchers] of this.#cancelWatchers) {
|
|
2612
|
+
if (watchers.shardKey !== shardKey || !cancelled.has(watchers.workflowID))
|
|
2613
|
+
continue;
|
|
2614
|
+
this.#cancelWatchers.delete(waiterKey);
|
|
2615
|
+
const reason = new error_1.DBOSWorkflowCancelledError(watchers.workflowID);
|
|
2616
|
+
for (const controller of watchers.controllers)
|
|
2617
|
+
controller.abort(reason);
|
|
2618
|
+
}
|
|
2036
2619
|
}
|
|
2037
2620
|
}
|
|
2038
2621
|
// A missing row normally means the workflow has not been inserted yet, so
|
|
@@ -2044,21 +2627,23 @@ class SystemDatabase {
|
|
|
2044
2627
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2045
2628
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2046
2629
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
2047
|
-
// Record the durable timeout deadline once before polling.
|
|
2630
|
+
// Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
|
|
2048
2631
|
// first call and reads back the same value on recovery, so it never changes across iterations.
|
|
2049
2632
|
if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
|
|
2050
|
-
finishTime = await this
|
|
2633
|
+
finishTime = await this.durableSleepInternal(callerID, timerFuncID, timeoutms);
|
|
2051
2634
|
}
|
|
2052
2635
|
while (true) {
|
|
2053
2636
|
if (callerID)
|
|
2054
2637
|
await this.checkIfCanceledLimited(callerID);
|
|
2055
2638
|
let rows;
|
|
2056
2639
|
try {
|
|
2640
|
+
const params = [];
|
|
2641
|
+
const target = this.wf('ws', workflowID, params);
|
|
2057
2642
|
({ rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT ws.status, COALESCE(wo.output, ws.output) AS output,
|
|
2058
2643
|
COALESCE(wo.error, ws.error) AS error, ws.serialization
|
|
2059
2644
|
FROM "${this.schemaName}".workflow_status ws
|
|
2060
|
-
LEFT JOIN "${this.schemaName}".workflow_output wo ON
|
|
2061
|
-
WHERE
|
|
2645
|
+
LEFT JOIN "${this.schemaName}".workflow_output wo ON ${this.wfJoin('ws', 'wo')}
|
|
2646
|
+
WHERE ${target}`, params)));
|
|
2062
2647
|
}
|
|
2063
2648
|
catch (e) {
|
|
2064
2649
|
const err = e;
|
|
@@ -2095,21 +2680,27 @@ class SystemDatabase {
|
|
|
2095
2680
|
}
|
|
2096
2681
|
}
|
|
2097
2682
|
async awaitFirstWorkflowId(workflowIds, callerID, pollingIntervalMs) {
|
|
2098
|
-
const placeholders = workflowIds.map((_, i) => `$${i + 1}`).join(', ');
|
|
2099
2683
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
2100
2684
|
while (true) {
|
|
2101
2685
|
if (callerID)
|
|
2102
2686
|
await this.checkIfCanceledLimited(callerID);
|
|
2103
|
-
const
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
LIMIT 1`, workflowIds));
|
|
2107
|
-
if (rows.length > 0) {
|
|
2108
|
-
return rows[0].workflow_uuid;
|
|
2687
|
+
const first = await this.checkFirstWorkflowId(workflowIds);
|
|
2688
|
+
if (first !== undefined) {
|
|
2689
|
+
return first;
|
|
2109
2690
|
}
|
|
2110
2691
|
await (0, utils_1.sleepms)(pollIntervalMs);
|
|
2111
2692
|
}
|
|
2112
2693
|
}
|
|
2694
|
+
/** The first of these workflows found complete, if any. */
|
|
2695
|
+
async checkFirstWorkflowId(workflowIds) {
|
|
2696
|
+
const params = [];
|
|
2697
|
+
const ids = this.wfIn(undefined, workflowIds, params);
|
|
2698
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
2699
|
+
WHERE ${ids}
|
|
2700
|
+
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')
|
|
2701
|
+
LIMIT 1`, params));
|
|
2702
|
+
return rows[0]?.workflow_uuid;
|
|
2703
|
+
}
|
|
2113
2704
|
async awaitWorkflowIds(workflowIds, callerID, pollingIntervalMs) {
|
|
2114
2705
|
const remainingWorkflowIds = new Set(workflowIds);
|
|
2115
2706
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
@@ -2117,11 +2708,8 @@ class SystemDatabase {
|
|
|
2117
2708
|
const currentWorkflowIds = [...remainingWorkflowIds];
|
|
2118
2709
|
if (callerID)
|
|
2119
2710
|
await this.checkIfCanceledLimited(callerID);
|
|
2120
|
-
const
|
|
2121
|
-
|
|
2122
|
-
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, [currentWorkflowIds]));
|
|
2123
|
-
for (const row of rows) {
|
|
2124
|
-
remainingWorkflowIds.delete(row.workflow_uuid);
|
|
2711
|
+
for (const workflowID of await this.checkCompletedWorkflowIds(currentWorkflowIds)) {
|
|
2712
|
+
remainingWorkflowIds.delete(workflowID);
|
|
2125
2713
|
}
|
|
2126
2714
|
if (remainingWorkflowIds.size === 0) {
|
|
2127
2715
|
return;
|
|
@@ -2129,9 +2717,18 @@ class SystemDatabase {
|
|
|
2129
2717
|
await (0, utils_1.sleepms)(pollIntervalMs);
|
|
2130
2718
|
}
|
|
2131
2719
|
}
|
|
2720
|
+
/** Those of these workflows found complete. */
|
|
2721
|
+
async checkCompletedWorkflowIds(workflowIds) {
|
|
2722
|
+
const params = [];
|
|
2723
|
+
const ids = this.wfIn(undefined, workflowIds, params);
|
|
2724
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
2725
|
+
WHERE ${ids}
|
|
2726
|
+
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')`, params));
|
|
2727
|
+
return rows.map((row) => row.workflow_uuid);
|
|
2728
|
+
}
|
|
2132
2729
|
// ==================== Sleep ====================
|
|
2133
2730
|
async durableSleepms(workflowID, functionID, durationMS) {
|
|
2134
|
-
const endTime = await this
|
|
2731
|
+
const endTime = await this.durableSleepInternal(workflowID, functionID, durationMS, true);
|
|
2135
2732
|
while (Date.now() < endTime) {
|
|
2136
2733
|
await (0, utils_1.sleepms)(Math.min(endTime - Date.now(), utils_1.sleepConfig.maxTimeoutMS));
|
|
2137
2734
|
}
|
|
@@ -2139,31 +2736,28 @@ class SystemDatabase {
|
|
|
2139
2736
|
}
|
|
2140
2737
|
// ==================== Messaging ====================
|
|
2141
2738
|
nullTopic = '__null__topic__';
|
|
2142
|
-
async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey
|
|
2143
|
-
|
|
2144
|
-
|
|
2739
|
+
async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey,
|
|
2740
|
+
// The destination's shard key; required on a sharded system database.
|
|
2741
|
+
shardKey) {
|
|
2145
2742
|
const startTime = Date.now();
|
|
2146
2743
|
let missing;
|
|
2147
2744
|
const client = await this.connect();
|
|
2148
2745
|
try {
|
|
2149
2746
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
2150
2747
|
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
|
|
2151
|
-
await
|
|
2152
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2153
|
-
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
2748
|
+
await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2154
2749
|
return undefined;
|
|
2155
2750
|
});
|
|
2156
2751
|
await client.query('COMMIT');
|
|
2157
2752
|
}
|
|
2158
2753
|
catch (error) {
|
|
2159
2754
|
await client.query('ROLLBACK');
|
|
2160
|
-
|
|
2161
|
-
if (err.code === '23503') {
|
|
2755
|
+
if (error instanceof error_1.DBOSNonExistentWorkflowError) {
|
|
2162
2756
|
// Foreign key constraint violation (only expected for the INSERT query)
|
|
2163
|
-
missing =
|
|
2757
|
+
missing = error;
|
|
2164
2758
|
}
|
|
2165
2759
|
else {
|
|
2166
|
-
throw
|
|
2760
|
+
throw error;
|
|
2167
2761
|
}
|
|
2168
2762
|
}
|
|
2169
2763
|
finally {
|
|
@@ -2176,22 +2770,22 @@ class SystemDatabase {
|
|
|
2176
2770
|
}
|
|
2177
2771
|
}
|
|
2178
2772
|
/** Runs on `client` if given, joining its transaction; otherwise on the pool with retries. */
|
|
2179
|
-
async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client) {
|
|
2773
|
+
async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client, shardKey) {
|
|
2180
2774
|
if (client !== undefined) {
|
|
2181
|
-
return await this
|
|
2775
|
+
return await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2182
2776
|
}
|
|
2183
|
-
return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey);
|
|
2777
|
+
return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2184
2778
|
}
|
|
2185
2779
|
/** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
|
|
2186
|
-
async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
|
|
2780
|
+
async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2187
2781
|
const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
|
|
2188
2782
|
const client = await this.connect();
|
|
2189
2783
|
try {
|
|
2190
|
-
await this
|
|
2191
|
-
await this
|
|
2784
|
+
await this.inTransaction(client, async () => {
|
|
2785
|
+
await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2192
2786
|
// After the insert, in the order the recorded send locks, so they cannot deadlock.
|
|
2193
2787
|
if (ownerXid !== undefined) {
|
|
2194
|
-
await this
|
|
2788
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
2195
2789
|
}
|
|
2196
2790
|
});
|
|
2197
2791
|
}
|
|
@@ -2199,17 +2793,33 @@ class SystemDatabase {
|
|
|
2199
2793
|
client.release();
|
|
2200
2794
|
}
|
|
2201
2795
|
}
|
|
2202
|
-
async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey) {
|
|
2203
|
-
return await this
|
|
2796
|
+
async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2797
|
+
return await this.sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2204
2798
|
}
|
|
2205
|
-
async
|
|
2799
|
+
async sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2800
|
+
if (db === this.pool) {
|
|
2801
|
+
// All sends need a transaction now because they wake up suspended workflows unconditionally.
|
|
2802
|
+
const client = await this.connect();
|
|
2803
|
+
try {
|
|
2804
|
+
return await this.inTransaction(client, () => this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey));
|
|
2805
|
+
}
|
|
2806
|
+
finally {
|
|
2807
|
+
client.release();
|
|
2808
|
+
}
|
|
2809
|
+
}
|
|
2206
2810
|
topic = topic ?? this.nullTopic;
|
|
2207
|
-
//
|
|
2811
|
+
// An idempotency key is scoped per destination: suffix it with the recipient's workflow ID.
|
|
2208
2812
|
const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
|
|
2813
|
+
const values = {
|
|
2814
|
+
...this.wfValues(destinationID, shardKey),
|
|
2815
|
+
destination_uuid: destinationID,
|
|
2816
|
+
topic,
|
|
2817
|
+
message,
|
|
2818
|
+
serialization,
|
|
2819
|
+
message_uuid: messageUUID,
|
|
2820
|
+
};
|
|
2209
2821
|
try {
|
|
2210
|
-
await
|
|
2211
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2212
|
-
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
2822
|
+
await this.insertRows(db, 'notifications', Object.keys(values), [Object.values(values)], `ON CONFLICT (${this.keyConflict('message_uuid')}) DO NOTHING`);
|
|
2213
2823
|
}
|
|
2214
2824
|
catch (error) {
|
|
2215
2825
|
const err = error;
|
|
@@ -2218,6 +2828,23 @@ class SystemDatabase {
|
|
|
2218
2828
|
}
|
|
2219
2829
|
throw err;
|
|
2220
2830
|
}
|
|
2831
|
+
await this.wakeReceivers(db, [destinationID], [topic]);
|
|
2832
|
+
}
|
|
2833
|
+
/**
|
|
2834
|
+
* Make SUSPENDED receivers waiting on one of `topics` due, for the background pass to re-enqueue.
|
|
2835
|
+
* Runs on every send, whether or not this process uses suspend mode, so any sender wakes a workflow
|
|
2836
|
+
* another process parked. A primary-key probe per destination; it matches nothing in the common case.
|
|
2837
|
+
*
|
|
2838
|
+
* The match is destinations x topics, so a batch sending A on topic X and B on topic Y also wakes A
|
|
2839
|
+
* if it waits on Y. The spurious wake-up is harmless: the replayed recv finds no message and parks again.
|
|
2840
|
+
*/
|
|
2841
|
+
async wakeReceivers(db, destinations, topics) {
|
|
2842
|
+
const params = [topics];
|
|
2843
|
+
await db.query(
|
|
2844
|
+
// Status is tested in SET, not WHERE, so the planner cannot scan idx_workflow_status_suspended.
|
|
2845
|
+
`UPDATE "${this.schemaName}".workflow_status
|
|
2846
|
+
SET delay_until_epoch_ms = CASE WHEN status = '${workflow_1.StatusString.SUSPENDED}' THEN 0 ELSE delay_until_epoch_ms END
|
|
2847
|
+
WHERE ${this.wfIn(undefined, destinations, params)} AND suspend_key = ANY($1::text[])`, params);
|
|
2221
2848
|
}
|
|
2222
2849
|
async recv(workflowID, functionID, timeoutFunctionID, topic, timeoutSeconds = dbos_executor_1.DBOSExecutor.defaultNotificationTimeoutSec, pollingIntervalMs) {
|
|
2223
2850
|
topic = topic ?? this.nullTopic;
|
|
@@ -2233,10 +2860,10 @@ class SystemDatabase {
|
|
|
2233
2860
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2234
2861
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2235
2862
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
|
|
2236
|
-
// Record the durable timeout deadline once before polling.
|
|
2863
|
+
// Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
|
|
2237
2864
|
// first call and reads back the same value on recovery, so it never changes across iterations.
|
|
2238
2865
|
if (timeoutms) {
|
|
2239
|
-
finishTime = await this
|
|
2866
|
+
finishTime = await this.durableSleepInternal(workflowID, timeoutFunctionID, timeoutms);
|
|
2240
2867
|
}
|
|
2241
2868
|
while (true) {
|
|
2242
2869
|
// register the key with the global notifications listener.
|
|
@@ -2244,12 +2871,14 @@ class SystemDatabase {
|
|
|
2244
2871
|
const messagePromise = new Promise((resolve) => {
|
|
2245
2872
|
resolveNotification = resolve;
|
|
2246
2873
|
});
|
|
2247
|
-
const payload =
|
|
2874
|
+
const payload = this.waiter(workflowID, topic);
|
|
2248
2875
|
const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
|
|
2249
2876
|
try {
|
|
2250
2877
|
await this.checkIfCanceledLimited(workflowID);
|
|
2251
2878
|
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
2252
|
-
const
|
|
2879
|
+
const params = [topic];
|
|
2880
|
+
const target = this.wf(undefined, workflowID, params, 'destination_uuid');
|
|
2881
|
+
const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE ${target} AND topic=$1 AND consumed = false;`, params))).rows;
|
|
2253
2882
|
if (initRecvRows.length !== 0)
|
|
2254
2883
|
break;
|
|
2255
2884
|
const ct = Date.now();
|
|
@@ -2269,28 +2898,46 @@ class SystemDatabase {
|
|
|
2269
2898
|
this.notificationsMap.deregisterCallback(cbr);
|
|
2270
2899
|
}
|
|
2271
2900
|
}
|
|
2272
|
-
await this.
|
|
2901
|
+
return await this.recvConsume(workflowID, functionID, topic, startTime);
|
|
2902
|
+
}
|
|
2903
|
+
/**
|
|
2904
|
+
* Consume and return the oldest message on `topic`, or null if there is none, recording it as the recv at `functionID`. Idempotent on `functionID`.
|
|
2905
|
+
*/
|
|
2906
|
+
async recvConsume(workflowID, functionID, topic, startTime) {
|
|
2907
|
+
topic = topic ?? this.nullTopic;
|
|
2908
|
+
const res = await this.getOperationResultAndThrowIfCancelled(workflowID, functionID);
|
|
2909
|
+
if (res) {
|
|
2910
|
+
if (res.functionName !== exports.DBOS_FUNCNAME_RECV) {
|
|
2911
|
+
throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_RECV, res.functionName);
|
|
2912
|
+
}
|
|
2913
|
+
const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
|
|
2914
|
+
if (ownerXid !== undefined && (await this.getWorkflowOwner(workflowID)) !== ownerXid) {
|
|
2915
|
+
throw new error_1.DBOSWorkflowConflictError(workflowID);
|
|
2916
|
+
}
|
|
2917
|
+
return { serializedValue: res.output, serialization: res.serialization ?? null };
|
|
2918
|
+
}
|
|
2273
2919
|
// Transactionally consume and return the message if it's in the DB, otherwise return null.
|
|
2274
2920
|
let message = null;
|
|
2275
2921
|
let serialization = null;
|
|
2276
2922
|
const client = await this.connect();
|
|
2277
2923
|
try {
|
|
2278
2924
|
await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
|
|
2925
|
+
const consumeParams = [topic, functionID];
|
|
2279
2926
|
const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
|
|
2280
|
-
SET consumed = true, consumed_by_function_id = $
|
|
2281
|
-
WHERE
|
|
2282
|
-
AND topic = $
|
|
2927
|
+
SET consumed = true, consumed_by_function_id = $2
|
|
2928
|
+
WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
|
|
2929
|
+
AND topic = $1
|
|
2283
2930
|
AND consumed = false
|
|
2284
2931
|
AND message_uuid = (
|
|
2285
2932
|
SELECT message_uuid
|
|
2286
2933
|
FROM "${this.schemaName}".notifications
|
|
2287
|
-
WHERE
|
|
2288
|
-
AND topic = $
|
|
2934
|
+
WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
|
|
2935
|
+
AND topic = $1
|
|
2289
2936
|
AND consumed = false
|
|
2290
2937
|
ORDER BY created_at_epoch_ms ASC
|
|
2291
2938
|
LIMIT 1
|
|
2292
2939
|
)
|
|
2293
|
-
RETURNING notifications.message, notifications.serialization;`,
|
|
2940
|
+
RETURNING notifications.message, notifications.serialization;`, consumeParams)).rows;
|
|
2294
2941
|
if (finalRecvRows.length > 0) {
|
|
2295
2942
|
message = finalRecvRows[0].message;
|
|
2296
2943
|
serialization = finalRecvRows[0].serialization;
|
|
@@ -2319,16 +2966,19 @@ class SystemDatabase {
|
|
|
2319
2966
|
// Only a real write (not a replay) should wake readers.
|
|
2320
2967
|
let didWrite = false;
|
|
2321
2968
|
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SETEVENT, workflowID, functionID, async () => {
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2969
|
+
const keys = this.wfValues(workflowID);
|
|
2970
|
+
const keyColumns = Object.keys(keys).map((c) => `${c}, `);
|
|
2971
|
+
const keyPlaceholders = (from) => Object.keys(keys).map((_, i) => `$${from + i}, `);
|
|
2972
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_events (${keyColumns.join('')}workflow_uuid, key, value, serialization)
|
|
2973
|
+
VALUES (${keyPlaceholders(5).join('')}$1, $2, $3, $4)
|
|
2974
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'key')})
|
|
2325
2975
|
DO UPDATE SET value = $3, serialization = $4
|
|
2326
|
-
RETURNING workflow_uuid;`, [workflowID, key, message, serialization]);
|
|
2976
|
+
RETURNING workflow_uuid;`, [workflowID, key, message, serialization, ...Object.values(keys)]);
|
|
2327
2977
|
// Also write to the immutable history table for fork support
|
|
2328
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (workflow_uuid, function_id, key, value, serialization)
|
|
2329
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2330
|
-
ON CONFLICT (workflow_uuid, function_id, key)
|
|
2331
|
-
DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization]);
|
|
2978
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (${keyColumns.join('')}workflow_uuid, function_id, key, value, serialization)
|
|
2979
|
+
VALUES (${keyPlaceholders(6).join('')}$1, $2, $3, $4, $5)
|
|
2980
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id', 'key')})
|
|
2981
|
+
DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization, ...Object.values(keys)]);
|
|
2332
2982
|
didWrite = true;
|
|
2333
2983
|
return undefined;
|
|
2334
2984
|
});
|
|
@@ -2362,15 +3012,15 @@ class SystemDatabase {
|
|
|
2362
3012
|
// Get the return the value. if it's in the DB, otherwise return null.
|
|
2363
3013
|
let value = null;
|
|
2364
3014
|
let valueSer = null;
|
|
2365
|
-
const payloadKey =
|
|
3015
|
+
const payloadKey = this.waiter(workflowID, key);
|
|
2366
3016
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2367
3017
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2368
3018
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
|
|
2369
3019
|
// If we have a callerWorkflow, we want a durable sleep, otherwise, not. Record the durable
|
|
2370
|
-
// timeout deadline once before polling.
|
|
3020
|
+
// timeout deadline once before polling. durableSleepInternal persists it on the first call and reads
|
|
2371
3021
|
// back the same value on recovery, so it never changes across iterations.
|
|
2372
3022
|
if (callerWorkflow && timeoutms) {
|
|
2373
|
-
finishTime = await this
|
|
3023
|
+
finishTime = await this.durableSleepInternal(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms);
|
|
2374
3024
|
}
|
|
2375
3025
|
// Register the key with the global notifications listener first... we do not want to look in the DB first
|
|
2376
3026
|
// or that would cause a timing hole.
|
|
@@ -2384,9 +3034,11 @@ class SystemDatabase {
|
|
|
2384
3034
|
if (callerWorkflow?.workflowID)
|
|
2385
3035
|
await this.checkIfCanceledLimited(callerWorkflow?.workflowID);
|
|
2386
3036
|
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
3037
|
+
const params = [key];
|
|
3038
|
+
const target = this.wf(undefined, workflowID, params);
|
|
2387
3039
|
const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT key, value, serialization
|
|
2388
3040
|
FROM "${this.schemaName}".workflow_events
|
|
2389
|
-
WHERE
|
|
3041
|
+
WHERE ${target} AND key=$1;`, params))).rows;
|
|
2390
3042
|
if (initRecvRows.length > 0) {
|
|
2391
3043
|
value = initRecvRows[0].value;
|
|
2392
3044
|
valueSer = initRecvRows[0].serialization;
|
|
@@ -2425,15 +3077,12 @@ class SystemDatabase {
|
|
|
2425
3077
|
try {
|
|
2426
3078
|
while (true) {
|
|
2427
3079
|
try {
|
|
2428
|
-
await this
|
|
3080
|
+
await this.inTransaction(client, async () => {
|
|
2429
3081
|
// Derives the first unused offset inside the insert; two writers can still pick the same one.
|
|
2430
|
-
await
|
|
2431
|
-
SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
|
|
2432
|
-
FROM "${this.schemaName}".streams s
|
|
2433
|
-
WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
|
|
3082
|
+
await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
|
|
2434
3083
|
// After the insert, in the order the workflow-level writes lock, so they cannot deadlock.
|
|
2435
3084
|
if (ownerXid !== undefined) {
|
|
2436
|
-
await this
|
|
3085
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
2437
3086
|
}
|
|
2438
3087
|
});
|
|
2439
3088
|
}
|
|
@@ -2467,10 +3116,7 @@ class SystemDatabase {
|
|
|
2467
3116
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
2468
3117
|
await this.#runAndRecordResult(client, functionName, workflowID, functionID, async () => {
|
|
2469
3118
|
// Derives the first unused offset inside the insert; two writers can still pick the same one.
|
|
2470
|
-
await
|
|
2471
|
-
SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
|
|
2472
|
-
FROM "${this.schemaName}".streams s
|
|
2473
|
-
WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
|
|
3119
|
+
await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
|
|
2474
3120
|
didWrite = true;
|
|
2475
3121
|
return undefined;
|
|
2476
3122
|
});
|
|
@@ -2502,6 +3148,15 @@ class SystemDatabase {
|
|
|
2502
3148
|
client.release();
|
|
2503
3149
|
}
|
|
2504
3150
|
}
|
|
3151
|
+
/** Append a value to a stream, at the first unused offset, derived inside the insert. */
|
|
3152
|
+
async #insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization) {
|
|
3153
|
+
const params = [key, serializedValue, functionID, serialization, workflowID];
|
|
3154
|
+
const keyColumns = this.wfColumns(workflowID, params);
|
|
3155
|
+
await client.query(`INSERT INTO "${this.schemaName}".streams (${keyColumns.map(([c]) => `${c}, `).join('')}workflow_uuid, key, value, "offset", function_id, serialization)
|
|
3156
|
+
SELECT ${keyColumns.map(([, e]) => `${e}, `).join('')}$5::text, $1::text, $2::text, COALESCE(MAX(s."offset"), -1) + 1, $3::int, $4::text
|
|
3157
|
+
FROM "${this.schemaName}".streams s
|
|
3158
|
+
WHERE ${this.wf('s', workflowID, params)} AND s.key = $1`, params);
|
|
3159
|
+
}
|
|
2505
3160
|
async closeStreamFromWorkflow(workflowID, functionID, key) {
|
|
2506
3161
|
await this.writeStreamFromWorkflow(workflowID, functionID, key, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name(), exports.DBOS_FUNCNAME_CLOSESTREAM);
|
|
2507
3162
|
}
|
|
@@ -2511,13 +3166,15 @@ class SystemDatabase {
|
|
|
2511
3166
|
// Read the value at `offset` and the workflow's status in one query: status null = no such workflow, value undefined = nothing at that offset.
|
|
2512
3167
|
async readStreamValue(workflowID, key, offset) {
|
|
2513
3168
|
// LEFT JOIN so a workflow with nothing at offset still reports its status (single PK lookup); under the poll limiter, inside @dbRetry so the permit frees across backoff.
|
|
3169
|
+
const params = [key, offset];
|
|
3170
|
+
const target = this.wf('ws', workflowID, params);
|
|
2514
3171
|
const result = await this.#pollWithLimiter(() => this.pool.query(
|
|
2515
3172
|
// "offset" is a reserved word, so alias it (stream_offset) to read it back plainly.
|
|
2516
3173
|
`SELECT ws.status AS status, s.value AS value, s.serialization AS serialization, s."offset" AS stream_offset
|
|
2517
3174
|
FROM "${this.schemaName}".workflow_status ws
|
|
2518
3175
|
LEFT OUTER JOIN "${this.schemaName}".streams s
|
|
2519
|
-
ON s
|
|
2520
|
-
WHERE
|
|
3176
|
+
ON ${this.wfJoin('s', 'ws')} AND s.key = $1 AND s."offset" = $2
|
|
3177
|
+
WHERE ${target}`, params));
|
|
2521
3178
|
if (result.rows.length === 0) {
|
|
2522
3179
|
return { status: null, value: undefined };
|
|
2523
3180
|
}
|
|
@@ -2624,15 +3281,16 @@ class SystemDatabase {
|
|
|
2624
3281
|
async cancelTimedOutWorkflows(limit) {
|
|
2625
3282
|
const params = [workflow_1.StatusString.CANCELLED, Date.now(), limit];
|
|
2626
3283
|
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
2627
|
-
// Literal statuses let the planner match
|
|
3284
|
+
// Literal statuses let the planner match idx_workflow_status_deadline_v2 under any plan mode.
|
|
2628
3285
|
// SKIP LOCKED leaves a row a dequeue or a peer's sweep holds for the next sweep.
|
|
2629
3286
|
const { rows } = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2630
3287
|
SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
3288
|
+
suspend_key = NULL,
|
|
2631
3289
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
2632
3290
|
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
2633
3291
|
WHERE workflow_uuid IN (
|
|
2634
3292
|
SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
2635
|
-
WHERE status IN ('${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.DELAYED}')
|
|
3293
|
+
WHERE status IN ('${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')
|
|
2636
3294
|
AND workflow_deadline_epoch_ms IS NOT NULL
|
|
2637
3295
|
AND workflow_deadline_epoch_ms <= $2
|
|
2638
3296
|
AND ${scope}
|
|
@@ -2654,7 +3312,7 @@ class SystemDatabase {
|
|
|
2654
3312
|
// Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v3, so cost scales with the number of partitions rather than the backlog depth.
|
|
2655
3313
|
const params = [queueName, workflow_1.StatusString.ENQUEUED];
|
|
2656
3314
|
// Only partitions this application can actually dequeue from.
|
|
2657
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3315
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2658
3316
|
const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
|
|
2659
3317
|
(SELECT MIN(queue_partition_key) AS pk
|
|
2660
3318
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -2671,7 +3329,14 @@ class SystemDatabase {
|
|
|
2671
3329
|
}
|
|
2672
3330
|
async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0,
|
|
2673
3331
|
// Written to every claimed row: the dispatches of this claim own their executions.
|
|
2674
|
-
ownerXid = null) {
|
|
3332
|
+
ownerXid = null, isLatestVersion) {
|
|
3333
|
+
const { claimedIDs } = await this.claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount, partitionLocalRunningCount, ownerXid, isLatestVersion);
|
|
3334
|
+
return claimedIDs;
|
|
3335
|
+
}
|
|
3336
|
+
/** Claim this queue's next workflows, plus the limit that left no room to claim any (undefined if every limit had room). */
|
|
3337
|
+
async claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0, ownerXid = null, isLatestVersion) {
|
|
3338
|
+
// Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
|
|
3339
|
+
const startTimeMs = Date.now();
|
|
2675
3340
|
const claimedIDs = [];
|
|
2676
3341
|
const partitionParams = queuePartitionKey !== undefined ? [queuePartitionKey] : [];
|
|
2677
3342
|
// Shares a concurrency or rate limit budget with other executors.
|
|
@@ -2679,13 +3344,49 @@ class SystemDatabase {
|
|
|
2679
3344
|
queue.partitionConcurrency !== undefined ||
|
|
2680
3345
|
queue.rateLimit !== undefined ||
|
|
2681
3346
|
queue.partitionRateLimit !== undefined;
|
|
2682
|
-
// Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
|
|
3347
|
+
// Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it: lock the whole queue.
|
|
2683
3348
|
const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
|
|
3349
|
+
// Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
|
|
3350
|
+
let maxTasks = Infinity;
|
|
3351
|
+
// The first limit found with no room, which the caller reads as why nothing was claimed.
|
|
3352
|
+
let stopReason = undefined;
|
|
3353
|
+
const limit = (room, reason) => {
|
|
3354
|
+
maxTasks = Math.min(maxTasks, room);
|
|
3355
|
+
if (maxTasks <= 0)
|
|
3356
|
+
stopReason ??= reason;
|
|
3357
|
+
};
|
|
3358
|
+
// This worker's own limits need no database, so a full worker returns before taking the claim lock a peer could use.
|
|
3359
|
+
if (queue.workerConcurrency !== undefined) {
|
|
3360
|
+
// Use the in-memory registry for this worker's running count — avoids a DB round trip.
|
|
3361
|
+
limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
|
|
3362
|
+
}
|
|
3363
|
+
if (queue.partitionWorkerConcurrency !== undefined) {
|
|
3364
|
+
limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
|
|
3365
|
+
}
|
|
3366
|
+
if (maxTasks <= 0) {
|
|
3367
|
+
return { claimedIDs, stopReason };
|
|
3368
|
+
}
|
|
3369
|
+
// A sharded database keeps the version row under another key, which the claim's transaction cannot span.
|
|
3370
|
+
if (isLatestVersion === undefined && this.sharded) {
|
|
3371
|
+
isLatestVersion = await this.isLatestVersion(appVersion);
|
|
3372
|
+
}
|
|
3373
|
+
// Row locks only collide when claimers see the same queue head, so serialize claims on the budget's scope; CockroachDB has no advisory locks before 26.3.
|
|
3374
|
+
const useClaimLock = hasSharedBudget && (this.sharded || !(await this.#isCockroach()));
|
|
2684
3375
|
const client = await this.connect();
|
|
2685
3376
|
try {
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
3377
|
+
if (useClaimLock) {
|
|
3378
|
+
await client.query('BEGIN');
|
|
3379
|
+
const lockParams = [];
|
|
3380
|
+
const { rows } = await client.query(this.claimLockQuery(queue.name, hasWriteSkew ? undefined : queuePartitionKey, lockParams), lockParams);
|
|
3381
|
+
if (rows[0]?.acquired !== true) {
|
|
3382
|
+
await client.query('COMMIT');
|
|
3383
|
+
// Another worker is claiming from this scope, so it still has work; the next poll retries.
|
|
3384
|
+
return rows.length > 0 ? { claimedIDs, stopReason: 'claim_lock' } : { claimedIDs };
|
|
3385
|
+
}
|
|
3386
|
+
}
|
|
3387
|
+
else if (hasSharedBudget) {
|
|
3388
|
+
// CockroachDB, without advisory locks before 26.3: SERIALIZABLE aborts claimers that spend the same budget.
|
|
3389
|
+
await client.query('BEGIN ISOLATION LEVEL SERIALIZABLE');
|
|
2689
3390
|
}
|
|
2690
3391
|
else {
|
|
2691
3392
|
await client.query('BEGIN');
|
|
@@ -2694,7 +3395,7 @@ class SystemDatabase {
|
|
|
2694
3395
|
const rateLimitRemaining = async (rateLimit, partitionScoped) => {
|
|
2695
3396
|
const params = [queue.name, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED, rateLimit.periodSec * 1000];
|
|
2696
3397
|
// Count only what this application would dequeue, matching the select below.
|
|
2697
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3398
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2698
3399
|
const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
|
|
2699
3400
|
const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
2700
3401
|
WHERE queue_name = $1
|
|
@@ -2713,35 +3414,26 @@ class SystemDatabase {
|
|
|
2713
3414
|
*/
|
|
2714
3415
|
const pendingCount = async (partitionScoped) => {
|
|
2715
3416
|
const params = [queue.name, workflow_1.StatusString.PENDING];
|
|
2716
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3417
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2717
3418
|
const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
|
|
2718
3419
|
const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
2719
3420
|
WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
|
|
2720
3421
|
return Number(rows[0]?.count ?? 0);
|
|
2721
3422
|
};
|
|
2722
|
-
// Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
|
|
2723
|
-
let maxTasks = Infinity;
|
|
2724
|
-
if (queue.workerConcurrency !== undefined) {
|
|
2725
|
-
// Use the in-memory registry for this worker's running count — avoids a DB round trip.
|
|
2726
|
-
maxTasks = Math.min(maxTasks, Math.max(0, queue.workerConcurrency - localRunningCount));
|
|
2727
|
-
}
|
|
2728
|
-
if (queue.partitionWorkerConcurrency !== undefined) {
|
|
2729
|
-
maxTasks = Math.min(maxTasks, Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount));
|
|
2730
|
-
}
|
|
2731
|
-
if (maxTasks <= 0) {
|
|
2732
|
-
await client.query('COMMIT');
|
|
2733
|
-
return claimedIDs;
|
|
2734
|
-
}
|
|
2735
3423
|
if (queue.rateLimit !== undefined) {
|
|
2736
3424
|
// Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
|
|
2737
|
-
|
|
3425
|
+
limit(await rateLimitRemaining(queue.rateLimit, false), 'limiter');
|
|
3426
|
+
if (maxTasks <= 0) {
|
|
3427
|
+
await client.query('COMMIT');
|
|
3428
|
+
return { claimedIDs, stopReason };
|
|
3429
|
+
}
|
|
2738
3430
|
}
|
|
2739
3431
|
if (queue.partitionRateLimit !== undefined) {
|
|
2740
|
-
|
|
3432
|
+
limit(await rateLimitRemaining(queue.partitionRateLimit, true), 'partition_limiter');
|
|
2741
3433
|
}
|
|
2742
3434
|
if (maxTasks <= 0) {
|
|
2743
3435
|
await client.query('COMMIT');
|
|
2744
|
-
return claimedIDs;
|
|
3436
|
+
return { claimedIDs, stopReason };
|
|
2745
3437
|
}
|
|
2746
3438
|
if (queue.concurrency !== undefined) {
|
|
2747
3439
|
// Global concurrency still requires a DB query since other workers may be running workflows too.
|
|
@@ -2749,23 +3441,26 @@ class SystemDatabase {
|
|
|
2749
3441
|
if (totalRunningTasks > queue.concurrency) {
|
|
2750
3442
|
this.logger.warn(`Total running tasks (${totalRunningTasks}) exceeds the global concurrency limit (${queue.concurrency})`);
|
|
2751
3443
|
}
|
|
2752
|
-
|
|
3444
|
+
limit(Math.max(0, queue.concurrency - totalRunningTasks), 'concurrency');
|
|
3445
|
+
if (maxTasks <= 0) {
|
|
3446
|
+
await client.query('COMMIT');
|
|
3447
|
+
return { claimedIDs, stopReason };
|
|
3448
|
+
}
|
|
2753
3449
|
}
|
|
2754
3450
|
if (queue.partitionConcurrency !== undefined) {
|
|
2755
3451
|
const partitionRunningTasks = await pendingCount(true);
|
|
2756
3452
|
if (partitionRunningTasks > queue.partitionConcurrency) {
|
|
2757
3453
|
this.logger.warn(`Total running tasks (${partitionRunningTasks}) on partition ${queuePartitionKey} of queue ${queue.name} exceeds the partition concurrency limit (${queue.partitionConcurrency})`);
|
|
2758
3454
|
}
|
|
2759
|
-
|
|
3455
|
+
limit(Math.max(0, queue.partitionConcurrency - partitionRunningTasks), 'partition_concurrency');
|
|
2760
3456
|
}
|
|
2761
3457
|
// Return immediately if there are no available tasks due to flow control limits
|
|
2762
3458
|
if (maxTasks <= 0) {
|
|
2763
3459
|
await client.query('COMMIT');
|
|
2764
|
-
return claimedIDs;
|
|
3460
|
+
return { claimedIDs, stopReason };
|
|
2765
3461
|
}
|
|
2766
3462
|
// Retrieve the first max_tasks workflows in the queue.
|
|
2767
|
-
|
|
2768
|
-
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
3463
|
+
isLatestVersion ??= await this.isLatestVersion(appVersion, client);
|
|
2769
3464
|
const versionClause = isLatestVersion
|
|
2770
3465
|
? '(application_version = $3 OR application_version IS NULL)'
|
|
2771
3466
|
: 'application_version = $3';
|
|
@@ -2775,13 +3470,17 @@ class SystemDatabase {
|
|
|
2775
3470
|
const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
|
|
2776
3471
|
const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
|
|
2777
3472
|
const selectScope = this.appNameFilter('application_name', this.appName, selectParams);
|
|
3473
|
+
const keyScope = this.scoped(undefined, selectParams);
|
|
3474
|
+
const claimGate = this.claimGate(queue.name, queuePartitionKey, selectParams);
|
|
2778
3475
|
const selectQuery = `
|
|
2779
3476
|
SELECT workflow_uuid
|
|
2780
3477
|
FROM "${this.schemaName}".workflow_status
|
|
2781
|
-
WHERE
|
|
3478
|
+
WHERE ${keyScope}
|
|
3479
|
+
AND status = $1
|
|
2782
3480
|
AND queue_name = $2
|
|
2783
3481
|
AND ${versionClause}
|
|
2784
3482
|
AND ${selectScope}
|
|
3483
|
+
AND ${claimGate}
|
|
2785
3484
|
${queuePartitionKey !== undefined ? 'AND queue_partition_key = $4' : ''}
|
|
2786
3485
|
ORDER BY priority ASC, created_at ASC
|
|
2787
3486
|
${limitClause}
|
|
@@ -2800,35 +3499,36 @@ class SystemDatabase {
|
|
|
2800
3499
|
executorID,
|
|
2801
3500
|
appVersion,
|
|
2802
3501
|
queue.rateLimit !== undefined || queue.partitionRateLimit !== undefined,
|
|
2803
|
-
workflowIDs,
|
|
2804
3502
|
workflow_1.StatusString.ENQUEUED,
|
|
2805
3503
|
// Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
|
|
2806
3504
|
this.appName ?? null,
|
|
2807
3505
|
ownerXid,
|
|
3506
|
+
startTimeMs,
|
|
2808
3507
|
];
|
|
3508
|
+
const claimedIDsFilter = this.wfIn(undefined, workflowIDs, updateParams);
|
|
2809
3509
|
// Re-check ownership alongside status, as the partitioned claim guard does.
|
|
2810
3510
|
const claimScope = this.appNameFilter('application_name', this.appName, updateParams);
|
|
2811
3511
|
// RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
|
|
2812
3512
|
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2813
3513
|
SET status = $1,
|
|
2814
|
-
owner_xid = $
|
|
3514
|
+
owner_xid = $7,
|
|
2815
3515
|
executor_id = $2,
|
|
2816
3516
|
application_version = $3,
|
|
2817
3517
|
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2818
3518
|
rate_limited = $4,
|
|
2819
|
-
application_name = COALESCE(application_name, $
|
|
3519
|
+
application_name = COALESCE(application_name, $6),
|
|
2820
3520
|
recovery_attempts = recovery_attempts + 1,
|
|
2821
3521
|
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2822
3522
|
workflow_deadline_epoch_ms = CASE
|
|
2823
3523
|
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
2824
3524
|
CASE
|
|
2825
|
-
WHEN workflow_timeout_ms > 9223372036854775807 -
|
|
3525
|
+
WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
|
|
2826
3526
|
THEN 9223372036854775807
|
|
2827
|
-
ELSE
|
|
3527
|
+
ELSE $8::bigint + workflow_timeout_ms
|
|
2828
3528
|
END
|
|
2829
3529
|
ELSE workflow_deadline_epoch_ms
|
|
2830
3530
|
END
|
|
2831
|
-
WHERE
|
|
3531
|
+
WHERE ${claimedIDsFilter} AND status = $5 AND ${claimScope}
|
|
2832
3532
|
RETURNING workflow_uuid`, updateParams);
|
|
2833
3533
|
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
2834
3534
|
for (const id of workflowIDs) {
|
|
@@ -2846,14 +3546,18 @@ class SystemDatabase {
|
|
|
2846
3546
|
client.release();
|
|
2847
3547
|
}
|
|
2848
3548
|
// Return the IDs of all functions we marked started
|
|
2849
|
-
return claimedIDs;
|
|
3549
|
+
return { claimedIDs };
|
|
2850
3550
|
}
|
|
2851
3551
|
/** Max heads admitted per sweep: bounds dispatch, not the partition walk; an unconstrained sweep takes the lowest keys, so higher keys can wait under sustained load. */
|
|
2852
3552
|
partitionedDequeueSweepCap = 8192;
|
|
2853
|
-
/**
|
|
3553
|
+
/** Max partitions a sweep claims per transaction, which bounds the locks it holds at once and for how long. */
|
|
3554
|
+
partitionedDequeueBatchSize = 64;
|
|
3555
|
+
/** Dequeue each partition's head-of-line workflow, at most {@link partitionedDequeueSweepCap} per sweep in transactions of {@link partitionedDequeueBatchSize}; only valid for partition-concurrency-1 queues with no queue-wide limit. */
|
|
2854
3556
|
async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity,
|
|
2855
3557
|
// Written to every claimed row: the dispatches of this claim own their executions.
|
|
2856
|
-
ownerXid = null) {
|
|
3558
|
+
ownerXid = null, isLatestVersion) {
|
|
3559
|
+
// Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
|
|
3560
|
+
const startTimeMs = Date.now();
|
|
2857
3561
|
if (queue.partitionConcurrency !== 1 ||
|
|
2858
3562
|
queue.concurrency !== undefined ||
|
|
2859
3563
|
queue.rateLimit !== undefined ||
|
|
@@ -2861,14 +3565,35 @@ class SystemDatabase {
|
|
|
2861
3565
|
throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
|
|
2862
3566
|
}
|
|
2863
3567
|
// partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
|
|
3568
|
+
// A sharded database keeps the version row under another key, which the claim's transaction cannot span.
|
|
3569
|
+
if (isLatestVersion === undefined && this.sharded) {
|
|
3570
|
+
isLatestVersion = await this.isLatestVersion(appVersion);
|
|
3571
|
+
}
|
|
3572
|
+
// Sweeps whose heads differ lock disjoint rows, so a claim also takes its partition's lock; CockroachDB, without advisory locks before 26.3, relies on its SERIALIZABLE default.
|
|
3573
|
+
const usePartitionLocks = this.sharded || !(await this.#isCockroach());
|
|
2864
3574
|
const client = await this.connect();
|
|
2865
3575
|
try {
|
|
2866
|
-
await
|
|
2867
|
-
const latestVersion = await this.#latestApplicationVersionName(client);
|
|
2868
|
-
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
3576
|
+
isLatestVersion ??= await this.isLatestVersion(appVersion, client);
|
|
2869
3577
|
const versionClause = (n) => isLatestVersion
|
|
2870
3578
|
? `(application_version = $${n} OR application_version IS NULL)`
|
|
2871
3579
|
: `application_version = $${n}`;
|
|
3580
|
+
// The partition's head: $1 queue, $2 ENQUEUED, `version` the app version, `pk` the partition key expression.
|
|
3581
|
+
const headOf = (pk, version, scope) => `SELECT workflow_uuid
|
|
3582
|
+
FROM "${this.schemaName}".workflow_status
|
|
3583
|
+
WHERE queue_name = $1 AND status = $2
|
|
3584
|
+
AND queue_partition_key = ${pk}
|
|
3585
|
+
AND ${versionClause(version)}
|
|
3586
|
+
AND ${scope}
|
|
3587
|
+
-- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
|
|
3588
|
+
ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
|
|
3589
|
+
LIMIT 1`;
|
|
3590
|
+
// No application filter by design: a mutual-exclusion probe must block on any owner's row.
|
|
3591
|
+
const noPending = (pk, pending, scope) => `NOT EXISTS (
|
|
3592
|
+
SELECT 1
|
|
3593
|
+
FROM "${this.schemaName}".workflow_status
|
|
3594
|
+
WHERE ${scope} AND queue_name = $1 AND status = $${pending}
|
|
3595
|
+
AND queue_partition_key IS NOT NULL AND queue_partition_key = ${pk}
|
|
3596
|
+
)`;
|
|
2872
3597
|
// This worker's own budget bounds the sweep alongside the cap.
|
|
2873
3598
|
const sweepLimit = Math.min(this.partitionedDequeueSweepCap, maxTasks);
|
|
2874
3599
|
// When the worker's budget is the binding constraint, probe partitions in random order to prevent starvation.
|
|
@@ -2880,8 +3605,10 @@ class SystemDatabase {
|
|
|
2880
3605
|
workflow_1.StatusString.PENDING,
|
|
2881
3606
|
sweepLimit,
|
|
2882
3607
|
];
|
|
2883
|
-
const candidateScope = this.appNameFilter('application_name', this.appName, candidateParams)
|
|
3608
|
+
const candidateScope = `${this.scoped(undefined, candidateParams)} AND ${this.appNameFilter('application_name', this.appName, candidateParams)}`;
|
|
3609
|
+
const probeScope = this.scoped(undefined, candidateParams);
|
|
2884
3610
|
// Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
|
|
3611
|
+
// The walk takes no lock: each batch below picks its heads again under its partitions' locks.
|
|
2885
3612
|
const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
|
|
2886
3613
|
(SELECT MIN(queue_partition_key) AS pk
|
|
2887
3614
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -2897,118 +3624,140 @@ class SystemDatabase {
|
|
|
2897
3624
|
SELECT partitions.pk
|
|
2898
3625
|
FROM partitions
|
|
2899
3626
|
WHERE partitions.pk IS NOT NULL
|
|
2900
|
-
|
|
2901
|
-
AND NOT EXISTS (
|
|
2902
|
-
SELECT 1
|
|
2903
|
-
FROM "${this.schemaName}".workflow_status
|
|
2904
|
-
WHERE queue_name = $1 AND status = $4
|
|
2905
|
-
AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
|
|
2906
|
-
)
|
|
3627
|
+
AND ${noPending('partitions.pk', 4, probeScope)}
|
|
2907
3628
|
ORDER BY ${sweepOrder}
|
|
2908
3629
|
LIMIT $5
|
|
2909
3630
|
)
|
|
2910
|
-
SELECT
|
|
3631
|
+
SELECT chosen.pk
|
|
2911
3632
|
FROM chosen
|
|
2912
|
-
--
|
|
2913
|
-
JOIN LATERAL (
|
|
2914
|
-
|
|
2915
|
-
FROM "${this.schemaName}".workflow_status
|
|
2916
|
-
WHERE queue_name = $1 AND status = $2
|
|
2917
|
-
AND queue_partition_key = chosen.pk
|
|
2918
|
-
AND ${versionClause(3)}
|
|
2919
|
-
AND ${candidateScope}
|
|
2920
|
-
-- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
|
|
2921
|
-
ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
|
|
2922
|
-
LIMIT 1
|
|
2923
|
-
) head ON TRUE
|
|
2924
|
-
-- Which partitions were chosen is settled above; order the result so the claim, and the dispatch it feeds, are deterministic.
|
|
3633
|
+
-- Only partitions with a head this worker may run; LATERAL plans as a tight nested loop.
|
|
3634
|
+
JOIN LATERAL (${headOf('chosen.pk', 3, candidateScope)}) head ON TRUE
|
|
3635
|
+
-- Which partitions were chosen is settled above; order them so the claim, and the dispatch it feeds, are deterministic.
|
|
2925
3636
|
ORDER BY chosen.pk ASC`, candidateParams);
|
|
2926
|
-
const
|
|
2927
|
-
if (candidateIDs.length === 0) {
|
|
2928
|
-
await client.query('COMMIT');
|
|
2929
|
-
return [];
|
|
2930
|
-
}
|
|
2931
|
-
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
|
|
3637
|
+
const partitionKeys = candidateResult.rows.map((row) => row.pk);
|
|
2932
3638
|
// Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
|
|
2933
|
-
const claimGuard = (ids, status, name, version, scope) =>
|
|
3639
|
+
const claimGuard = (ids, status, name, version, scope) => `${ids}
|
|
2934
3640
|
AND status = $${status}
|
|
2935
3641
|
AND queue_name = $${name}
|
|
2936
3642
|
AND queue_partition_key IS NOT NULL
|
|
2937
3643
|
AND ${versionClause(version)}
|
|
2938
3644
|
AND ${scope}`;
|
|
2939
|
-
|
|
2940
|
-
const
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
3645
|
+
/** Claim the heads of these partitions inside the open transaction, returning the IDs it marked PENDING. */
|
|
3646
|
+
const claimBatch = async (batch) => {
|
|
3647
|
+
let heldPks = batch;
|
|
3648
|
+
if (usePartitionLocks) {
|
|
3649
|
+
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_BEFORE_LOCKS);
|
|
3650
|
+
const lockParams = [];
|
|
3651
|
+
const { rows } = await client.query(this.partitionLockQuery(queue.name, batch, lockParams), lockParams);
|
|
3652
|
+
// Partitions another sweep holds are left to it, or to the next poll.
|
|
3653
|
+
heldPks = rows[0]?.held ?? [];
|
|
3654
|
+
if (heldPks.length === 0)
|
|
3655
|
+
return [];
|
|
3656
|
+
}
|
|
3657
|
+
// Pick the heads again under the locks, so a previous holder's claims and any newly visible head are seen.
|
|
3658
|
+
const headParams = [queue.name, workflow_1.StatusString.ENQUEUED, appVersion, workflow_1.StatusString.PENDING];
|
|
3659
|
+
const headScope = `${this.scoped(undefined, headParams)} AND ${this.appNameFilter('application_name', this.appName, headParams)}`;
|
|
3660
|
+
const headProbeScope = this.scoped(undefined, headParams);
|
|
3661
|
+
headParams.push(heldPks);
|
|
3662
|
+
const headResult = await client.query(`SELECT head.workflow_uuid
|
|
3663
|
+
FROM unnest($${headParams.length}::text[]) AS chosen(pk)
|
|
3664
|
+
JOIN LATERAL (${headOf('chosen.pk', 3, headScope)}) head ON TRUE
|
|
3665
|
+
WHERE ${noPending('chosen.pk', 4, headProbeScope)}
|
|
3666
|
+
ORDER BY chosen.pk ASC`, headParams);
|
|
3667
|
+
const candidateIDs = headResult.rows.map((row) => row.workflow_uuid);
|
|
3668
|
+
if (candidateIDs.length === 0)
|
|
3669
|
+
return [];
|
|
3670
|
+
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
|
|
3671
|
+
const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
|
|
3672
|
+
const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
|
|
3673
|
+
const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
|
|
3674
|
+
// Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
|
|
3675
|
+
const lockedResult = await client.query(`SELECT workflow_uuid
|
|
3676
|
+
FROM "${this.schemaName}".workflow_status
|
|
3677
|
+
WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
|
|
3678
|
+
FOR UPDATE SKIP LOCKED`, lockedParams);
|
|
3679
|
+
const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
|
|
3680
|
+
// Preserve partition order for submission.
|
|
3681
|
+
const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
|
|
3682
|
+
if (claimIDs.length === 0)
|
|
3683
|
+
return [];
|
|
3684
|
+
const flipParams = [
|
|
3685
|
+
workflow_1.StatusString.PENDING,
|
|
3686
|
+
executorID,
|
|
3687
|
+
workflow_1.StatusString.ENQUEUED,
|
|
3688
|
+
queue.name,
|
|
3689
|
+
appVersion,
|
|
3690
|
+
// Claim the row, as the unpartitioned dequeue does.
|
|
3691
|
+
this.appName ?? null,
|
|
3692
|
+
ownerXid,
|
|
3693
|
+
startTimeMs,
|
|
3694
|
+
];
|
|
3695
|
+
const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
|
|
3696
|
+
const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
|
|
3697
|
+
// Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
|
|
3698
|
+
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
3699
|
+
SET status = $1,
|
|
3700
|
+
owner_xid = $7,
|
|
3701
|
+
executor_id = $2,
|
|
3702
|
+
application_version = $5,
|
|
3703
|
+
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
3704
|
+
rate_limited = FALSE,
|
|
3705
|
+
application_name = COALESCE(application_name, $6),
|
|
3706
|
+
recovery_attempts = recovery_attempts + 1,
|
|
3707
|
+
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
3708
|
+
workflow_deadline_epoch_ms = CASE
|
|
3709
|
+
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
3710
|
+
CASE
|
|
3711
|
+
WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
|
|
3712
|
+
THEN 9223372036854775807
|
|
3713
|
+
ELSE $8::bigint + workflow_timeout_ms
|
|
3714
|
+
END
|
|
3715
|
+
ELSE workflow_deadline_epoch_ms
|
|
3716
|
+
END
|
|
3717
|
+
WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
|
|
3718
|
+
RETURNING workflow_uuid`, flipParams);
|
|
3719
|
+
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
3720
|
+
return claimIDs.filter((id) => flippedIDs.has(id));
|
|
3721
|
+
};
|
|
3722
|
+
// One short transaction per batch, so a sweep holds at most a batch's locks, and only while it claims that batch.
|
|
3723
|
+
const claimedIDs = [];
|
|
3724
|
+
for (let start = 0; start < partitionKeys.length; start += this.partitionedDequeueBatchSize) {
|
|
3725
|
+
const batch = partitionKeys.slice(start, start + this.partitionedDequeueBatchSize);
|
|
3726
|
+
try {
|
|
3727
|
+
await client.query('BEGIN');
|
|
3728
|
+
const batchIDs = await claimBatch(batch);
|
|
3729
|
+
await client.query('COMMIT');
|
|
3730
|
+
claimedIDs.push(...batchIDs);
|
|
3731
|
+
}
|
|
3732
|
+
catch (error) {
|
|
3733
|
+
// Swallowed: a rollback failure must not mask why we are rolling back.
|
|
3734
|
+
await client.query('ROLLBACK').catch(() => undefined);
|
|
3735
|
+
// Earlier batches committed their claims, which must still reach dispatch; the next poll retries the rest.
|
|
3736
|
+
if (claimedIDs.length === 0)
|
|
3737
|
+
throw error;
|
|
3738
|
+
this.logger.warn(`[${queue.name}] stopping a partitioned sweep after ${claimedIDs.length} claim(s): ${String(error)}`);
|
|
3739
|
+
break;
|
|
3740
|
+
}
|
|
2952
3741
|
}
|
|
2953
|
-
const flipParams = [
|
|
2954
|
-
workflow_1.StatusString.PENDING,
|
|
2955
|
-
executorID,
|
|
2956
|
-
claimIDs,
|
|
2957
|
-
workflow_1.StatusString.ENQUEUED,
|
|
2958
|
-
queue.name,
|
|
2959
|
-
appVersion,
|
|
2960
|
-
// Claim the row, as the unpartitioned dequeue does.
|
|
2961
|
-
this.appName ?? null,
|
|
2962
|
-
ownerXid,
|
|
2963
|
-
];
|
|
2964
|
-
const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
|
|
2965
|
-
// Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
|
|
2966
|
-
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2967
|
-
SET status = $1,
|
|
2968
|
-
owner_xid = $8,
|
|
2969
|
-
executor_id = $2,
|
|
2970
|
-
application_version = $6,
|
|
2971
|
-
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2972
|
-
rate_limited = FALSE,
|
|
2973
|
-
application_name = COALESCE(application_name, $7),
|
|
2974
|
-
recovery_attempts = recovery_attempts + 1,
|
|
2975
|
-
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2976
|
-
workflow_deadline_epoch_ms = CASE
|
|
2977
|
-
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
2978
|
-
CASE
|
|
2979
|
-
WHEN workflow_timeout_ms > 9223372036854775807 - (EXTRACT(epoch FROM now()) * 1000)::bigint
|
|
2980
|
-
THEN 9223372036854775807
|
|
2981
|
-
ELSE (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
|
|
2982
|
-
END
|
|
2983
|
-
ELSE workflow_deadline_epoch_ms
|
|
2984
|
-
END
|
|
2985
|
-
WHERE ${claimGuard(3, 4, 5, 6, flipScope)}
|
|
2986
|
-
RETURNING workflow_uuid`, flipParams);
|
|
2987
|
-
await client.query('COMMIT');
|
|
2988
|
-
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
2989
|
-
const claimedIDs = claimIDs.filter((id) => flippedIDs.has(id));
|
|
2990
3742
|
if (claimedIDs.length > 0) {
|
|
2991
3743
|
this.logger.debug(`[${queue.name}] dequeueing ${claimedIDs.length} task(s)`);
|
|
2992
3744
|
}
|
|
2993
3745
|
return claimedIDs;
|
|
2994
3746
|
}
|
|
2995
|
-
catch (error) {
|
|
2996
|
-
await client.query('ROLLBACK');
|
|
2997
|
-
throw error;
|
|
2998
|
-
}
|
|
2999
3747
|
finally {
|
|
3000
3748
|
client.release();
|
|
3001
3749
|
}
|
|
3002
3750
|
}
|
|
3003
3751
|
// ==================== Queries & Maintenance ====================
|
|
3004
|
-
|
|
3005
|
-
|
|
3752
|
+
/** With `keyedIDs`, the listed workflow IDs are addressed by their shard keys, as a batch read of known workflows. */
|
|
3753
|
+
async listWorkflows(input, options = {}) {
|
|
3754
|
+
return await this.#listWorkflows(input, options);
|
|
3006
3755
|
}
|
|
3007
3756
|
/**
|
|
3008
3757
|
* Unretried body of {@link listWorkflows}. An ID-keyed read runs on `client` when given, so a caller
|
|
3009
3758
|
* holding a transaction never waits on a second pool connection; the caller owns the retry.
|
|
3010
3759
|
*/
|
|
3011
|
-
async #listWorkflows(input, client) {
|
|
3760
|
+
async #listWorkflows(input, options = {}, client) {
|
|
3012
3761
|
const schemaName = this.schemaName;
|
|
3013
3762
|
const selectColumns = [
|
|
3014
3763
|
'workflow_uuid',
|
|
@@ -3050,16 +3799,23 @@ class SystemDatabase {
|
|
|
3050
3799
|
const payloadJoins = [];
|
|
3051
3800
|
if (input.loadInput) {
|
|
3052
3801
|
payloadColumns.push('COALESCE(wi.inputs, workflow_status.inputs) AS inputs');
|
|
3053
|
-
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON wi
|
|
3802
|
+
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON ${this.wfJoin('wi', 'workflow_status')}`);
|
|
3054
3803
|
}
|
|
3055
3804
|
if (input.loadOutput) {
|
|
3056
3805
|
payloadColumns.push('COALESCE(wo.output, workflow_status.output) AS output');
|
|
3057
3806
|
payloadColumns.push('COALESCE(wo.error, workflow_status.error) AS error');
|
|
3058
|
-
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON wo
|
|
3807
|
+
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON ${this.wfJoin('wo', 'workflow_status')}`);
|
|
3059
3808
|
}
|
|
3060
3809
|
if (input.loadInput || input.loadOutput) {
|
|
3061
3810
|
selectColumns.push('serialization');
|
|
3062
3811
|
}
|
|
3812
|
+
const keyColumns = [];
|
|
3813
|
+
const keyColumn = this.keyColumn('workflow_status');
|
|
3814
|
+
if (keyColumn !== undefined)
|
|
3815
|
+
keyColumns.push(`${keyColumn} AS shard_key`);
|
|
3816
|
+
const parentKeyColumn = this.keyColumn('workflow_status', 'parent_shard_key');
|
|
3817
|
+
if (parentKeyColumn !== undefined)
|
|
3818
|
+
keyColumns.push(`${parentKeyColumn} AS parent_shard_key`);
|
|
3063
3819
|
input.sortDesc = input.sortDesc ?? false; // By default, sort in ascending order
|
|
3064
3820
|
// Build WHERE clauses
|
|
3065
3821
|
const whereClauses = [];
|
|
@@ -3085,9 +3841,9 @@ class SystemDatabase {
|
|
|
3085
3841
|
// If queuesOnly, filter for queued workflows
|
|
3086
3842
|
if (input.queuesOnly) {
|
|
3087
3843
|
whereClauses.push(`queue_name IS NOT NULL`);
|
|
3088
|
-
whereClauses.push(`status IN ($${paramCounter}, $${paramCounter + 1}, $${paramCounter + 2})`);
|
|
3089
|
-
params.push(workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING, workflow_1.StatusString.DELAYED);
|
|
3090
|
-
paramCounter +=
|
|
3844
|
+
whereClauses.push(`status IN ($${paramCounter}, $${paramCounter + 1}, $${paramCounter + 2}, $${paramCounter + 3})`);
|
|
3845
|
+
params.push(workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING, workflow_1.StatusString.DELAYED, workflow_1.StatusString.SUSPENDED);
|
|
3846
|
+
paramCounter += 4;
|
|
3091
3847
|
}
|
|
3092
3848
|
addFilter('name', input.workflowName);
|
|
3093
3849
|
addFilter('queue_name', input.queueName);
|
|
@@ -3102,6 +3858,18 @@ class SystemDatabase {
|
|
|
3102
3858
|
whereClauses.push(idKeyed
|
|
3103
3859
|
? this.appNameFilter('application_name', input.applicationName, params)
|
|
3104
3860
|
: this.observabilityFilter('application_name', input.applicationName, params));
|
|
3861
|
+
// An empty key list matches nothing, rather than every key.
|
|
3862
|
+
if (input.shardKey !== undefined && input.shardKey !== null) {
|
|
3863
|
+
const keys = Array.isArray(input.shardKey) ? input.shardKey : [input.shardKey];
|
|
3864
|
+
whereClauses.push(this.keyIn('workflow_status', keys, params));
|
|
3865
|
+
}
|
|
3866
|
+
if (input.parentShardKey !== undefined && input.parentShardKey !== null) {
|
|
3867
|
+
const keys = Array.isArray(input.parentShardKey) ? input.parentShardKey : [input.parentShardKey];
|
|
3868
|
+
whereClauses.push(this.keyIn('workflow_status', keys, params, 'parent_shard_key'));
|
|
3869
|
+
}
|
|
3870
|
+
if (options.keyedIDs && input.workflowIDs) {
|
|
3871
|
+
whereClauses.push(this.wfIn('workflow_status', input.workflowIDs, params));
|
|
3872
|
+
}
|
|
3105
3873
|
paramCounter = params.length + 1;
|
|
3106
3874
|
if (input.workflow_id_prefix) {
|
|
3107
3875
|
if (Array.isArray(input.workflow_id_prefix)) {
|
|
@@ -3116,7 +3884,7 @@ class SystemDatabase {
|
|
|
3116
3884
|
paramCounter++;
|
|
3117
3885
|
}
|
|
3118
3886
|
}
|
|
3119
|
-
if (input.workflowIDs) {
|
|
3887
|
+
if (input.workflowIDs && !options.keyedIDs) {
|
|
3120
3888
|
const placeholders = input.workflowIDs.map((_, i) => `$${paramCounter + i}`).join(', ');
|
|
3121
3889
|
whereClauses.push(`workflow_status.workflow_uuid IN (${placeholders})`);
|
|
3122
3890
|
params.push(...input.workflowIDs);
|
|
@@ -3198,7 +3966,7 @@ class SystemDatabase {
|
|
|
3198
3966
|
params.push(input.offset);
|
|
3199
3967
|
paramCounter++;
|
|
3200
3968
|
}
|
|
3201
|
-
const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns].join(', ');
|
|
3969
|
+
const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns, ...keyColumns].join(', ');
|
|
3202
3970
|
const query = `
|
|
3203
3971
|
SELECT ${projection}
|
|
3204
3972
|
FROM "${schemaName}".workflow_status
|
|
@@ -3218,23 +3986,16 @@ class SystemDatabase {
|
|
|
3218
3986
|
async createSchedule(schedule, client) {
|
|
3219
3987
|
const q = client ?? this.pool;
|
|
3220
3988
|
const owner = await this.#resolveRowOwner(q, 'workflow_schedules', 'schedule_name', schedule.scheduleName, schedule.applicationName, 'Schedule');
|
|
3989
|
+
const values = {
|
|
3990
|
+
...this.globalValues(),
|
|
3991
|
+
...this.scheduleValues(schedule),
|
|
3992
|
+
...scheduleRow(schedule, owner),
|
|
3993
|
+
};
|
|
3221
3994
|
try {
|
|
3222
|
-
await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
3223
|
-
(
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
schedule.scheduleName,
|
|
3227
|
-
schedule.workflowName,
|
|
3228
|
-
schedule.workflowClassName,
|
|
3229
|
-
schedule.schedule,
|
|
3230
|
-
schedule.status,
|
|
3231
|
-
schedule.context,
|
|
3232
|
-
schedule.lastFiredAt,
|
|
3233
|
-
schedule.automaticBackfill,
|
|
3234
|
-
schedule.cronTimezone,
|
|
3235
|
-
schedule.queueName,
|
|
3236
|
-
owner ?? null,
|
|
3237
|
-
]);
|
|
3995
|
+
await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
|
|
3996
|
+
VALUES (${Object.keys(values)
|
|
3997
|
+
.map((_, i) => `$${i + 1}`)
|
|
3998
|
+
.join(', ')})`, Object.values(values));
|
|
3238
3999
|
}
|
|
3239
4000
|
catch (e) {
|
|
3240
4001
|
if (isPgDatabaseError(e) && e.code === '23505') {
|
|
@@ -3281,31 +4042,37 @@ class SystemDatabase {
|
|
|
3281
4042
|
// Unset scopes to this application, as on every other observability query.
|
|
3282
4043
|
conditions.push(this.observabilityFilter('application_name', filters?.applicationName, params));
|
|
3283
4044
|
paramIdx = params.length + 1;
|
|
4045
|
+
conditions.push(this.global(undefined, params));
|
|
3284
4046
|
const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
|
|
3285
|
-
const result = await q.query(`SELECT ${
|
|
4047
|
+
const result = await q.query(`SELECT ${this.#scheduleColumns()}
|
|
3286
4048
|
FROM "${this.schemaName}".workflow_schedules${where}
|
|
3287
4049
|
ORDER BY schedule_name`, params);
|
|
3288
4050
|
return result.rows.map((row) => mapWorkflowSchedule(row));
|
|
3289
4051
|
}
|
|
3290
4052
|
async getSchedule(name, client) {
|
|
3291
4053
|
const q = client ?? this.pool;
|
|
3292
|
-
const
|
|
4054
|
+
const params = [name];
|
|
4055
|
+
const result = await q.query(`SELECT ${this.#scheduleColumns()}
|
|
3293
4056
|
FROM "${this.schemaName}".workflow_schedules
|
|
3294
|
-
WHERE schedule_name = $1`,
|
|
4057
|
+
WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
|
|
3295
4058
|
if (result.rows.length === 0)
|
|
3296
4059
|
return null;
|
|
3297
4060
|
return mapWorkflowSchedule(result.rows[0]);
|
|
3298
4061
|
}
|
|
4062
|
+
/** The columns a schedule is read back with, its workflows' shard key among them on a sharded database. */
|
|
4063
|
+
#scheduleColumns() {
|
|
4064
|
+
const keyColumn = this.scheduleKeyColumn();
|
|
4065
|
+
return keyColumn !== undefined ? `${SCHEDULE_COLUMNS}, ${keyColumn}` : SCHEDULE_COLUMNS;
|
|
4066
|
+
}
|
|
3299
4067
|
async deleteSchedule(name, client) {
|
|
3300
4068
|
const q = client ?? this.pool;
|
|
3301
|
-
|
|
4069
|
+
const params = [name];
|
|
4070
|
+
await q.query(`DELETE FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
|
|
3302
4071
|
}
|
|
3303
4072
|
async setScheduleStatus(name, status, client) {
|
|
3304
4073
|
const q = client ?? this.pool;
|
|
3305
|
-
|
|
3306
|
-
|
|
3307
|
-
name,
|
|
3308
|
-
]);
|
|
4074
|
+
const params = [status, name];
|
|
4075
|
+
await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET status = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
|
|
3309
4076
|
}
|
|
3310
4077
|
async updateSchedule(name, updates, client) {
|
|
3311
4078
|
const q = client ?? this.pool;
|
|
@@ -3328,35 +4095,46 @@ class SystemDatabase {
|
|
|
3328
4095
|
}
|
|
3329
4096
|
if (setClauses.length === 0) {
|
|
3330
4097
|
// Nothing to change, but still surface a missing schedule as an error.
|
|
3331
|
-
const
|
|
3332
|
-
|
|
3333
|
-
]);
|
|
4098
|
+
const existingParams = [name];
|
|
4099
|
+
const existing = await q.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, existingParams)} AND schedule_name = $1`, existingParams);
|
|
3334
4100
|
if (existing.rows.length === 0) {
|
|
3335
4101
|
throw new error_1.DBOSError(`Schedule '${name}' not found`);
|
|
3336
4102
|
}
|
|
3337
4103
|
return;
|
|
3338
4104
|
}
|
|
3339
4105
|
params.push(name);
|
|
3340
|
-
const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx}`, params);
|
|
4106
|
+
const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx} AND ${this.global(undefined, params)}`, params);
|
|
3341
4107
|
if (result.rowCount === 0) {
|
|
3342
4108
|
throw new error_1.DBOSError(`Schedule '${name}' not found`);
|
|
3343
4109
|
}
|
|
3344
4110
|
}
|
|
3345
4111
|
async updateLastFiredAt(name, lastFiredAt) {
|
|
3346
|
-
|
|
4112
|
+
const params = [lastFiredAt, name];
|
|
4113
|
+
await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
|
|
3347
4114
|
}
|
|
3348
4115
|
async applySchedules(schedules) {
|
|
3349
4116
|
const client = await this.connect();
|
|
3350
4117
|
try {
|
|
3351
4118
|
await client.query('BEGIN');
|
|
3352
|
-
|
|
4119
|
+
// Sorted, so concurrent writers lock schedule rows in one order.
|
|
4120
|
+
const sorted = [...schedules].sort((a, b) => compareCodeUnits(a.scheduleName, b.scheduleName));
|
|
4121
|
+
for (const sched of sorted) {
|
|
3353
4122
|
const owner = await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
4123
|
+
const scheduleKeys = this.scheduleValues(sched);
|
|
4124
|
+
const values = {
|
|
4125
|
+
...this.globalValues(),
|
|
4126
|
+
...scheduleKeys,
|
|
4127
|
+
...scheduleRow(sched, owner),
|
|
4128
|
+
};
|
|
3354
4129
|
// Upsert on schedule_name; on conflict, preserve schedule_id and runtime state (status, last_fired_at) and update only the declared definition fields, so an unchanged re-apply is a no-op.
|
|
3355
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
3356
|
-
(
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
4130
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
|
|
4131
|
+
VALUES (${Object.keys(values)
|
|
4132
|
+
.map((_, i) => `$${i + 1}`)
|
|
4133
|
+
.join(', ')})
|
|
4134
|
+
ON CONFLICT (${this.keyConflict('schedule_name')}) DO UPDATE SET
|
|
4135
|
+
${Object.keys(scheduleKeys)
|
|
4136
|
+
.map((c) => `${c} = EXCLUDED.${c}, `)
|
|
4137
|
+
.join('')}workflow_name = EXCLUDED.workflow_name,
|
|
3360
4138
|
workflow_class_name = EXCLUDED.workflow_class_name,
|
|
3361
4139
|
schedule = EXCLUDED.schedule,
|
|
3362
4140
|
context = EXCLUDED.context,
|
|
@@ -3364,20 +4142,7 @@ class SystemDatabase {
|
|
|
3364
4142
|
cron_timezone = EXCLUDED.cron_timezone,
|
|
3365
4143
|
queue_name = EXCLUDED.queue_name,
|
|
3366
4144
|
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it took.
|
|
3367
|
-
application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`,
|
|
3368
|
-
sched.scheduleId,
|
|
3369
|
-
sched.scheduleName,
|
|
3370
|
-
sched.workflowName,
|
|
3371
|
-
sched.workflowClassName,
|
|
3372
|
-
sched.schedule,
|
|
3373
|
-
sched.status,
|
|
3374
|
-
sched.context,
|
|
3375
|
-
sched.lastFiredAt,
|
|
3376
|
-
sched.automaticBackfill,
|
|
3377
|
-
sched.cronTimezone,
|
|
3378
|
-
sched.queueName,
|
|
3379
|
-
owner ?? null,
|
|
3380
|
-
]);
|
|
4145
|
+
application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, Object.values(values));
|
|
3381
4146
|
// Read back, since the guard above is silent about why it declined to claim.
|
|
3382
4147
|
await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
3383
4148
|
}
|
|
@@ -3402,14 +4167,23 @@ class SystemDatabase {
|
|
|
3402
4167
|
try {
|
|
3403
4168
|
await client.query('BEGIN');
|
|
3404
4169
|
// Claim a pre-upgrade row in place, so the version is not recreated or retimed.
|
|
4170
|
+
const claimParams = [owner ?? null, versionName];
|
|
3405
4171
|
const claimed = await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3406
4172
|
SET application_name = $1
|
|
3407
|
-
WHERE version_name = $2 AND application_name IS NULL`,
|
|
4173
|
+
WHERE ${this.global(undefined, claimParams)} AND version_name = $2 AND application_name IS NULL`, claimParams);
|
|
3408
4174
|
if ((claimed.rowCount ?? 0) === 0) {
|
|
4175
|
+
const values = {
|
|
4176
|
+
...this.globalValues(),
|
|
4177
|
+
version_id: (0, crypto_1.randomUUID)(),
|
|
4178
|
+
version_name: versionName,
|
|
4179
|
+
application_name: owner ?? null,
|
|
4180
|
+
};
|
|
3409
4181
|
// Targetless DO NOTHING: names no arbiter, so it survives version_name's uniqueness being dropped while still absorbing a concurrent registrar.
|
|
3410
|
-
await client.query(`INSERT INTO "${this.schemaName}".application_versions (
|
|
3411
|
-
VALUES ($
|
|
3412
|
-
|
|
4182
|
+
await client.query(`INSERT INTO "${this.schemaName}".application_versions (${Object.keys(values).join(', ')})
|
|
4183
|
+
VALUES (${Object.keys(values)
|
|
4184
|
+
.map((_, i) => `$${i + 1}`)
|
|
4185
|
+
.join(', ')})
|
|
4186
|
+
ON CONFLICT DO NOTHING`, Object.values(values));
|
|
3413
4187
|
}
|
|
3414
4188
|
// Read back, since the writes above are silent about why they declined to claim.
|
|
3415
4189
|
await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
|
|
@@ -3440,7 +4214,7 @@ class SystemDatabase {
|
|
|
3440
4214
|
params.push(resolved);
|
|
3441
4215
|
await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3442
4216
|
SET version_timestamp = $1, application_name = ${resolved === undefined ? 'application_name' : '$3'}
|
|
3443
|
-
WHERE version_name = $2 AND ${scope}`, params);
|
|
4217
|
+
WHERE ${this.global(undefined, params)} AND version_name = $2 AND ${scope}`, params);
|
|
3444
4218
|
await client.query('COMMIT');
|
|
3445
4219
|
}
|
|
3446
4220
|
catch (e) {
|
|
@@ -3453,7 +4227,7 @@ class SystemDatabase {
|
|
|
3453
4227
|
}
|
|
3454
4228
|
async listApplicationVersions() {
|
|
3455
4229
|
const params = [];
|
|
3456
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
4230
|
+
const scope = `${this.global(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
3457
4231
|
const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
|
|
3458
4232
|
FROM "${this.schemaName}".application_versions
|
|
3459
4233
|
WHERE ${scope}
|
|
@@ -3473,9 +4247,10 @@ class SystemDatabase {
|
|
|
3473
4247
|
}
|
|
3474
4248
|
// ==================== Queues ====================
|
|
3475
4249
|
async getQueue(name) {
|
|
4250
|
+
const params = [name];
|
|
3476
4251
|
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3477
4252
|
FROM "${this.schemaName}".queues
|
|
3478
|
-
WHERE name = $1`,
|
|
4253
|
+
WHERE ${this.global(undefined, params)} AND name = $1`, params);
|
|
3479
4254
|
return rows.length === 0 ? null : queueRecordFromRow(rows[0]);
|
|
3480
4255
|
}
|
|
3481
4256
|
/**
|
|
@@ -3484,14 +4259,15 @@ class SystemDatabase {
|
|
|
3484
4259
|
*/
|
|
3485
4260
|
async listQueues(applicationName) {
|
|
3486
4261
|
const params = [];
|
|
3487
|
-
const scope = this.observabilityFilter('application_name', applicationName, params)
|
|
4262
|
+
const scope = `${this.global(undefined, params)} AND ${this.observabilityFilter('application_name', applicationName, params)}`;
|
|
3488
4263
|
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3489
4264
|
FROM "${this.schemaName}".queues
|
|
3490
4265
|
WHERE ${scope}`, params);
|
|
3491
4266
|
return rows.map(queueRecordFromRow);
|
|
3492
4267
|
}
|
|
3493
4268
|
async deleteQueue(name) {
|
|
3494
|
-
|
|
4269
|
+
const params = [name];
|
|
4270
|
+
await this.pool.query(`DELETE FROM "${this.schemaName}".queues WHERE ${this.global(undefined, params)} AND name = $1`, params);
|
|
3495
4271
|
}
|
|
3496
4272
|
async updateQueue(name, fields) {
|
|
3497
4273
|
const setClauses = [];
|
|
@@ -3507,7 +4283,7 @@ class SystemDatabase {
|
|
|
3507
4283
|
setClauses.push(`"updated_at" = $${idx++}`);
|
|
3508
4284
|
params.push(Date.now());
|
|
3509
4285
|
params.push(name);
|
|
3510
|
-
await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx}`, params);
|
|
4286
|
+
await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx} AND ${this.global(undefined, params)}`, params);
|
|
3511
4287
|
}
|
|
3512
4288
|
/** Returns true iff this call inserted a new row (i.e. the queue did not
|
|
3513
4289
|
* previously exist). False if the row already existed, regardless of
|
|
@@ -3515,7 +4291,7 @@ class SystemDatabase {
|
|
|
3515
4291
|
async upsertQueue(record, updateExisting) {
|
|
3516
4292
|
const now = Date.now();
|
|
3517
4293
|
const onConflict = updateExisting
|
|
3518
|
-
? `ON CONFLICT (name) DO UPDATE SET
|
|
4294
|
+
? `ON CONFLICT (${this.keyConflict('name')}) DO UPDATE SET
|
|
3519
4295
|
concurrency = EXCLUDED.concurrency,
|
|
3520
4296
|
worker_concurrency = EXCLUDED.worker_concurrency,
|
|
3521
4297
|
rate_limit_max = EXCLUDED.rate_limit_max,
|
|
@@ -3530,21 +4306,27 @@ class SystemDatabase {
|
|
|
3530
4306
|
updated_at = EXCLUDED.updated_at,
|
|
3531
4307
|
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it just took.
|
|
3532
4308
|
application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
|
|
3533
|
-
: `ON CONFLICT (name) DO NOTHING`;
|
|
4309
|
+
: `ON CONFLICT (${this.keyConflict('name')}) DO NOTHING`;
|
|
3534
4310
|
const owner = record.applicationName ?? this.appName;
|
|
3535
4311
|
const client = await this.connect();
|
|
3536
4312
|
try {
|
|
3537
4313
|
await client.query('BEGIN');
|
|
3538
|
-
const
|
|
4314
|
+
const existedParams = [record.name];
|
|
4315
|
+
const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE ${this.global(undefined, existedParams)} AND name = $1`, existedParams);
|
|
3539
4316
|
// A name collision is a conflict in every mode: the name is the queue's address.
|
|
3540
4317
|
const resolvedOwner = await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
4318
|
+
const keys = this.globalValues();
|
|
3541
4319
|
await client.query(`INSERT INTO "${this.schemaName}".queues
|
|
3542
|
-
(
|
|
4320
|
+
(${Object.keys(keys)
|
|
4321
|
+
.map((c) => `${c}, `)
|
|
4322
|
+
.join('')}name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
|
|
3543
4323
|
priority_enabled, partition_queue, partition_concurrency, partition_worker_concurrency,
|
|
3544
4324
|
partition_rate_limit_max, partition_rate_limit_period_sec,
|
|
3545
4325
|
polling_interval_sec, updated_at, application_name)
|
|
3546
4326
|
-- priority_enabled is vestigial: every queue dispatches in priority order.
|
|
3547
|
-
VALUES ($
|
|
4327
|
+
VALUES (${Object.keys(keys)
|
|
4328
|
+
.map((_, i) => `$${14 + i}, `)
|
|
4329
|
+
.join('')}$1, $2, $3, $4, $5, TRUE, $6, $7, $8, $9, $10, $11, $12, $13)
|
|
3548
4330
|
${onConflict}`, [
|
|
3549
4331
|
record.name,
|
|
3550
4332
|
record.concurrency,
|
|
@@ -3559,6 +4341,7 @@ class SystemDatabase {
|
|
|
3559
4341
|
record.pollingIntervalSec,
|
|
3560
4342
|
now,
|
|
3561
4343
|
resolvedOwner ?? null,
|
|
4344
|
+
...Object.values(keys),
|
|
3562
4345
|
]);
|
|
3563
4346
|
// Read back, since the guard above is silent about why it declined to claim.
|
|
3564
4347
|
await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
@@ -3668,6 +4451,9 @@ class SystemDatabase {
|
|
|
3668
4451
|
params.push(statuses);
|
|
3669
4452
|
where = `${where} AND status = ANY($${params.length})`;
|
|
3670
4453
|
}
|
|
4454
|
+
else {
|
|
4455
|
+
where = `${where} AND ${this.global(undefined, params)}`;
|
|
4456
|
+
}
|
|
3671
4457
|
const res = await client.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${where}`, params);
|
|
3672
4458
|
return res.rowCount ?? 0;
|
|
3673
4459
|
};
|
|
@@ -3675,7 +4461,12 @@ class SystemDatabase {
|
|
|
3675
4461
|
queues = await move('queues');
|
|
3676
4462
|
schedules = await move('workflow_schedules');
|
|
3677
4463
|
versions = await move('application_versions');
|
|
3678
|
-
inFlight = await move('workflow_status', [
|
|
4464
|
+
inFlight = await move('workflow_status', [
|
|
4465
|
+
workflow_1.StatusString.PENDING,
|
|
4466
|
+
workflow_1.StatusString.ENQUEUED,
|
|
4467
|
+
workflow_1.StatusString.DELAYED,
|
|
4468
|
+
workflow_1.StatusString.SUSPENDED,
|
|
4469
|
+
]);
|
|
3679
4470
|
await client.query('COMMIT');
|
|
3680
4471
|
}
|
|
3681
4472
|
catch (e) {
|
|
@@ -3692,84 +4483,64 @@ class SystemDatabase {
|
|
|
3692
4483
|
}
|
|
3693
4484
|
// ==================== Internal ====================
|
|
3694
4485
|
async insertWorkflowStatus(client, initStatus, creatorXid) {
|
|
4486
|
+
const values = {
|
|
4487
|
+
...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
|
|
4488
|
+
workflow_uuid: initStatus.workflowUUID,
|
|
4489
|
+
status: initStatus.status,
|
|
4490
|
+
name: initStatus.workflowName,
|
|
4491
|
+
// For cross-language compatibility, these variables MUST be NULL in the database when not set
|
|
4492
|
+
class_name: initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
|
|
4493
|
+
config_name: initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
|
|
4494
|
+
queue_name: initStatus.queueName ?? null,
|
|
4495
|
+
authenticated_user: initStatus.authenticatedUser,
|
|
4496
|
+
assumed_role: initStatus.assumedRole,
|
|
4497
|
+
authenticated_roles: JSON.stringify(initStatus.authenticatedRoles),
|
|
4498
|
+
executor_id: initStatus.executorId,
|
|
4499
|
+
application_version: initStatus.applicationVersion ?? null,
|
|
4500
|
+
application_id: initStatus.applicationID,
|
|
4501
|
+
recovery_attempts: initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
|
|
4502
|
+
workflow_timeout_ms: initStatus.timeoutMS ?? null,
|
|
4503
|
+
workflow_deadline_epoch_ms: initStatus.deadlineEpochMS ?? null,
|
|
4504
|
+
// Legacy column: the payload lives in workflow_input.
|
|
4505
|
+
inputs: null,
|
|
4506
|
+
deduplication_id: initStatus.deduplicationID ?? null,
|
|
4507
|
+
priority: initStatus.priority,
|
|
4508
|
+
queue_partition_key: initStatus.queuePartitionKey ?? null,
|
|
4509
|
+
forked_from: initStatus.forkedFrom ?? null,
|
|
4510
|
+
parent_workflow_id: initStatus.parentWorkflowID ?? null,
|
|
4511
|
+
...this.parentValues(initStatus.parentWorkflowID),
|
|
4512
|
+
serialization: initStatus.serialization,
|
|
4513
|
+
creator_xid: creatorXid,
|
|
4514
|
+
delay_until_epoch_ms: initStatus.delayUntilEpochMS ?? null,
|
|
4515
|
+
attributes: initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
|
|
4516
|
+
schedule_name: initStatus.scheduleName ?? null,
|
|
4517
|
+
debounce_deadline_epoch_ms: initStatus.debounceDeadlineEpochMS ?? null,
|
|
4518
|
+
is_debounced: initStatus.isDebounced ?? false,
|
|
4519
|
+
application_name: initStatus.applicationName ?? null,
|
|
4520
|
+
// A direct start runs at once, so its creator token also owns the execution.
|
|
4521
|
+
owner_xid: initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
|
|
4522
|
+
};
|
|
4523
|
+
const columns = Object.keys(values);
|
|
4524
|
+
const inputValues = {
|
|
4525
|
+
...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
|
|
4526
|
+
workflow_uuid: initStatus.workflowUUID,
|
|
4527
|
+
inputs: initStatus.input ?? null,
|
|
4528
|
+
};
|
|
4529
|
+
const inputColumns = Object.keys(inputValues);
|
|
3695
4530
|
try {
|
|
3696
|
-
const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
name,
|
|
3700
|
-
class_name,
|
|
3701
|
-
config_name,
|
|
3702
|
-
queue_name,
|
|
3703
|
-
authenticated_user,
|
|
3704
|
-
assumed_role,
|
|
3705
|
-
authenticated_roles,
|
|
3706
|
-
executor_id,
|
|
3707
|
-
application_version,
|
|
3708
|
-
application_id,
|
|
3709
|
-
recovery_attempts,
|
|
3710
|
-
workflow_timeout_ms,
|
|
3711
|
-
workflow_deadline_epoch_ms,
|
|
3712
|
-
inputs,
|
|
3713
|
-
deduplication_id,
|
|
3714
|
-
priority,
|
|
3715
|
-
queue_partition_key,
|
|
3716
|
-
forked_from,
|
|
3717
|
-
parent_workflow_id,
|
|
3718
|
-
serialization,
|
|
3719
|
-
creator_xid,
|
|
3720
|
-
delay_until_epoch_ms,
|
|
3721
|
-
attributes,
|
|
3722
|
-
schedule_name,
|
|
3723
|
-
debounce_deadline_epoch_ms,
|
|
3724
|
-
is_debounced,
|
|
3725
|
-
application_name,
|
|
3726
|
-
owner_xid
|
|
3727
|
-
) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)
|
|
3728
|
-
ON CONFLICT (workflow_uuid)
|
|
4531
|
+
const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
|
|
4532
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
4533
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')})
|
|
3729
4534
|
-- A no-op update, so an existing row comes back unchanged for the caller to inspect.
|
|
3730
4535
|
DO UPDATE SET creator_xid = workflow_status.creator_xid
|
|
3731
|
-
RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`,
|
|
3732
|
-
initStatus.workflowUUID,
|
|
3733
|
-
initStatus.status,
|
|
3734
|
-
initStatus.workflowName,
|
|
3735
|
-
// For cross-language compatibility, these variables MUST be NULL in the database when not set
|
|
3736
|
-
initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
|
|
3737
|
-
initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
|
|
3738
|
-
initStatus.queueName ?? null,
|
|
3739
|
-
initStatus.authenticatedUser,
|
|
3740
|
-
initStatus.assumedRole,
|
|
3741
|
-
JSON.stringify(initStatus.authenticatedRoles),
|
|
3742
|
-
initStatus.executorId,
|
|
3743
|
-
initStatus.applicationVersion ?? null,
|
|
3744
|
-
initStatus.applicationID,
|
|
3745
|
-
initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
|
|
3746
|
-
initStatus.timeoutMS ?? null,
|
|
3747
|
-
initStatus.deadlineEpochMS ?? null,
|
|
3748
|
-
// Legacy column: the payload lives in workflow_input.
|
|
3749
|
-
null,
|
|
3750
|
-
initStatus.deduplicationID ?? null,
|
|
3751
|
-
initStatus.priority,
|
|
3752
|
-
initStatus.queuePartitionKey ?? null,
|
|
3753
|
-
initStatus.forkedFrom ?? null,
|
|
3754
|
-
initStatus.parentWorkflowID ?? null,
|
|
3755
|
-
initStatus.serialization,
|
|
3756
|
-
creatorXid,
|
|
3757
|
-
initStatus.delayUntilEpochMS ?? null,
|
|
3758
|
-
initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
|
|
3759
|
-
initStatus.scheduleName ?? null,
|
|
3760
|
-
initStatus.debounceDeadlineEpochMS ?? null,
|
|
3761
|
-
initStatus.isDebounced ?? false,
|
|
3762
|
-
initStatus.applicationName ?? null,
|
|
3763
|
-
// A direct start runs at once, so its creator token also owns the execution.
|
|
3764
|
-
initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
|
|
3765
|
-
]);
|
|
4536
|
+
RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`, Object.values(values));
|
|
3766
4537
|
if (rows.length === 0) {
|
|
3767
4538
|
throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
|
|
3768
4539
|
}
|
|
3769
4540
|
// Two statements, not a data-modifying CTE: at scale the CTE costs more than the round trip it saves.
|
|
3770
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (
|
|
3771
|
-
VALUES ($1,
|
|
3772
|
-
ON CONFLICT (workflow_uuid) DO NOTHING`,
|
|
4541
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (${inputColumns.join(', ')})
|
|
4542
|
+
VALUES (${inputColumns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
4543
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`, Object.values(inputValues));
|
|
3773
4544
|
const ret = rows[0];
|
|
3774
4545
|
ret.class_name = ret.class_name ?? '';
|
|
3775
4546
|
ret.config_name = ret.config_name ?? '';
|
|
@@ -3785,16 +4556,17 @@ class SystemDatabase {
|
|
|
3785
4556
|
}
|
|
3786
4557
|
}
|
|
3787
4558
|
async getWorkflowStatusValue(client, workflowID) {
|
|
3788
|
-
const
|
|
4559
|
+
const params = [];
|
|
4560
|
+
const { rows } = await client.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
|
|
3789
4561
|
return rows.length === 0 ? undefined : rows[0].status;
|
|
3790
4562
|
}
|
|
3791
4563
|
async updateWorkflowStatus(client, workflowID, status, options = {}) {
|
|
3792
4564
|
// Use SQL now() so updated_at and completed_at (when set together) are
|
|
3793
4565
|
// computed in the same statement against the same clock.
|
|
3794
4566
|
const nowMsExpr = `(EXTRACT(EPOCH FROM now()) * 1000)::bigint`;
|
|
3795
|
-
let setClause = `SET status=$
|
|
3796
|
-
|
|
3797
|
-
|
|
4567
|
+
let setClause = `SET status=$1, updated_at=${nowMsExpr}`;
|
|
4568
|
+
const args = [status];
|
|
4569
|
+
let whereClause = `WHERE ${this.wf(undefined, workflowID, args)}`;
|
|
3798
4570
|
const update = options.update ?? {};
|
|
3799
4571
|
if (update.resetRecoveryAttempts) {
|
|
3800
4572
|
setClause += `, recovery_attempts = 0`;
|
|
@@ -3853,27 +4625,31 @@ class SystemDatabase {
|
|
|
3853
4625
|
async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
3854
4626
|
const ownerXid = options.ownerChecked ? undefined : (0, context_1.currentOwnerXid)(workflowID);
|
|
3855
4627
|
if (ownerXid !== undefined) {
|
|
3856
|
-
await this
|
|
3857
|
-
}
|
|
4628
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
4629
|
+
}
|
|
4630
|
+
const values = {
|
|
4631
|
+
...this.wfValues(workflowID),
|
|
4632
|
+
workflow_uuid: workflowID,
|
|
4633
|
+
function_id: functionID,
|
|
4634
|
+
output: options.output ?? null,
|
|
4635
|
+
error: options.error ?? null,
|
|
4636
|
+
function_name: functionName,
|
|
4637
|
+
child_workflow_id: options.childWorkflowID ?? null,
|
|
4638
|
+
...(options.childValues ?? this.childValues(options.childWorkflowID)),
|
|
4639
|
+
started_at_epoch_ms: startTimeEpochMs,
|
|
4640
|
+
completed_at_epoch_ms: endTimeEpochMs,
|
|
4641
|
+
serialization: options.serialization ?? null,
|
|
4642
|
+
// Mirrors the parent: only the running application records its steps.
|
|
4643
|
+
application_name: this.appName ?? null,
|
|
4644
|
+
};
|
|
4645
|
+
const columns = Object.keys(values);
|
|
3858
4646
|
try {
|
|
3859
4647
|
const out = await client.query(`INSERT INTO "${this.schemaName}".operation_outputs
|
|
3860
|
-
(
|
|
3861
|
-
VALUES ($
|
|
3862
|
-
ON CONFLICT (workflow_uuid, function_id) DO UPDATE
|
|
4648
|
+
(${columns.join(', ')}, retention_timestamp)
|
|
4649
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')}, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
|
|
4650
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id')}) DO UPDATE
|
|
3863
4651
|
SET completed_at_epoch_ms = operation_outputs.completed_at_epoch_ms
|
|
3864
|
-
RETURNING completed_at_epoch_ms;`,
|
|
3865
|
-
workflowID,
|
|
3866
|
-
functionID,
|
|
3867
|
-
options.output ?? null,
|
|
3868
|
-
options.error ?? null,
|
|
3869
|
-
functionName,
|
|
3870
|
-
options.childWorkflowID ?? null,
|
|
3871
|
-
startTimeEpochMs,
|
|
3872
|
-
endTimeEpochMs,
|
|
3873
|
-
options.serialization ?? null,
|
|
3874
|
-
// Mirrors the parent: only the running application records its steps.
|
|
3875
|
-
this.appName ?? null,
|
|
3876
|
-
]);
|
|
4652
|
+
RETURNING completed_at_epoch_ms;`, Object.values(values));
|
|
3877
4653
|
// A row already there with another completion time is this execution's own doing
|
|
3878
4654
|
// (a retry carries the same time), so it is nondeterminism, not a duplicate.
|
|
3879
4655
|
if (checkConflict &&
|
|
@@ -3890,7 +4666,7 @@ class SystemDatabase {
|
|
|
3890
4666
|
}
|
|
3891
4667
|
}
|
|
3892
4668
|
/** Run `fn` in a READ COMMITTED transaction on `client`, so an ownership check holds its row lock until the write commits. */
|
|
3893
|
-
async
|
|
4669
|
+
async inTransaction(client, fn) {
|
|
3894
4670
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
3895
4671
|
let result;
|
|
3896
4672
|
try {
|
|
@@ -3905,22 +4681,26 @@ class SystemDatabase {
|
|
|
3905
4681
|
return result;
|
|
3906
4682
|
}
|
|
3907
4683
|
/** Throw DBOSWorkflowConflictError unless ownerXid still owns the workflow; the row lock holds until commit, so no hand-off lands in between. */
|
|
3908
|
-
async
|
|
3909
|
-
const
|
|
4684
|
+
async checkOwner(client, workflowID, ownerXid) {
|
|
4685
|
+
const params = [];
|
|
4686
|
+
const { rows } = await client.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)} FOR NO KEY UPDATE`, params);
|
|
3910
4687
|
if (rows[0]?.owner_xid !== ownerXid) {
|
|
3911
4688
|
throw new error_1.DBOSWorkflowConflictError(workflowID);
|
|
3912
4689
|
}
|
|
3913
4690
|
}
|
|
3914
4691
|
/** The workflow's current ownership token; null if unowned or missing. */
|
|
3915
4692
|
async getWorkflowOwner(workflowID) {
|
|
3916
|
-
const
|
|
4693
|
+
const params = [];
|
|
4694
|
+
const { rows } = await this.pool.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
|
|
3917
4695
|
return rows[0]?.owner_xid ?? null;
|
|
3918
4696
|
}
|
|
3919
4697
|
async #getOperationResultAndThrowIfCancelled(client, workflowID, functionID) {
|
|
3920
4698
|
await this.#checkIfCanceled(client, workflowID);
|
|
3921
|
-
const
|
|
4699
|
+
const childKey = this.keyColumn(undefined, 'child_shard_key');
|
|
4700
|
+
const params = [functionID];
|
|
4701
|
+
const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name, serialization${childKey !== undefined ? `, ${childKey}` : ''}
|
|
3922
4702
|
FROM "${this.schemaName}".operation_outputs
|
|
3923
|
-
WHERE
|
|
4703
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
|
|
3924
4704
|
if (rows.length === 0) {
|
|
3925
4705
|
return undefined;
|
|
3926
4706
|
}
|
|
@@ -3929,6 +4709,7 @@ class SystemDatabase {
|
|
|
3929
4709
|
output: rows[0].output,
|
|
3930
4710
|
error: rows[0].error,
|
|
3931
4711
|
childWorkflowID: rows[0].child_workflow_id,
|
|
4712
|
+
childShardKey: rows[0].child_shard_key ?? undefined,
|
|
3932
4713
|
functionName: rows[0].function_name,
|
|
3933
4714
|
// Return serialization so recv/getEvent replay deserializes with the stored format, not the default.
|
|
3934
4715
|
serialization: rows[0].serialization,
|
|
@@ -3966,7 +4747,11 @@ class SystemDatabase {
|
|
|
3966
4747
|
// been cancelled.
|
|
3967
4748
|
// For an actual sleep, completed_at is the wake deadline so the step's duration reflects the
|
|
3968
4749
|
// sleep; a timeout marker records zero duration since its deadline may never be reached.
|
|
3969
|
-
async
|
|
4750
|
+
async durableSleepInternal(workflowID, functionID, durationMS, recordCompletionAtDeadline = false) {
|
|
4751
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
4752
|
+
return await this.ownKey(workflowID, undefined, () => this.#durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline));
|
|
4753
|
+
}
|
|
4754
|
+
async #durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline) {
|
|
3970
4755
|
const startTimeMs = Date.now();
|
|
3971
4756
|
// Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
|
|
3972
4757
|
const endTimeMs = startTimeMs + Math.ceil(durationMS);
|
|
@@ -3979,7 +4764,7 @@ class SystemDatabase {
|
|
|
3979
4764
|
}
|
|
3980
4765
|
return JSON.parse(res.output);
|
|
3981
4766
|
}
|
|
3982
|
-
await this
|
|
4767
|
+
await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
|
|
3983
4768
|
output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
|
|
3984
4769
|
serialization: serialization_1.DBOSPortableJSON.name(),
|
|
3985
4770
|
}));
|
|
@@ -4132,6 +4917,12 @@ __decorate([
|
|
|
4132
4917
|
__metadata("design:paramtypes", [Array, Number]),
|
|
4133
4918
|
__metadata("design:returntype", Promise)
|
|
4134
4919
|
], SystemDatabase.prototype, "deadLetterWorkflows", null);
|
|
4920
|
+
__decorate([
|
|
4921
|
+
dbRetry(),
|
|
4922
|
+
__metadata("design:type", Function),
|
|
4923
|
+
__metadata("design:paramtypes", [Array, Map]),
|
|
4924
|
+
__metadata("design:returntype", Promise)
|
|
4925
|
+
], SystemDatabase.prototype, "enqueueScheduledWorkflows", null);
|
|
4135
4926
|
__decorate([
|
|
4136
4927
|
dbRetry(),
|
|
4137
4928
|
__metadata("design:type", Function),
|
|
@@ -4159,7 +4950,7 @@ __decorate([
|
|
|
4159
4950
|
__decorate([
|
|
4160
4951
|
dbRetry(),
|
|
4161
4952
|
__metadata("design:type", Function),
|
|
4162
|
-
__metadata("design:paramtypes", [String, Number]),
|
|
4953
|
+
__metadata("design:paramtypes", [String, Number, String]),
|
|
4163
4954
|
__metadata("design:returntype", Promise)
|
|
4164
4955
|
], SystemDatabase.prototype, "getOperationResultAndThrowIfCancelled", null);
|
|
4165
4956
|
__decorate([
|
|
@@ -4219,9 +5010,9 @@ __decorate([
|
|
|
4219
5010
|
__decorate([
|
|
4220
5011
|
dbRetry(),
|
|
4221
5012
|
__metadata("design:type", Function),
|
|
4222
|
-
__metadata("design:paramtypes", [Array
|
|
5013
|
+
__metadata("design:paramtypes", [Array]),
|
|
4223
5014
|
__metadata("design:returntype", Promise)
|
|
4224
|
-
], SystemDatabase.prototype, "
|
|
5015
|
+
], SystemDatabase.prototype, "deleteWorkflowsInternal", null);
|
|
4225
5016
|
__decorate([
|
|
4226
5017
|
dbRetry()
|
|
4227
5018
|
// A missing row normally means the workflow has not been inserted yet, so
|
|
@@ -4255,19 +5046,19 @@ __decorate([
|
|
|
4255
5046
|
__decorate([
|
|
4256
5047
|
dbRetry(),
|
|
4257
5048
|
__metadata("design:type", Function),
|
|
4258
|
-
__metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String]),
|
|
5049
|
+
__metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String, String]),
|
|
4259
5050
|
__metadata("design:returntype", Promise)
|
|
4260
5051
|
], SystemDatabase.prototype, "send", null);
|
|
4261
5052
|
__decorate([
|
|
4262
5053
|
dbRetry(),
|
|
4263
5054
|
__metadata("design:type", Function),
|
|
4264
|
-
__metadata("design:paramtypes", [String, String, Object, Object, Object, String]),
|
|
5055
|
+
__metadata("design:paramtypes", [String, String, Object, Object, Object, String, String]),
|
|
4265
5056
|
__metadata("design:returntype", Promise)
|
|
4266
5057
|
], SystemDatabase.prototype, "sendFromStep", null);
|
|
4267
5058
|
__decorate([
|
|
4268
5059
|
dbRetry(),
|
|
4269
5060
|
__metadata("design:type", Function),
|
|
4270
|
-
__metadata("design:paramtypes", [String, Object, Object, Object, String]),
|
|
5061
|
+
__metadata("design:paramtypes", [String, Object, Object, Object, String, String]),
|
|
4271
5062
|
__metadata("design:returntype", Promise)
|
|
4272
5063
|
], SystemDatabase.prototype, "sendDirectStandalone", null);
|
|
4273
5064
|
__decorate([
|
|
@@ -4276,6 +5067,12 @@ __decorate([
|
|
|
4276
5067
|
__metadata("design:paramtypes", [String, Number, Number, String, Number, Number]),
|
|
4277
5068
|
__metadata("design:returntype", Promise)
|
|
4278
5069
|
], SystemDatabase.prototype, "recv", null);
|
|
5070
|
+
__decorate([
|
|
5071
|
+
dbRetry(),
|
|
5072
|
+
__metadata("design:type", Function),
|
|
5073
|
+
__metadata("design:paramtypes", [String, Number, Object, Number]),
|
|
5074
|
+
__metadata("design:returntype", Promise)
|
|
5075
|
+
], SystemDatabase.prototype, "recvConsume", null);
|
|
4279
5076
|
__decorate([
|
|
4280
5077
|
dbRetry(),
|
|
4281
5078
|
__metadata("design:type", Function),
|
|
@@ -4321,7 +5118,7 @@ __decorate([
|
|
|
4321
5118
|
__decorate([
|
|
4322
5119
|
dbRetry(),
|
|
4323
5120
|
__metadata("design:type", Function),
|
|
4324
|
-
__metadata("design:paramtypes", [Object]),
|
|
5121
|
+
__metadata("design:paramtypes", [Object, Object]),
|
|
4325
5122
|
__metadata("design:returntype", Promise)
|
|
4326
5123
|
], SystemDatabase.prototype, "listWorkflows", null);
|
|
4327
5124
|
__decorate([
|
|
@@ -4342,4 +5139,10 @@ __decorate([
|
|
|
4342
5139
|
__metadata("design:paramtypes", [String]),
|
|
4343
5140
|
__metadata("design:returntype", Promise)
|
|
4344
5141
|
], SystemDatabase.prototype, "getWorkflowOwner", null);
|
|
5142
|
+
__decorate([
|
|
5143
|
+
dbRetry(),
|
|
5144
|
+
__metadata("design:type", Function),
|
|
5145
|
+
__metadata("design:paramtypes", [String, Number, Number, Boolean]),
|
|
5146
|
+
__metadata("design:returntype", Promise)
|
|
5147
|
+
], SystemDatabase.prototype, "durableSleepInternal", null);
|
|
4345
5148
|
//# sourceMappingURL=system_database.js.map
|