@dbos-inc/dbos-sdk 5.4.10-preview → 5.4.11-preview
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli/cli.js +56 -13
- package/dist/src/cli/cli.js.map +1 -1
- package/dist/src/client.d.ts +20 -10
- package/dist/src/client.d.ts.map +1 -1
- package/dist/src/client.js +76 -44
- package/dist/src/client.js.map +1 -1
- package/dist/src/config.d.ts +1 -0
- package/dist/src/config.d.ts.map +1 -1
- package/dist/src/config.js +1 -0
- package/dist/src/config.js.map +1 -1
- package/dist/src/context.d.ts +6 -0
- package/dist/src/context.d.ts.map +1 -1
- package/dist/src/context.js +15 -1
- package/dist/src/context.js.map +1 -1
- package/dist/src/datasource.d.ts +2 -0
- package/dist/src/datasource.d.ts.map +1 -1
- package/dist/src/datasource.js +7 -1
- package/dist/src/datasource.js.map +1 -1
- package/dist/src/dbos-executor.d.ts +7 -4
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js +56 -30
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts +30 -10
- package/dist/src/dbos.d.ts.map +1 -1
- package/dist/src/dbos.js +136 -64
- package/dist/src/dbos.js.map +1 -1
- package/dist/src/enqueue_options.d.ts +2 -0
- package/dist/src/enqueue_options.d.ts.map +1 -1
- package/dist/src/enqueue_options.js +3 -0
- package/dist/src/enqueue_options.js.map +1 -1
- package/dist/src/enqueue_workflow.d.ts.map +1 -1
- package/dist/src/enqueue_workflow.js +11 -5
- package/dist/src/enqueue_workflow.js.map +1 -1
- package/dist/src/enterprise.d.ts +23 -3
- package/dist/src/enterprise.d.ts.map +1 -1
- package/dist/src/enterprise.js +81 -44
- package/dist/src/enterprise.js.map +1 -1
- package/dist/src/eventreceiver.d.ts +2 -0
- package/dist/src/eventreceiver.d.ts.map +1 -1
- package/dist/src/eventreceiver.js +7 -1
- package/dist/src/eventreceiver.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/scheduler/scheduler.d.ts +5 -0
- package/dist/src/scheduler/scheduler.d.ts.map +1 -1
- package/dist/src/scheduler/scheduler.js +14 -4
- package/dist/src/scheduler/scheduler.js.map +1 -1
- package/dist/src/shard_keys.d.ts +13 -0
- package/dist/src/shard_keys.d.ts.map +1 -0
- package/dist/src/shard_keys.js +55 -0
- package/dist/src/shard_keys.js.map +1 -0
- package/dist/src/streams.d.ts +2 -0
- package/dist/src/streams.d.ts.map +1 -1
- package/dist/src/streams.js +5 -2
- package/dist/src/streams.js.map +1 -1
- package/dist/src/system_database.d.ts +143 -9
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +1095 -503
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/wfqueue.d.ts +1 -2
- package/dist/src/wfqueue.d.ts.map +1 -1
- package/dist/src/wfqueue.js +83 -63
- package/dist/src/wfqueue.js.map +1 -1
- package/dist/src/workflow.d.ts +12 -2
- package/dist/src/workflow.d.ts.map +1 -1
- package/dist/src/workflow.js +14 -9
- package/dist/src/workflow.js.map +1 -1
- package/dist/src/workflow_management.d.ts +5 -3
- package/dist/src/workflow_management.d.ts.map +1 -1
- package/dist/src/workflow_management.js +45 -14
- package/dist/src/workflow_management.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
|
@@ -1,15 +1,38 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
2
18
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
19
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
20
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
21
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
22
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
23
|
};
|
|
24
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
25
|
+
if (mod && mod.__esModule) return mod;
|
|
26
|
+
var result = {};
|
|
27
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
28
|
+
__setModuleDefault(result, mod);
|
|
29
|
+
return result;
|
|
30
|
+
};
|
|
8
31
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
32
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
33
|
};
|
|
11
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.SystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
35
|
+
exports.SystemDatabase = exports.createSystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.qualify = exports.holdsShardedSchema = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
13
36
|
const dbos_executor_1 = require("./dbos-executor");
|
|
14
37
|
const pg_1 = require("pg");
|
|
15
38
|
const error_1 = require("./error");
|
|
@@ -18,6 +41,8 @@ const utils_1 = require("./utils");
|
|
|
18
41
|
const context_1 = require("./context");
|
|
19
42
|
const async_hooks_1 = require("async_hooks");
|
|
20
43
|
const crypto_1 = require("crypto");
|
|
44
|
+
const shard_keys_1 = require("./shard_keys");
|
|
45
|
+
const enterprise = __importStar(require("./enterprise"));
|
|
21
46
|
const utils_2 = require("./utils");
|
|
22
47
|
const database_utils_1 = require("./database_utils");
|
|
23
48
|
const migration_runner_1 = require("./sysdb_migrations/migration_runner");
|
|
@@ -70,6 +95,8 @@ function validateObservabilityQueryTimeoutMs(value) {
|
|
|
70
95
|
exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutMs;
|
|
71
96
|
// Bounds how long a frozen or unreachable client can hold system database locks.
|
|
72
97
|
exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = 60_000;
|
|
98
|
+
// How long a background job's lock may sit idle before Postgres ends its holder's session.
|
|
99
|
+
exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = 5000;
|
|
73
100
|
/** The timeout is rendered into a `SET idle_in_transaction_session_timeout`, which takes only an in-range integer. */
|
|
74
101
|
function validateIdleTransactionTimeoutMs(value) {
|
|
75
102
|
if (value !== undefined && (!Number.isFinite(value) || value > MAX_STATEMENT_TIMEOUT_MS)) {
|
|
@@ -157,6 +184,7 @@ async function grantDbosSchemaPermissions(databaseUrl, roleName, logger, schemaN
|
|
|
157
184
|
await client.end();
|
|
158
185
|
}
|
|
159
186
|
}
|
|
187
|
+
exports.grantDbosSchemaPermissions = grantDbosSchemaPermissions;
|
|
160
188
|
/**
|
|
161
189
|
* Check out a connection carrying our own error handler, so a socket death while we hold it is
|
|
162
190
|
* logged instead of thrown at an emitter with no listener. `release()` takes the handler back off,
|
|
@@ -219,6 +247,7 @@ async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = '
|
|
|
219
247
|
}
|
|
220
248
|
const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
|
|
221
249
|
try {
|
|
250
|
+
await refuseShardedSchema(client, schemaName);
|
|
222
251
|
const isCockroach = await isCockroachDB(client);
|
|
223
252
|
await (0, migration_runner_1.runSysMigrationsPg)(client, (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }), schemaName, {
|
|
224
253
|
onWarn: (e) => logger.info(e),
|
|
@@ -242,6 +271,7 @@ exports.migrateSystemDatabase = migrateSystemDatabase;
|
|
|
242
271
|
async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
|
|
243
272
|
const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
|
|
244
273
|
try {
|
|
274
|
+
await refuseShardedSchema(client, schemaName);
|
|
245
275
|
const isCockroach = await isCockroachDB(client);
|
|
246
276
|
const requiredVersion = (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }).length;
|
|
247
277
|
const currentVersion = await (0, migration_runner_1.getCurrentSysDBVersion)(client, schemaName);
|
|
@@ -259,6 +289,41 @@ async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = '
|
|
|
259
289
|
}
|
|
260
290
|
}
|
|
261
291
|
exports.verifySystemDatabase = verifySystemDatabase;
|
|
292
|
+
/** Throw if this schema is sharded: an unsharded handle's statements would span every tenant's rows. */
|
|
293
|
+
async function refuseShardedSchema(client, schemaName) {
|
|
294
|
+
const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
|
|
295
|
+
WHERE table_schema = $1 AND table_name = 'workflow_status' AND column_name = 'shard_key'`, [schemaName]);
|
|
296
|
+
if (rows.length > 0) {
|
|
297
|
+
throw new error_1.DBOSError(`The DBOS system schema ${schemaName} is sharded. Enable sharding to use it: ` +
|
|
298
|
+
'set sharding in your DBOS configuration, or pass sharding: true to DBOSClient.');
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
/** Whether this database holds a sharded DBOS system schema (this one, if named); false if it can't be reached. */
|
|
302
|
+
async function holdsShardedSchema(databaseUrl, schemaName) {
|
|
303
|
+
let client;
|
|
304
|
+
try {
|
|
305
|
+
// Bounded, so an unreachable host fails as fast as the caller's own connection would.
|
|
306
|
+
client = new pg_1.Client({ ...(0, utils_2.getClientConfig)(databaseUrl), connectionTimeoutMillis: 10_000 });
|
|
307
|
+
}
|
|
308
|
+
catch {
|
|
309
|
+
// A malformed URL, for the caller's own connection to report.
|
|
310
|
+
return false;
|
|
311
|
+
}
|
|
312
|
+
client.on('error', () => { });
|
|
313
|
+
try {
|
|
314
|
+
await client.connect();
|
|
315
|
+
const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
|
|
316
|
+
WHERE table_name = 'workflow_status' AND column_name = 'shard_key'${schemaName !== undefined ? ' AND table_schema = $1' : ''} LIMIT 1`, schemaName !== undefined ? [schemaName] : []);
|
|
317
|
+
return rows.length > 0;
|
|
318
|
+
}
|
|
319
|
+
catch {
|
|
320
|
+
return false;
|
|
321
|
+
}
|
|
322
|
+
finally {
|
|
323
|
+
await client.end().catch(() => { });
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
exports.holdsShardedSchema = holdsShardedSchema;
|
|
262
327
|
class NotificationMap {
|
|
263
328
|
map = new Map();
|
|
264
329
|
curCK = 0;
|
|
@@ -300,6 +365,23 @@ function mapVersionInfo(row) {
|
|
|
300
365
|
};
|
|
301
366
|
}
|
|
302
367
|
const SCHEDULE_COLUMNS = 'schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name';
|
|
368
|
+
/** A schedule's own columns, as a new row stores them. */
|
|
369
|
+
function scheduleRow(schedule, owner) {
|
|
370
|
+
return {
|
|
371
|
+
schedule_id: schedule.scheduleId,
|
|
372
|
+
schedule_name: schedule.scheduleName,
|
|
373
|
+
workflow_name: schedule.workflowName,
|
|
374
|
+
workflow_class_name: schedule.workflowClassName,
|
|
375
|
+
schedule: schedule.schedule,
|
|
376
|
+
status: schedule.status,
|
|
377
|
+
context: schedule.context,
|
|
378
|
+
last_fired_at: schedule.lastFiredAt,
|
|
379
|
+
automatic_backfill: schedule.automaticBackfill,
|
|
380
|
+
cron_timezone: schedule.cronTimezone,
|
|
381
|
+
queue_name: schedule.queueName,
|
|
382
|
+
application_name: owner ?? null,
|
|
383
|
+
};
|
|
384
|
+
}
|
|
303
385
|
function mapWorkflowSchedule(row) {
|
|
304
386
|
return {
|
|
305
387
|
scheduleId: row.schedule_id,
|
|
@@ -314,6 +396,7 @@ function mapWorkflowSchedule(row) {
|
|
|
314
396
|
cronTimezone: row.cron_timezone ?? null,
|
|
315
397
|
queueName: row.queue_name ?? null,
|
|
316
398
|
applicationName: row.application_name ?? undefined,
|
|
399
|
+
shardKey: row.workflow_shard_key ?? undefined,
|
|
317
400
|
};
|
|
318
401
|
}
|
|
319
402
|
function mapWorkflowStatus(row) {
|
|
@@ -356,8 +439,16 @@ function mapWorkflowStatus(row) {
|
|
|
356
439
|
debounceDeadlineEpochMS: row.debounce_deadline_epoch_ms ? Number(row.debounce_deadline_epoch_ms) : undefined,
|
|
357
440
|
isDebounced: row.is_debounced ?? false,
|
|
358
441
|
applicationName: row.application_name ?? undefined,
|
|
442
|
+
// Read back only from a sharded system database, whose rows carry them.
|
|
443
|
+
shardKey: row.shard_key ?? undefined,
|
|
444
|
+
parentShardKey: row.parent_shard_key ?? undefined,
|
|
359
445
|
};
|
|
360
446
|
}
|
|
447
|
+
/** `column`, qualified by `alias` if one is given. */
|
|
448
|
+
function qualify(alias, column) {
|
|
449
|
+
return alias ? `${alias}.${column}` : column;
|
|
450
|
+
}
|
|
451
|
+
exports.qualify = qualify;
|
|
361
452
|
// SQLSTATE classes/codes that are generally safe to retry
|
|
362
453
|
// https://www.postgresql.org/docs/current/errcodes-appendix.html
|
|
363
454
|
const RETRY_SQLSTATE_PREFIXES = new Set([
|
|
@@ -527,6 +618,26 @@ function dbRetry(options = {}) {
|
|
|
527
618
|
return descriptor;
|
|
528
619
|
};
|
|
529
620
|
}
|
|
621
|
+
/** Build the system database these options describe. */
|
|
622
|
+
function createSystemDatabase(options) {
|
|
623
|
+
if (options.sharding) {
|
|
624
|
+
const { ShardedSystemDatabase } = enterprise.load(enterprise.SHARDING.description);
|
|
625
|
+
return new ShardedSystemDatabase({
|
|
626
|
+
systemDatabaseUrl: options.systemDatabaseUrl,
|
|
627
|
+
logger: options.logger,
|
|
628
|
+
serializer: options.serializer,
|
|
629
|
+
sysDbPoolSize: options.sysDbPoolSize,
|
|
630
|
+
systemDatabasePool: options.systemDatabasePool,
|
|
631
|
+
schemaName: options.schemaName,
|
|
632
|
+
pollingConcurrency: options.pollingConcurrency,
|
|
633
|
+
appName: options.appName,
|
|
634
|
+
observabilityQueryTimeoutMs: options.observabilityQueryTimeoutMs,
|
|
635
|
+
idleTransactionTimeoutMs: options.idleTransactionTimeoutMs,
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
return new SystemDatabase(options.systemDatabaseUrl, options.logger, options.serializer, options.sysDbPoolSize, options.systemDatabasePool, options.schemaName, options.useListenNotify, options.pollingConcurrency, options.notificationCoalesceMs, options.appName, options.observabilityQueryTimeoutMs, options.idleTransactionTimeoutMs);
|
|
639
|
+
}
|
|
640
|
+
exports.createSystemDatabase = createSystemDatabase;
|
|
530
641
|
/**
|
|
531
642
|
* General notes:
|
|
532
643
|
* The responsibilities of the `SystemDatabase` are to store data for workflows, and
|
|
@@ -595,6 +706,7 @@ class SystemDatabase {
|
|
|
595
706
|
pollLimiter;
|
|
596
707
|
/** Executions running in this process, by workflow ID; a re-dispatch runs alongside its stale execution, which parks at its next write. */
|
|
597
708
|
runningWorkflowMap = new Map();
|
|
709
|
+
// Keyed by waiter key, as one ID can live under several shard keys.
|
|
598
710
|
#cancelWatchers = new Map();
|
|
599
711
|
#cancelPollerLoop = undefined;
|
|
600
712
|
#cancelPollerWake = null;
|
|
@@ -727,6 +839,192 @@ class SystemDatabase {
|
|
|
727
839
|
observabilityFilter(column, value, params) {
|
|
728
840
|
return this.appNameFilter(column, value ?? this.appName, params);
|
|
729
841
|
}
|
|
842
|
+
// ==================== Shard Keys ====================
|
|
843
|
+
// Statements locate rows through these, so a sharded subclass adds its key; each appends to `params` and returns SQL.
|
|
844
|
+
/** Whether this handle routes every statement by shard key. */
|
|
845
|
+
get sharded() {
|
|
846
|
+
return false;
|
|
847
|
+
}
|
|
848
|
+
#rejectShardKey(workflowID, shardKey) {
|
|
849
|
+
if ((shardKey !== undefined && shardKey !== null) || (0, shard_keys_1.boundShardKey)(workflowID) !== undefined) {
|
|
850
|
+
throw new error_1.DBOSError('A shard key was given, but this system database is not sharded');
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
/** Predicate selecting one workflow's rows: those whose `column` is `workflowID`. */
|
|
854
|
+
wf(alias, workflowID, params, column = 'workflow_uuid', shardKey) {
|
|
855
|
+
this.#rejectShardKey(workflowID, shardKey);
|
|
856
|
+
params.push(workflowID);
|
|
857
|
+
return `${qualify(alias, column)} = $${params.length}`;
|
|
858
|
+
}
|
|
859
|
+
/** Predicate selecting several workflows' rows. */
|
|
860
|
+
wfIn(alias, workflowIDs, params) {
|
|
861
|
+
for (const workflowID of workflowIDs) {
|
|
862
|
+
this.#rejectShardKey(workflowID, undefined);
|
|
863
|
+
}
|
|
864
|
+
params.push(workflowIDs);
|
|
865
|
+
return `${qualify(alias, 'workflow_uuid')} = ANY($${params.length}::text[])`;
|
|
866
|
+
}
|
|
867
|
+
/** Condition joining two tables' rows of the same workflow. */
|
|
868
|
+
wfJoin(left, right) {
|
|
869
|
+
return `${left}.workflow_uuid = ${right}.workflow_uuid`;
|
|
870
|
+
}
|
|
871
|
+
/** Extra column values for a new row of this workflow. */
|
|
872
|
+
wfValues(workflowID, shardKey) {
|
|
873
|
+
this.#rejectShardKey(workflowID, shardKey);
|
|
874
|
+
return {};
|
|
875
|
+
}
|
|
876
|
+
/** Extra column values storing the shard key of a new workflow's parent. */
|
|
877
|
+
parentValues(_parentWorkflowID) {
|
|
878
|
+
return {};
|
|
879
|
+
}
|
|
880
|
+
/** Extra column values storing the shard key of the workflow a step refers to. */
|
|
881
|
+
childValues(_childWorkflowID) {
|
|
882
|
+
return {};
|
|
883
|
+
}
|
|
884
|
+
/** Extra (column, expression) pairs for an INSERT ... SELECT of this workflow's rows. */
|
|
885
|
+
wfColumns(_workflowID, _params) {
|
|
886
|
+
return [];
|
|
887
|
+
}
|
|
888
|
+
/** Conflict target for a unique key of a workflow table. */
|
|
889
|
+
keyConflict(...columns) {
|
|
890
|
+
return columns.join(', ');
|
|
891
|
+
}
|
|
892
|
+
/** Insert rows into `table` in one statement, returning the RETURNING rows; a sharded subclass inserts row by row. */
|
|
893
|
+
async insertRows(client, table, columns, rows, suffix = '') {
|
|
894
|
+
if (rows.length === 0)
|
|
895
|
+
return [];
|
|
896
|
+
const params = [];
|
|
897
|
+
const tuples = rows.map((row) => `(${row.map((value) => `$${params.push(value)}`).join(', ')})`);
|
|
898
|
+
const { rows: returned } = await client.query(`INSERT INTO "${this.schemaName}".${table} (${columns.join(', ')}) VALUES ${tuples.join(', ')} ${suffix}`, params);
|
|
899
|
+
return returned;
|
|
900
|
+
}
|
|
901
|
+
/** Predicate a queue claim must also meet, evaluated once before it scans; sharded, it caps claimers per key. */
|
|
902
|
+
claimGate(_queueName, _partitionKey, _params) {
|
|
903
|
+
return 'TRUE';
|
|
904
|
+
}
|
|
905
|
+
/** Predicate confining a statement that names no single workflow to the current shard. */
|
|
906
|
+
scoped(_alias, _params) {
|
|
907
|
+
return 'TRUE';
|
|
908
|
+
}
|
|
909
|
+
/** Extra (column, expression) pairs for an INSERT ... SELECT confined to the current shard. */
|
|
910
|
+
scopedColumns(_params) {
|
|
911
|
+
return [];
|
|
912
|
+
}
|
|
913
|
+
/** Predicate selecting a global table's rows (queues, schedules, versions). */
|
|
914
|
+
global(_alias, _params) {
|
|
915
|
+
return 'TRUE';
|
|
916
|
+
}
|
|
917
|
+
/** Extra column values for a new row of a global table. */
|
|
918
|
+
globalValues() {
|
|
919
|
+
return {};
|
|
920
|
+
}
|
|
921
|
+
/** Predicate selecting rows whose `column` holds one of these shard keys. */
|
|
922
|
+
keyIn(_alias, _shardKeys, _params, _column = 'shard_key') {
|
|
923
|
+
throw new error_1.DBOSError('Filtering by shard key requires a sharded system database');
|
|
924
|
+
}
|
|
925
|
+
/** A shard key column to read back with a row, if this handle has one. */
|
|
926
|
+
keyColumn(_alias, _column = 'shard_key') {
|
|
927
|
+
return undefined;
|
|
928
|
+
}
|
|
929
|
+
/** Extra column values storing the shard key a schedule's workflows run under. */
|
|
930
|
+
scheduleValues(schedule) {
|
|
931
|
+
if (schedule.shardKey !== undefined && schedule.shardKey !== null) {
|
|
932
|
+
throw new error_1.DBOSError('Scheduling under a shard key requires a sharded system database');
|
|
933
|
+
}
|
|
934
|
+
return {};
|
|
935
|
+
}
|
|
936
|
+
/** The column holding the shard key a schedule's workflows run under, if any. */
|
|
937
|
+
scheduleKeyColumn() {
|
|
938
|
+
return undefined;
|
|
939
|
+
}
|
|
940
|
+
/** This workflow's shard key, resolved now for a statement or waiter that needs it later. */
|
|
941
|
+
keyHint(workflowID) {
|
|
942
|
+
this.#rejectShardKey(workflowID, undefined);
|
|
943
|
+
return undefined;
|
|
944
|
+
}
|
|
945
|
+
/** The notification-map key for a workflow's topic or key, distinct per shard key as IDs repeat across keys. */
|
|
946
|
+
waiter(workflowID, name) {
|
|
947
|
+
const key = this.keyHint(workflowID);
|
|
948
|
+
const payload = `${workflowID}::${name}`;
|
|
949
|
+
return key === undefined ? payload : `${[...key].length}:${key}:${payload}`;
|
|
950
|
+
}
|
|
951
|
+
/** Bind a workflow to `key`, else to the running workflow's own key even where a same-ID target is bound. */
|
|
952
|
+
ownKey(workflowID, key, callback) {
|
|
953
|
+
if (key === undefined) {
|
|
954
|
+
const ctx = (0, context_1.getCurrentContextStore)();
|
|
955
|
+
if (ctx !== undefined && ctx.workflowId === workflowID) {
|
|
956
|
+
key = ctx.shardKey;
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
return (0, shard_keys_1.bindShardKeys)([[workflowID, key]], callback);
|
|
960
|
+
}
|
|
961
|
+
/** Run `callback` under this application's lock for a background job, passing whether it was taken. */
|
|
962
|
+
async backgroundJobLock(jobName, callback) {
|
|
963
|
+
// CockroachDB grants every advisory lock, so each process runs every job.
|
|
964
|
+
if (await this.#isCockroach()) {
|
|
965
|
+
return await callback(true);
|
|
966
|
+
}
|
|
967
|
+
// Leading 8 bytes of SHA-256, big-endian signed, as for the retention lock.
|
|
968
|
+
const key = advisoryLockKey(`dbos.background_job.${this.schemaName}.${this.appName ?? ''}.${jobName}`);
|
|
969
|
+
return await this.lockedBackgroundJob(jobName, `SELECT pg_try_advisory_xact_lock($1) AS acquired FROM set_config('idle_in_transaction_session_timeout', $2, true)`, [key.toString(), String(exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS)], callback);
|
|
970
|
+
}
|
|
971
|
+
/** Take a transaction-scoped background job lock with `lockQuery`, which returns `acquired`, and run `callback`. */
|
|
972
|
+
async lockedBackgroundJob(jobName, lockQuery, lockParams, callback) {
|
|
973
|
+
const client = await this.connect();
|
|
974
|
+
try {
|
|
975
|
+
await client.query('BEGIN');
|
|
976
|
+
// The local idle timeout ends a frozen holder's session, releasing its lock.
|
|
977
|
+
const { rows } = await client.query(lockQuery, lockParams);
|
|
978
|
+
return await callback(rows[0]?.acquired === true);
|
|
979
|
+
}
|
|
980
|
+
finally {
|
|
981
|
+
try {
|
|
982
|
+
await client.query('ROLLBACK');
|
|
983
|
+
client.release();
|
|
984
|
+
}
|
|
985
|
+
catch (e) {
|
|
986
|
+
// A session the idle timeout ended has released the lock already.
|
|
987
|
+
this.logger.debug(`Could not release the ${jobName} lock: ${String(e)}`);
|
|
988
|
+
client.release(e instanceof Error ? e : true);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
#cockroach = undefined;
|
|
993
|
+
/** Whether this handle's database is CockroachDB, read on first use. */
|
|
994
|
+
#isCockroach() {
|
|
995
|
+
this.#cockroach ??= (async () => {
|
|
996
|
+
const client = await this.connect();
|
|
997
|
+
try {
|
|
998
|
+
return await isCockroachDB(client);
|
|
999
|
+
}
|
|
1000
|
+
finally {
|
|
1001
|
+
client.release();
|
|
1002
|
+
}
|
|
1003
|
+
})();
|
|
1004
|
+
// A failed read is retried on next use, rather than cached.
|
|
1005
|
+
this.#cockroach.catch(() => {
|
|
1006
|
+
this.#cockroach = undefined;
|
|
1007
|
+
});
|
|
1008
|
+
return this.#cockroach;
|
|
1009
|
+
}
|
|
1010
|
+
/** Scopes a queue's workers dequeue from in turn; one unkeyed scope unless sharded. */
|
|
1011
|
+
dequeueScopes(_queueName) {
|
|
1012
|
+
return Promise.resolve([undefined]);
|
|
1013
|
+
}
|
|
1014
|
+
/** Whether a dequeue failed because another worker holds the rows it needs. */
|
|
1015
|
+
isLockConflict(e) {
|
|
1016
|
+
return isPgDatabaseError(e) && e.code === '55P03';
|
|
1017
|
+
}
|
|
1018
|
+
/** Whether a dequeue failed on a serialization conflict with a concurrent one. */
|
|
1019
|
+
isSerializationConflict(e) {
|
|
1020
|
+
return isPgDatabaseError(e) && e.code === '40001';
|
|
1021
|
+
}
|
|
1022
|
+
/** Whether `appVersion` is this application's latest registered version, or none is registered. */
|
|
1023
|
+
async isLatestVersion(appVersion, client) {
|
|
1024
|
+
// Own plus unclaimed: a named peer's deploy must not demote this one.
|
|
1025
|
+
const latest = await this.#latestApplicationVersionName(client ?? this.pool);
|
|
1026
|
+
return latest === undefined || latest === appVersion;
|
|
1027
|
+
}
|
|
730
1028
|
/**
|
|
731
1029
|
* The version name a dequeue treats as latest: this application's own plus unclaimed
|
|
732
1030
|
* ones, so a named peer's deploy does not demote this one.
|
|
@@ -737,15 +1035,17 @@ class SystemDatabase {
|
|
|
737
1035
|
async #latestApplicationVersion(client, applicationName) {
|
|
738
1036
|
const columns = 'version_id, version_name, version_timestamp, created_at, application_name';
|
|
739
1037
|
const table = `"${this.schemaName}".application_versions`;
|
|
1038
|
+
const params = applicationName ? [applicationName] : [];
|
|
1039
|
+
const scope = this.global(undefined, params);
|
|
740
1040
|
// Probe each eligible owner separately; an OR filter can scan an unbounded history of newer peer versions.
|
|
741
1041
|
// Keep the constant owner in the ordering so the IS NULL probe uses the composite index too.
|
|
742
1042
|
const query = applicationName
|
|
743
|
-
? `(SELECT ${columns} FROM ${table} WHERE application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
1043
|
+
? `(SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
744
1044
|
UNION ALL
|
|
745
|
-
(SELECT ${columns} FROM ${table} WHERE application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
1045
|
+
(SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
746
1046
|
ORDER BY version_timestamp DESC LIMIT 1`
|
|
747
|
-
: `SELECT ${columns} FROM ${table} ORDER BY version_timestamp DESC LIMIT 1`;
|
|
748
|
-
const { rows } = await client.query(query,
|
|
1047
|
+
: `SELECT ${columns} FROM ${table} WHERE ${scope} ORDER BY version_timestamp DESC LIMIT 1`;
|
|
1048
|
+
const { rows } = await client.query(query, params);
|
|
749
1049
|
return rows[0];
|
|
750
1050
|
}
|
|
751
1051
|
/**
|
|
@@ -753,7 +1053,8 @@ class SystemDatabase {
|
|
|
753
1053
|
* leaves the owner intact; a named one collides only with a different name.
|
|
754
1054
|
*/
|
|
755
1055
|
async #resolveRowOwner(client, table, keyColumn, name, owner, kind) {
|
|
756
|
-
const
|
|
1056
|
+
const params = [name];
|
|
1057
|
+
const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${this.global(undefined, params)} AND ${keyColumn} = $1`, params);
|
|
757
1058
|
const current = rows[0]?.application_name ?? null;
|
|
758
1059
|
if (current === null)
|
|
759
1060
|
return owner;
|
|
@@ -770,8 +1071,12 @@ class SystemDatabase {
|
|
|
770
1071
|
}
|
|
771
1072
|
/** Migrates the system database, or, when `runMigrations` is false, verifies it is already migrated. */
|
|
772
1073
|
async init(runMigrations = true) {
|
|
773
|
-
|
|
774
|
-
|
|
1074
|
+
if (runMigrations) {
|
|
1075
|
+
await this.runMigrations();
|
|
1076
|
+
}
|
|
1077
|
+
else {
|
|
1078
|
+
await this.verifyMigrations();
|
|
1079
|
+
}
|
|
775
1080
|
if (this.shouldUseDBNotifications) {
|
|
776
1081
|
await this.#listenForNotifications();
|
|
777
1082
|
// Push coalesced stream and event notifications off the write path.
|
|
@@ -779,6 +1084,32 @@ class SystemDatabase {
|
|
|
779
1084
|
this.#notifierLoop = this.#runNotifier();
|
|
780
1085
|
}
|
|
781
1086
|
}
|
|
1087
|
+
/** Create or migrate the system database. */
|
|
1088
|
+
async runMigrations() {
|
|
1089
|
+
await ensureSystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
|
|
1090
|
+
}
|
|
1091
|
+
/** Throw if this unsharded handle's schema is sharded; an unreachable database is left for its first statement. */
|
|
1092
|
+
async checkShardedSchema() {
|
|
1093
|
+
if (this.sharded)
|
|
1094
|
+
return;
|
|
1095
|
+
let client;
|
|
1096
|
+
try {
|
|
1097
|
+
client = await this.connect();
|
|
1098
|
+
}
|
|
1099
|
+
catch {
|
|
1100
|
+
return;
|
|
1101
|
+
}
|
|
1102
|
+
try {
|
|
1103
|
+
await refuseShardedSchema(client, this.schemaName);
|
|
1104
|
+
}
|
|
1105
|
+
finally {
|
|
1106
|
+
client.release();
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
/** Check the system database is migrated, creating and changing nothing. */
|
|
1110
|
+
async verifyMigrations() {
|
|
1111
|
+
await verifySystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
|
|
1112
|
+
}
|
|
782
1113
|
async destroy() {
|
|
783
1114
|
// Set synchronously, before any await, so no reconnect is scheduled or published after this point.
|
|
784
1115
|
this.#notificationsStopped = true;
|
|
@@ -824,7 +1155,7 @@ class SystemDatabase {
|
|
|
824
1155
|
/** Runs on `client` if given, joining its transaction; otherwise in its own retried transaction. */
|
|
825
1156
|
async initWorkflowStatus(initStatus, creatorXid, client, reusePolicy = 'return-existing') {
|
|
826
1157
|
if (client !== undefined) {
|
|
827
|
-
return await this
|
|
1158
|
+
return await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
828
1159
|
}
|
|
829
1160
|
return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
|
|
830
1161
|
}
|
|
@@ -833,7 +1164,7 @@ class SystemDatabase {
|
|
|
833
1164
|
let shouldCommit = false;
|
|
834
1165
|
try {
|
|
835
1166
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
836
|
-
const result = await this
|
|
1167
|
+
const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
837
1168
|
// If there is an existing DB record and we aren't here to recover it, leave it be.
|
|
838
1169
|
shouldCommit = result.shouldExecuteOnThisExecutor;
|
|
839
1170
|
return result;
|
|
@@ -861,10 +1192,12 @@ class SystemDatabase {
|
|
|
861
1192
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
862
1193
|
const parentOwnerXid = (0, context_1.currentOwnerXid)(parentWorkflowID);
|
|
863
1194
|
if (parentOwnerXid !== undefined) {
|
|
864
|
-
await this
|
|
1195
|
+
await this.checkOwner(client, parentWorkflowID, parentOwnerXid);
|
|
865
1196
|
}
|
|
866
|
-
const result = await this
|
|
867
|
-
await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs,
|
|
1197
|
+
const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
1198
|
+
await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs,
|
|
1199
|
+
// Unkeyed, as only a sharded database records the child's key, through its own override.
|
|
1200
|
+
{ childWorkflowID: initStatus.workflowUUID, ownerChecked: true, childValues: {} });
|
|
868
1201
|
shouldCommit = true;
|
|
869
1202
|
return result;
|
|
870
1203
|
}
|
|
@@ -883,7 +1216,7 @@ class SystemDatabase {
|
|
|
883
1216
|
}
|
|
884
1217
|
}
|
|
885
1218
|
}
|
|
886
|
-
async
|
|
1219
|
+
async initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy) {
|
|
887
1220
|
const resRow = await this.insertWorkflowStatus(client, initStatus, creatorXid);
|
|
888
1221
|
if (reusePolicy === 'reject' && (creatorXid === null || resRow.creator_xid !== creatorXid)) {
|
|
889
1222
|
throw new error_1.DBOSWorkflowIDInUseError(initStatus.workflowUUID, resRow.status, resRow.name);
|
|
@@ -915,6 +1248,7 @@ class SystemDatabase {
|
|
|
915
1248
|
async deadLetterWorkflows(workflowIDs, minRecoveryAttempts) {
|
|
916
1249
|
if (workflowIDs.length === 0)
|
|
917
1250
|
return;
|
|
1251
|
+
const params = [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflow_1.StatusString.PENDING, minRecoveryAttempts];
|
|
918
1252
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
919
1253
|
SET status = $1,
|
|
920
1254
|
owner_xid = NULL,
|
|
@@ -923,17 +1257,18 @@ class SystemDatabase {
|
|
|
923
1257
|
queue_name = NULL,
|
|
924
1258
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
925
1259
|
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
926
|
-
WHERE
|
|
1260
|
+
WHERE ${this.wfIn(undefined, workflowIDs, params)} AND status = $2 AND recovery_attempts >= $3`, params);
|
|
927
1261
|
}
|
|
928
1262
|
/** Highest created_at among still-active rows per partition key, used to seed the in-memory cursor. */
|
|
929
1263
|
async #maxPartitionKeyCreatedAt(keys) {
|
|
930
1264
|
const maxima = new Map();
|
|
931
1265
|
if (keys.length === 0)
|
|
932
1266
|
return maxima;
|
|
1267
|
+
const params = [keys, [workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING]];
|
|
933
1268
|
const { rows } = await this.pool.query(`SELECT queue_partition_key, MAX(created_at) AS max_created_at
|
|
934
1269
|
FROM "${this.schemaName}".workflow_status
|
|
935
|
-
WHERE queue_partition_key = ANY($1) AND status = ANY($2)
|
|
936
|
-
GROUP BY queue_partition_key`,
|
|
1270
|
+
WHERE ${this.scoped(undefined, params)} AND queue_partition_key = ANY($1) AND status = ANY($2)
|
|
1271
|
+
GROUP BY queue_partition_key`, params);
|
|
937
1272
|
for (const row of rows) {
|
|
938
1273
|
if (row.max_created_at !== null) {
|
|
939
1274
|
maxima.set(row.queue_partition_key, Number(row.max_created_at));
|
|
@@ -1039,39 +1374,60 @@ class SystemDatabase {
|
|
|
1039
1374
|
const chunkSize = 500;
|
|
1040
1375
|
for (let start = 0; start < statuses.length; start += chunkSize) {
|
|
1041
1376
|
const chunk = statuses.slice(start, start + chunkSize);
|
|
1042
|
-
const
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1377
|
+
const keyed = chunk.map((status) => ({
|
|
1378
|
+
...this.wfValues(status.workflowUUID, status.shardKey),
|
|
1379
|
+
...this.parentValues(status.parentWorkflowID),
|
|
1380
|
+
}));
|
|
1381
|
+
const rows = chunk.map((status, i) => {
|
|
1047
1382
|
const createdAt = createdAts[start + i];
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1383
|
+
return [
|
|
1384
|
+
...Object.values(keyed[i]),
|
|
1385
|
+
status.workflowUUID,
|
|
1386
|
+
status.status,
|
|
1387
|
+
status.workflowName,
|
|
1388
|
+
// For cross-language compatibility, these MUST be NULL in the database when not set
|
|
1389
|
+
status.workflowClassName === '' ? null : status.workflowClassName,
|
|
1390
|
+
status.workflowConfigName === '' ? null : status.workflowConfigName,
|
|
1391
|
+
status.queueName ?? null,
|
|
1392
|
+
status.authenticatedUser,
|
|
1393
|
+
status.assumedRole,
|
|
1394
|
+
JSON.stringify(status.authenticatedRoles),
|
|
1395
|
+
status.executorId,
|
|
1396
|
+
status.applicationVersion ?? null,
|
|
1397
|
+
status.applicationID,
|
|
1398
|
+
createdAt,
|
|
1399
|
+
0,
|
|
1400
|
+
createdAt,
|
|
1401
|
+
status.timeoutMS ?? null,
|
|
1402
|
+
status.deadlineEpochMS ?? null,
|
|
1403
|
+
// Legacy column: the payload lives in workflow_input.
|
|
1404
|
+
null,
|
|
1405
|
+
null,
|
|
1406
|
+
status.priority,
|
|
1407
|
+
status.queuePartitionKey ?? null,
|
|
1408
|
+
status.parentWorkflowID ?? null,
|
|
1409
|
+
status.serialization,
|
|
1410
|
+
null,
|
|
1411
|
+
status.delayUntilEpochMS ?? null,
|
|
1412
|
+
status.attributes ? JSON.stringify(status.attributes) : null,
|
|
1413
|
+
status.scheduleName ?? null,
|
|
1414
|
+
status.applicationName ?? null,
|
|
1415
|
+
];
|
|
1416
|
+
});
|
|
1417
|
+
const returned = await this.insertRows(client, 'workflow_status', [...Object.keys(keyed[0]), ...columns], rows, `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING RETURNING workflow_uuid`);
|
|
1418
|
+
for (const row of returned) {
|
|
1060
1419
|
inserted.add(row.workflow_uuid);
|
|
1061
1420
|
}
|
|
1062
1421
|
}
|
|
1063
1422
|
for (let start = 0; start < statuses.length; start += chunkSize) {
|
|
1064
1423
|
const chunk = statuses.slice(start, start + chunkSize);
|
|
1065
|
-
const
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
}
|
|
1072
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
|
|
1073
|
-
VALUES ${tuples.join(', ')}
|
|
1074
|
-
ON CONFLICT (workflow_uuid) DO NOTHING`, params);
|
|
1424
|
+
const keyed = chunk.map((status) => this.wfValues(status.workflowUUID, status.shardKey));
|
|
1425
|
+
await this.insertRows(client, 'workflow_input', [...Object.keys(keyed[0]), 'workflow_uuid', 'inputs', 'retention_timestamp'], chunk.map((status, i) => [
|
|
1426
|
+
...Object.values(keyed[i]),
|
|
1427
|
+
status.workflowUUID,
|
|
1428
|
+
status.input,
|
|
1429
|
+
createdAts[start + i],
|
|
1430
|
+
]), `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`);
|
|
1075
1431
|
}
|
|
1076
1432
|
await client.query('COMMIT');
|
|
1077
1433
|
}
|
|
@@ -1085,22 +1441,10 @@ class SystemDatabase {
|
|
|
1085
1441
|
return inserted;
|
|
1086
1442
|
}
|
|
1087
1443
|
async recordWorkflowOutput(workflowID, status, ownerXid) {
|
|
1088
|
-
|
|
1089
|
-
try {
|
|
1090
|
-
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
|
|
1091
|
-
}
|
|
1092
|
-
finally {
|
|
1093
|
-
client.release();
|
|
1094
|
-
}
|
|
1444
|
+
return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
|
|
1095
1445
|
}
|
|
1096
1446
|
async recordWorkflowError(workflowID, status, ownerXid) {
|
|
1097
|
-
|
|
1098
|
-
try {
|
|
1099
|
-
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
|
|
1100
|
-
}
|
|
1101
|
-
finally {
|
|
1102
|
-
client.release();
|
|
1103
|
-
}
|
|
1447
|
+
return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
|
|
1104
1448
|
}
|
|
1105
1449
|
// Record a workflow's terminal outcome (SUCCESS or ERROR), reporting whether
|
|
1106
1450
|
// the write landed. The write applies only to a PENDING row still owned by
|
|
@@ -1111,8 +1455,49 @@ class SystemDatabase {
|
|
|
1111
1455
|
// terminal, handed to another execution (e.g. by a concurrent resume or
|
|
1112
1456
|
// recovery), or deleted; the caller resolves which by awaiting the recorded
|
|
1113
1457
|
// outcome.
|
|
1114
|
-
async #recordWorkflowOutcome(
|
|
1458
|
+
async #recordWorkflowOutcome(workflowID, status, outcome, ownerXid) {
|
|
1115
1459
|
ownerXid ??= (0, context_1.currentOwnerXid)(workflowID);
|
|
1460
|
+
if (!(await this.outcomeInOneStatement())) {
|
|
1461
|
+
return await this.#recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid);
|
|
1462
|
+
}
|
|
1463
|
+
// One autocommit round trip; the upsert reads the UPDATE's rows, so the output lands only with the status.
|
|
1464
|
+
const params = [status, workflow_1.StatusString.PENDING, outcome.output ?? null, outcome.error ?? null];
|
|
1465
|
+
const target = this.wf(undefined, workflowID, params);
|
|
1466
|
+
const owned = ownerXid !== undefined ? `AND owner_xid = $${params.push(ownerXid)}` : '';
|
|
1467
|
+
const keys = this.wfValues(workflowID);
|
|
1468
|
+
const keyColumns = Object.keys(keys);
|
|
1469
|
+
const keyExprs = Object.values(keys).map((value) => `$${params.push(value)}::text`);
|
|
1470
|
+
const workflowParam = `$${params.push(workflowID)}::text`;
|
|
1471
|
+
const { rows } = await this.pool.query(`WITH updated AS (
|
|
1472
|
+
UPDATE "${this.schemaName}".workflow_status
|
|
1473
|
+
SET status = $1,
|
|
1474
|
+
-- As the workflow is complete, remove its deduplication ID; a finished workflow has no owner.
|
|
1475
|
+
deduplication_id = NULL,
|
|
1476
|
+
owner_xid = NULL,
|
|
1477
|
+
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1478
|
+
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1479
|
+
WHERE ${target} AND status = $2 ${owned}
|
|
1480
|
+
RETURNING 1
|
|
1481
|
+
), stored AS (
|
|
1482
|
+
INSERT INTO "${this.schemaName}".workflow_output (${[...keyColumns, 'workflow_uuid', 'output', 'error'].join(', ')})
|
|
1483
|
+
SELECT ${[...keyExprs, workflowParam, '$3::text', '$4::text'].join(', ')}
|
|
1484
|
+
WHERE EXISTS (SELECT 1 FROM updated)
|
|
1485
|
+
-- EXCLUDED is the row this INSERT proposed, so these are the values above.
|
|
1486
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error
|
|
1487
|
+
-- CockroachDB refuses a data-modifying CTE without RETURNING.
|
|
1488
|
+
RETURNING 1
|
|
1489
|
+
)
|
|
1490
|
+
-- Selecting from the CTEs rather than inserting at the top level lets a router see the shard key.
|
|
1491
|
+
SELECT count(*) AS count FROM updated`, params);
|
|
1492
|
+
return Number(rows[0]?.count ?? 0) > 0;
|
|
1493
|
+
}
|
|
1494
|
+
/** Whether a workflow's outcome is written in one statement, rather than a transaction of two. */
|
|
1495
|
+
outcomeInOneStatement() {
|
|
1496
|
+
return Promise.resolve(true);
|
|
1497
|
+
}
|
|
1498
|
+
/** The outcome write as a transaction of two statements, for a backend that cannot run it as one. */
|
|
1499
|
+
async #recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid) {
|
|
1500
|
+
const client = await this.connect();
|
|
1116
1501
|
let committed = false;
|
|
1117
1502
|
try {
|
|
1118
1503
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
@@ -1126,19 +1511,31 @@ class SystemDatabase {
|
|
|
1126
1511
|
// The outcome was not ours to write, so leave no orphan payload.
|
|
1127
1512
|
return false;
|
|
1128
1513
|
}
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1514
|
+
const values = {
|
|
1515
|
+
...this.wfValues(workflowID),
|
|
1516
|
+
workflow_uuid: workflowID,
|
|
1517
|
+
output: outcome.output ?? null,
|
|
1518
|
+
error: outcome.error ?? null,
|
|
1519
|
+
};
|
|
1520
|
+
const columns = Object.keys(values);
|
|
1521
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_output (${columns.join(', ')})
|
|
1522
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
1523
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error`, Object.values(values));
|
|
1132
1524
|
committed = true;
|
|
1133
1525
|
return true;
|
|
1134
1526
|
}
|
|
1135
1527
|
finally {
|
|
1136
|
-
|
|
1137
|
-
|
|
1528
|
+
try {
|
|
1529
|
+
if (committed) {
|
|
1530
|
+
await client.query('COMMIT');
|
|
1531
|
+
}
|
|
1532
|
+
else {
|
|
1533
|
+
// Swallowed: a rollback failure must not mask why we are rolling back.
|
|
1534
|
+
await client.query('ROLLBACK').catch(() => undefined);
|
|
1535
|
+
}
|
|
1138
1536
|
}
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
await client.query('ROLLBACK').catch(() => undefined);
|
|
1537
|
+
finally {
|
|
1538
|
+
client.release();
|
|
1142
1539
|
}
|
|
1143
1540
|
}
|
|
1144
1541
|
}
|
|
@@ -1147,6 +1544,7 @@ class SystemDatabase {
|
|
|
1147
1544
|
const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
|
|
1148
1545
|
// executor_id defaults to "local", so it collides across applications.
|
|
1149
1546
|
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
1547
|
+
const keyColumn = this.keyColumn(undefined);
|
|
1150
1548
|
const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1151
1549
|
SET started_at_epoch_ms = NULL,
|
|
1152
1550
|
status = $1,
|
|
@@ -1157,27 +1555,31 @@ class SystemDatabase {
|
|
|
1157
1555
|
AND executor_id = $4
|
|
1158
1556
|
AND application_version = $5
|
|
1159
1557
|
AND ${scope}
|
|
1160
|
-
RETURNING workflow_uuid`, params);
|
|
1161
|
-
return result.rows.map((row) => row.workflow_uuid);
|
|
1558
|
+
RETURNING workflow_uuid${keyColumn !== undefined ? `, ${keyColumn} AS shard_key` : ''}`, params);
|
|
1559
|
+
return result.rows.map((row) => ({ workflowID: row.workflow_uuid, shardKey: row.shard_key ?? undefined }));
|
|
1162
1560
|
}
|
|
1163
1561
|
async getWorkflowStatus(workflowID, callerID, callerFN) {
|
|
1562
|
+
// Resolved before binding the caller's own key, which a same-ID target would share.
|
|
1563
|
+
const shardKey = this.keyHint(workflowID);
|
|
1164
1564
|
// Inside a workflow this runs on the transaction's client: taking a second pool connection while
|
|
1165
1565
|
// holding the first deadlocks the pool once every connection is held by a waiting caller.
|
|
1166
1566
|
const funcGetStatus = async (client) => {
|
|
1167
|
-
const statuses = await this.#listWorkflows({ workflowIDs: [workflowID] }, client);
|
|
1567
|
+
const statuses = await this.#listWorkflows({ workflowIDs: [workflowID], shardKey }, {}, client);
|
|
1168
1568
|
const status = statuses.find((s) => s.workflowUUID === workflowID);
|
|
1169
1569
|
return status ? JSON.stringify(status) : null;
|
|
1170
1570
|
};
|
|
1171
1571
|
if (callerID && callerFN !== undefined) {
|
|
1172
|
-
const
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1572
|
+
const json = await this.ownKey(callerID, undefined, async () => {
|
|
1573
|
+
const client = await this.connect();
|
|
1574
|
+
try {
|
|
1575
|
+
// Check if the operation has been done before for OAOO (only do this inside a workflow).
|
|
1576
|
+
return await this.inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, () => funcGetStatus(client)));
|
|
1577
|
+
}
|
|
1578
|
+
finally {
|
|
1579
|
+
client.release();
|
|
1580
|
+
}
|
|
1581
|
+
});
|
|
1582
|
+
return parseStatus(json);
|
|
1181
1583
|
}
|
|
1182
1584
|
else {
|
|
1183
1585
|
const json = await funcGetStatus();
|
|
@@ -1191,7 +1593,7 @@ class SystemDatabase {
|
|
|
1191
1593
|
statusFetchChunkSize = 500;
|
|
1192
1594
|
// Retried per chunk so a reconnect refetches one chunk, not every chunk before it.
|
|
1193
1595
|
async fetchWorkflowStatusChunk(workflowIDs) {
|
|
1194
|
-
return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false });
|
|
1596
|
+
return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false }, { keyedIDs: true });
|
|
1195
1597
|
}
|
|
1196
1598
|
/** Fetch many statuses in as few round trips as possible. IDs with no row are omitted. */
|
|
1197
1599
|
async getWorkflowStatuses(workflowIDs) {
|
|
@@ -1222,23 +1624,26 @@ class SystemDatabase {
|
|
|
1222
1624
|
}
|
|
1223
1625
|
}
|
|
1224
1626
|
// ==================== Step Results ====================
|
|
1225
|
-
async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1627
|
+
async getOperationResultAndThrowIfCancelled(workflowID, functionID, ownerKey) {
|
|
1628
|
+
return await this.ownKey(workflowID, ownerKey, async () => {
|
|
1629
|
+
const client = await this.connect();
|
|
1630
|
+
try {
|
|
1631
|
+
return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
1632
|
+
}
|
|
1633
|
+
finally {
|
|
1634
|
+
client.release();
|
|
1635
|
+
}
|
|
1636
|
+
});
|
|
1233
1637
|
}
|
|
1234
1638
|
async getAllOperationResults(workflowID, limit, offset, loadOutput = true) {
|
|
1235
1639
|
// Skip the output/error payload columns when the caller does not need them,
|
|
1236
1640
|
// mirroring how listWorkflows skips its payload joins when loadOutput is false.
|
|
1641
|
+
const childKey = this.keyColumn(undefined, 'child_shard_key');
|
|
1237
1642
|
const columns = loadOutput
|
|
1238
1643
|
? '*'
|
|
1239
|
-
:
|
|
1240
|
-
|
|
1241
|
-
|
|
1644
|
+
: `workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? `, ${childKey}` : ''}`;
|
|
1645
|
+
const params = [];
|
|
1646
|
+
let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE ${this.wf(undefined, workflowID, params)} ORDER BY function_id`;
|
|
1242
1647
|
if (limit !== undefined) {
|
|
1243
1648
|
params.push(limit);
|
|
1244
1649
|
query += ` LIMIT $${params.length}`;
|
|
@@ -1251,14 +1656,18 @@ class SystemDatabase {
|
|
|
1251
1656
|
return rows;
|
|
1252
1657
|
}
|
|
1253
1658
|
async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1659
|
+
// Resolved before binding the caller's own key, which a same-ID child would share.
|
|
1660
|
+
const childValues = this.childValues(options.childWorkflowID);
|
|
1661
|
+
await this.ownKey(workflowID, options.ownerKey, async () => {
|
|
1662
|
+
const client = await this.connect();
|
|
1663
|
+
try {
|
|
1664
|
+
await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, { ...options, childValues }));
|
|
1665
|
+
}
|
|
1666
|
+
finally {
|
|
1667
|
+
client.release();
|
|
1668
|
+
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
|
|
1669
|
+
}
|
|
1670
|
+
});
|
|
1262
1671
|
}
|
|
1263
1672
|
/** Checkpoint an operation's failure so a replay re-throws it instead of re-running. */
|
|
1264
1673
|
async recordOperationError(workflowID, functionID, functionName, startTimeEpochMs, error, childWorkflowID) {
|
|
@@ -1332,9 +1741,10 @@ class SystemDatabase {
|
|
|
1332
1741
|
throw new TypeError('functionID must be defined');
|
|
1333
1742
|
patchName = `DBOS.patch-${patchName}`;
|
|
1334
1743
|
const readCheckpointName = async (queryable) => {
|
|
1744
|
+
const params = [functionID];
|
|
1335
1745
|
const { rows } = await queryable.query(`SELECT function_name
|
|
1336
1746
|
FROM "${this.schemaName}".operation_outputs
|
|
1337
|
-
WHERE
|
|
1747
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
|
|
1338
1748
|
return rows.length === 0 ? undefined : rows[0].function_name;
|
|
1339
1749
|
};
|
|
1340
1750
|
if (deprecated) {
|
|
@@ -1349,7 +1759,7 @@ class SystemDatabase {
|
|
|
1349
1759
|
// If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
|
|
1350
1760
|
const client = await this.connect();
|
|
1351
1761
|
try {
|
|
1352
|
-
return await this
|
|
1762
|
+
return await this.inTransaction(client, async () => {
|
|
1353
1763
|
const checkpointName = await readCheckpointName(client);
|
|
1354
1764
|
if (checkpointName !== undefined) {
|
|
1355
1765
|
const matches = checkpointName === patchName;
|
|
@@ -1375,7 +1785,7 @@ class SystemDatabase {
|
|
|
1375
1785
|
let frontier = workflowIDs;
|
|
1376
1786
|
while (frontier.length > 0) {
|
|
1377
1787
|
await this.#cancelWorkflows(frontier);
|
|
1378
|
-
const children = await this
|
|
1788
|
+
const children = await this.getDirectChildren(frontier);
|
|
1379
1789
|
frontier = children.filter((id) => !visited.has(id));
|
|
1380
1790
|
for (const id of frontier) {
|
|
1381
1791
|
visited.add(id);
|
|
@@ -1383,41 +1793,44 @@ class SystemDatabase {
|
|
|
1383
1793
|
}
|
|
1384
1794
|
}
|
|
1385
1795
|
async #cancelWorkflows(workflowIDs) {
|
|
1796
|
+
const params = [workflow_1.StatusString.CANCELLED, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR];
|
|
1386
1797
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1387
1798
|
SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
1388
1799
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1389
1800
|
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1390
|
-
WHERE
|
|
1391
|
-
AND status NOT IN ($
|
|
1801
|
+
WHERE ${this.wfIn(undefined, workflowIDs, params)}
|
|
1802
|
+
AND status NOT IN ($2, $3)`, params);
|
|
1392
1803
|
}
|
|
1393
1804
|
async checkIfCanceled(workflowID) {
|
|
1394
|
-
|
|
1805
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
1806
|
+
await this.ownKey(workflowID, undefined, () => this.#checkIfCanceled(this.pool, workflowID));
|
|
1395
1807
|
}
|
|
1396
1808
|
async resumeWorkflows(workflowIDs, queueName) {
|
|
1397
1809
|
const client = await this.connect();
|
|
1398
1810
|
try {
|
|
1399
1811
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1400
1812
|
// Check existence separately: a zero-row update also means "already complete", a legal no-op.
|
|
1401
|
-
const
|
|
1813
|
+
const existingParams = [];
|
|
1814
|
+
const { rows } = await client.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${this.wfIn(undefined, workflowIDs, existingParams)}`, existingParams);
|
|
1402
1815
|
const existing = new Set(rows.map((r) => r.workflow_uuid));
|
|
1403
1816
|
const missing = workflowIDs.filter((id) => !existing.has(id));
|
|
1404
1817
|
if (missing.length > 0) {
|
|
1405
1818
|
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${missing.join(', ')} does not exist`);
|
|
1406
1819
|
}
|
|
1820
|
+
const updateParams = [
|
|
1821
|
+
workflow_1.StatusString.ENQUEUED,
|
|
1822
|
+
queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1823
|
+
workflow_1.StatusString.SUCCESS,
|
|
1824
|
+
workflow_1.StatusString.ERROR,
|
|
1825
|
+
];
|
|
1407
1826
|
await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1408
1827
|
SET status = $1, owner_xid = NULL, queue_name = $2, recovery_attempts = 0,
|
|
1409
1828
|
workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
|
|
1410
1829
|
started_at_epoch_ms = NULL,
|
|
1411
1830
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1412
1831
|
completed_at = NULL
|
|
1413
|
-
WHERE
|
|
1414
|
-
AND status NOT IN ($
|
|
1415
|
-
workflow_1.StatusString.ENQUEUED,
|
|
1416
|
-
queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1417
|
-
workflowIDs,
|
|
1418
|
-
workflow_1.StatusString.SUCCESS,
|
|
1419
|
-
workflow_1.StatusString.ERROR,
|
|
1420
|
-
]);
|
|
1832
|
+
WHERE ${this.wfIn(undefined, workflowIDs, updateParams)}
|
|
1833
|
+
AND status NOT IN ($3, $4)`, updateParams);
|
|
1421
1834
|
await client.query('COMMIT');
|
|
1422
1835
|
}
|
|
1423
1836
|
catch (e) {
|
|
@@ -1430,16 +1843,18 @@ class SystemDatabase {
|
|
|
1430
1843
|
}
|
|
1431
1844
|
}
|
|
1432
1845
|
async setWorkflowPriority(workflowID, priority) {
|
|
1846
|
+
const params = [priority, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED];
|
|
1433
1847
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1434
1848
|
SET priority = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1435
|
-
WHERE
|
|
1436
|
-
AND status IN ($
|
|
1849
|
+
WHERE ${this.wf(undefined, workflowID, params)}
|
|
1850
|
+
AND status IN ($2, $3)`, params);
|
|
1437
1851
|
}
|
|
1438
1852
|
async setWorkflowDelay(workflowID, delayUntilEpochMS) {
|
|
1853
|
+
const params = [delayUntilEpochMS, workflow_1.StatusString.DELAYED];
|
|
1439
1854
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1440
1855
|
SET delay_until_epoch_ms = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1441
|
-
WHERE
|
|
1442
|
-
AND status = $
|
|
1856
|
+
WHERE ${this.wf(undefined, workflowID, params)}
|
|
1857
|
+
AND status = $2`, params);
|
|
1443
1858
|
}
|
|
1444
1859
|
/**
|
|
1445
1860
|
* Extend an existing debounced DELAYED workflow's delay and update its inputs, atomically.
|
|
@@ -1505,7 +1920,7 @@ class SystemDatabase {
|
|
|
1505
1920
|
if (updated.rows.length > 0) {
|
|
1506
1921
|
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
|
|
1507
1922
|
VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
|
|
1508
|
-
ON CONFLICT (workflow_uuid) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
|
|
1923
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
|
|
1509
1924
|
return {
|
|
1510
1925
|
bouncedWorkflowID: updated.rows[0].workflow_uuid,
|
|
1511
1926
|
holderWorkflowID: null,
|
|
@@ -1515,10 +1930,11 @@ class SystemDatabase {
|
|
|
1515
1930
|
holderApplicationName: null,
|
|
1516
1931
|
};
|
|
1517
1932
|
}
|
|
1518
|
-
//
|
|
1933
|
+
// No application filter, so a holder that blocked the update above is reportable.
|
|
1934
|
+
const holderParams = [params.queueName, params.deduplicationID];
|
|
1519
1935
|
const holder = await client.query(`SELECT workflow_uuid, is_debounced, name, class_name, application_name
|
|
1520
1936
|
FROM "${this.schemaName}".workflow_status
|
|
1521
|
-
WHERE queue_name = $1 AND deduplication_id = $2`,
|
|
1937
|
+
WHERE ${this.scoped(undefined, holderParams)} AND queue_name = $1 AND deduplication_id = $2`, holderParams);
|
|
1522
1938
|
if (holder.rows.length === 0) {
|
|
1523
1939
|
return {
|
|
1524
1940
|
bouncedWorkflowID: null,
|
|
@@ -1539,7 +1955,7 @@ class SystemDatabase {
|
|
|
1539
1955
|
};
|
|
1540
1956
|
}
|
|
1541
1957
|
// Get the immediate (one-level) child workflow IDs for a set of workflows.
|
|
1542
|
-
async
|
|
1958
|
+
async getDirectChildren(workflowIDs) {
|
|
1543
1959
|
if (workflowIDs.length === 0) {
|
|
1544
1960
|
return [];
|
|
1545
1961
|
}
|
|
@@ -1553,7 +1969,7 @@ class SystemDatabase {
|
|
|
1553
1969
|
const descendants = new Set();
|
|
1554
1970
|
let frontier = [workflowID];
|
|
1555
1971
|
while (frontier.length > 0) {
|
|
1556
|
-
const children = await this
|
|
1972
|
+
const children = await this.getDirectChildren(frontier);
|
|
1557
1973
|
frontier = children.filter((id) => !descendants.has(id));
|
|
1558
1974
|
for (const id of frontier) {
|
|
1559
1975
|
descendants.add(id);
|
|
@@ -1561,6 +1977,7 @@ class SystemDatabase {
|
|
|
1561
1977
|
}
|
|
1562
1978
|
return [...descendants];
|
|
1563
1979
|
}
|
|
1980
|
+
/** Delete workflows and all associated data, and with deleteChildren every workflow they started, recursively. */
|
|
1564
1981
|
async deleteWorkflows(workflowIDs, deleteChildren = false) {
|
|
1565
1982
|
const allIds = [...workflowIDs];
|
|
1566
1983
|
if (deleteChildren) {
|
|
@@ -1568,15 +1985,18 @@ class SystemDatabase {
|
|
|
1568
1985
|
allIds.push(...(await this.getWorkflowChildren(wfid)));
|
|
1569
1986
|
}
|
|
1570
1987
|
}
|
|
1988
|
+
await this.deleteWorkflowsInternal(allIds);
|
|
1989
|
+
}
|
|
1990
|
+
async deleteWorkflowsInternal(allIds) {
|
|
1571
1991
|
const client = await this.connect();
|
|
1572
1992
|
try {
|
|
1573
1993
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1574
1994
|
// The payload tables carry no foreign key, so the status delete does not cascade
|
|
1575
1995
|
// into them; without this they survive as orphans.
|
|
1576
|
-
for (const table of ['workflow_input', 'workflow_output', 'operation_outputs']) {
|
|
1577
|
-
|
|
1996
|
+
for (const table of ['workflow_input', 'workflow_output', 'operation_outputs', 'workflow_status']) {
|
|
1997
|
+
const params = [];
|
|
1998
|
+
await client.query(`DELETE FROM "${this.schemaName}".${table} WHERE ${this.wfIn(undefined, allIds, params)}`, params);
|
|
1578
1999
|
}
|
|
1579
|
-
await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [allIds]);
|
|
1580
2000
|
await client.query('COMMIT');
|
|
1581
2001
|
}
|
|
1582
2002
|
catch (e) {
|
|
@@ -1617,7 +2037,8 @@ class SystemDatabase {
|
|
|
1617
2037
|
const client = await this.connect();
|
|
1618
2038
|
try {
|
|
1619
2039
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1620
|
-
const
|
|
2040
|
+
const statusParams = [];
|
|
2041
|
+
const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE ${this.wf(undefined, workflowID, statusParams)}`, statusParams);
|
|
1621
2042
|
if (statusRows.length === 0) {
|
|
1622
2043
|
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
|
|
1623
2044
|
}
|
|
@@ -1628,56 +2049,61 @@ class SystemDatabase {
|
|
|
1628
2049
|
// Roll workflow_events back to the last value published before the cut, using
|
|
1629
2050
|
// workflow_events_history as an undo log, then rewind that history too.
|
|
1630
2051
|
// `publishedPastCut` is the correlated EXISTS both halves key off.
|
|
1631
|
-
const publishedPastCut = `EXISTS (
|
|
2052
|
+
const publishedPastCut = (key, params) => `EXISTS (
|
|
1632
2053
|
SELECT 1 FROM "${schema}".workflow_events_history discarded
|
|
1633
|
-
WHERE discarded
|
|
2054
|
+
WHERE ${this.wf('discarded', workflowID, params)} AND discarded.key = ${key} AND discarded.function_id >= $1
|
|
1634
2055
|
)`;
|
|
1635
2056
|
// First unpublish everything the discarded run touched.
|
|
2057
|
+
const unpublishParams = [startStep];
|
|
1636
2058
|
await client.query(`DELETE FROM "${schema}".workflow_events
|
|
1637
|
-
WHERE
|
|
2059
|
+
WHERE ${this.wf(undefined, workflowID, unpublishParams)}
|
|
2060
|
+
AND ${publishedPastCut('"' + schema + '".workflow_events.key', unpublishParams)}`, unpublishParams);
|
|
1638
2061
|
// Then restore those keys, if any, to the value published below the cut.
|
|
1639
|
-
|
|
1640
|
-
|
|
2062
|
+
const restoreParams = [startStep];
|
|
2063
|
+
const keyColumns = this.wfColumns(workflowID, restoreParams);
|
|
2064
|
+
await client.query(`INSERT INTO "${schema}".workflow_events (${[...keyColumns.map(([c]) => c), 'workflow_uuid', 'key', 'value', 'serialization'].join(', ')})
|
|
2065
|
+
SELECT ${[...keyColumns.map(([, e]) => e), 'surviving.workflow_uuid', 'surviving.key', 'surviving.value', 'surviving.serialization'].join(', ')}
|
|
1641
2066
|
FROM (
|
|
1642
2067
|
SELECT weh.workflow_uuid, weh.key, weh.value, weh.serialization,
|
|
1643
2068
|
ROW_NUMBER() OVER (PARTITION BY weh.key ORDER BY weh.function_id DESC) AS rn
|
|
1644
2069
|
FROM "${schema}".workflow_events_history weh
|
|
1645
|
-
WHERE weh
|
|
1646
|
-
AND ${publishedPastCut
|
|
2070
|
+
WHERE ${this.wf('weh', workflowID, restoreParams)} AND weh.function_id < $1
|
|
2071
|
+
AND ${publishedPastCut('weh.key', restoreParams)}
|
|
1647
2072
|
) surviving
|
|
1648
|
-
WHERE surviving.rn = 1`,
|
|
2073
|
+
WHERE surviving.rn = 1`, restoreParams);
|
|
1649
2074
|
// Clear the streams' close sentinels so the replay can append again.
|
|
2075
|
+
const sentinelParams = [startStep, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name()];
|
|
1650
2076
|
await client.query(`DELETE FROM "${schema}".streams
|
|
1651
|
-
WHERE
|
|
2077
|
+
WHERE ${this.wf(undefined, workflowID, sentinelParams)} AND function_id >= $1 AND value = $2 AND serialization = $3`, sentinelParams);
|
|
1652
2078
|
// Discard the steps and the events history past the cut.
|
|
1653
2079
|
for (const table of ['operation_outputs', 'workflow_events_history']) {
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
startStep,
|
|
1657
|
-
]);
|
|
2080
|
+
const params = [startStep];
|
|
2081
|
+
await client.query(`DELETE FROM "${schema}".${table} WHERE ${this.wf(undefined, workflowID, params)} AND function_id >= $1`, params);
|
|
1658
2082
|
}
|
|
1659
2083
|
// Delete messages consumed or received after the rewind point
|
|
1660
|
-
|
|
1661
|
-
await client.query(`DELETE FROM "${schema}".
|
|
2084
|
+
const messageParams = [startStep];
|
|
2085
|
+
await client.query(`DELETE FROM "${schema}".notifications
|
|
2086
|
+
WHERE ${this.wf(undefined, workflowID, messageParams, 'destination_uuid')}
|
|
2087
|
+
AND (consumed_by_function_id >= $1 OR consumed = false)`, messageParams);
|
|
2088
|
+
const outputParams = [];
|
|
2089
|
+
await client.query(`DELETE FROM "${schema}".workflow_output WHERE ${this.wf(undefined, workflowID, outputParams)}`, outputParams);
|
|
1662
2090
|
// Re-enqueue the workflow. Re-asserting the status we read keeps a workflow that
|
|
1663
2091
|
// moved on underneath us from being resurrected.
|
|
1664
|
-
const setVersion = options.applicationVersion !== undefined ? ', application_version = $6' : '';
|
|
1665
2092
|
const params = [
|
|
1666
2093
|
workflow_1.StatusString.ENQUEUED,
|
|
1667
2094
|
options.queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1668
2095
|
options.queuePartitionKey ?? null,
|
|
1669
|
-
workflowID,
|
|
1670
2096
|
status,
|
|
1671
2097
|
];
|
|
1672
|
-
|
|
1673
|
-
params.push(options.applicationVersion)
|
|
1674
|
-
|
|
2098
|
+
const setVersion = options.applicationVersion !== undefined
|
|
2099
|
+
? `, application_version = $${params.push(options.applicationVersion)}`
|
|
2100
|
+
: '';
|
|
1675
2101
|
const { rowCount } = await client.query(`UPDATE "${schema}".workflow_status
|
|
1676
2102
|
SET status = $1, owner_xid = NULL, queue_name = $2, queue_partition_key = $3, recovery_attempts = 0,
|
|
1677
2103
|
workflow_deadline_epoch_ms = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
1678
2104
|
completed_at = NULL, output = NULL, error = NULL,
|
|
1679
2105
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint${setVersion}
|
|
1680
|
-
WHERE
|
|
2106
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND status = $4`, params);
|
|
1681
2107
|
if (rowCount !== 1) {
|
|
1682
2108
|
throw new error_1.DBOSError(`Workflow ${workflowID} changed status while being rewound; retry the rewind`);
|
|
1683
2109
|
}
|
|
@@ -1707,13 +2133,14 @@ class SystemDatabase {
|
|
|
1707
2133
|
try {
|
|
1708
2134
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1709
2135
|
// Fetch the status of all original workflows inside the transaction.
|
|
2136
|
+
const statusParams = [];
|
|
1710
2137
|
const { rows: statusRows } = await client.query(`SELECT ws.workflow_uuid, ws.name, ws.class_name, ws.config_name, ws.application_id,
|
|
1711
2138
|
ws.authenticated_user, ws.authenticated_roles, ws.assumed_role,
|
|
1712
2139
|
COALESCE(wi.inputs, ws.inputs) AS inputs, ws.serialization,
|
|
1713
2140
|
ws.application_version, ws.attributes, ws.application_name
|
|
1714
2141
|
FROM "${this.schemaName}".workflow_status ws
|
|
1715
|
-
LEFT JOIN "${this.schemaName}".workflow_input wi ON
|
|
1716
|
-
WHERE ws
|
|
2142
|
+
LEFT JOIN "${this.schemaName}".workflow_input wi ON ${this.wfJoin('ws', 'wi')}
|
|
2143
|
+
WHERE ${this.wfIn('ws', originalWorkflowIDs, statusParams)}`, statusParams);
|
|
1717
2144
|
const statusByID = new Map(statusRows.map((r) => [r.workflow_uuid, r]));
|
|
1718
2145
|
for (const wid of originalWorkflowIDs) {
|
|
1719
2146
|
if (!statusByID.has(wid)) {
|
|
@@ -1749,35 +2176,42 @@ class SystemDatabase {
|
|
|
1749
2176
|
forkedWorkflowIDs[i],
|
|
1750
2177
|
statusByID.get(origID).application_name ?? this.appName ?? null,
|
|
1751
2178
|
]));
|
|
1752
|
-
const
|
|
1753
|
-
|
|
1754
|
-
let paramIdx = 1;
|
|
1755
|
-
for (let i = 0; i < originalWorkflowIDs.length; i++) {
|
|
1756
|
-
const origID = originalWorkflowIDs[i];
|
|
2179
|
+
const forkKeys = forkedWorkflowIDs.map((forkID) => this.wfValues(forkID));
|
|
2180
|
+
await this.insertRows(client, 'workflow_status', [...Object.keys(forkKeys[0]), ...insertCols], originalWorkflowIDs.map((origID, i) => {
|
|
1757
2181
|
const forkID = forkedWorkflowIDs[i];
|
|
1758
2182
|
const ws = statusByID.get(origID);
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
2183
|
+
return [
|
|
2184
|
+
...Object.values(forkKeys[i]),
|
|
2185
|
+
forkID,
|
|
2186
|
+
workflow_1.StatusString.ENQUEUED,
|
|
2187
|
+
ws.name,
|
|
2188
|
+
ws.class_name ?? null,
|
|
2189
|
+
ws.config_name ?? null,
|
|
2190
|
+
queueName,
|
|
2191
|
+
ws.authenticated_user,
|
|
2192
|
+
ws.assumed_role,
|
|
2193
|
+
ws.authenticated_roles,
|
|
2194
|
+
options.applicationVersion ?? ws.application_version ?? null,
|
|
2195
|
+
ws.application_id,
|
|
2196
|
+
// Legacy column: the payload lives in workflow_input.
|
|
2197
|
+
null,
|
|
2198
|
+
options.queuePartitionKey ?? null,
|
|
2199
|
+
origID,
|
|
2200
|
+
ws.serialization,
|
|
2201
|
+
ws.attributes ? JSON.stringify(ws.attributes) : null,
|
|
2202
|
+
forkOwners.get(forkID) ?? null,
|
|
2203
|
+
...(options.timeoutMS !== undefined ? [options.timeoutMS] : []),
|
|
2204
|
+
];
|
|
2205
|
+
}));
|
|
2206
|
+
await this.insertRows(client, 'workflow_input', [...Object.keys(forkKeys[0]), 'workflow_uuid', 'inputs'], originalWorkflowIDs.map((origID, i) => [
|
|
2207
|
+
...Object.values(forkKeys[i]),
|
|
2208
|
+
forkedWorkflowIDs[i],
|
|
2209
|
+
statusByID.get(origID).inputs,
|
|
2210
|
+
]));
|
|
1779
2211
|
// Mark all original workflows as having been forked from.
|
|
1780
|
-
|
|
2212
|
+
const forkedFromParams = [];
|
|
2213
|
+
await client.query(`UPDATE "${this.schemaName}".workflow_status SET was_forked_from = TRUE
|
|
2214
|
+
WHERE ${this.wfIn(undefined, originalWorkflowIDs, forkedFromParams)}`, forkedFromParams);
|
|
1781
2215
|
// For workflows with start_step > 0, copy checkpoints/events/streams.
|
|
1782
2216
|
// Build a mapping CTE so each copy is a single SQL statement.
|
|
1783
2217
|
const forkMappings = originalWorkflowIDs
|
|
@@ -1794,55 +2228,85 @@ class SystemDatabase {
|
|
|
1794
2228
|
mIdx += 4;
|
|
1795
2229
|
}
|
|
1796
2230
|
const mappingCTE = `WITH mapping(orig_id, fork_id, start_step, owner) AS (VALUES ${mappingValues.join(', ')})`;
|
|
2231
|
+
// The forks live under their originals' shard key, read from the scope; a copy's params extend the mapping's.
|
|
2232
|
+
const copyParams = () => [...mappingParams];
|
|
2233
|
+
const keyed = (params) => {
|
|
2234
|
+
const columns = this.scopedColumns(params);
|
|
2235
|
+
return { names: columns.map(([c]) => `${c}, `).join(''), exprs: columns.map(([, e]) => `${e}, `).join('') };
|
|
2236
|
+
};
|
|
2237
|
+
// A replacement child is a fork, so it shares the replaced child's key.
|
|
2238
|
+
const childKey = this.keyColumn('oo', 'child_shard_key');
|
|
2239
|
+
if (options.replacementChildren && childKey !== undefined) {
|
|
2240
|
+
// A replacement names a child by ID, which can't pick one of several keys.
|
|
2241
|
+
const ambiguityParams = copyParams();
|
|
2242
|
+
const replaced = `$${ambiguityParams.push(Object.keys(options.replacementChildren))}::text[]`;
|
|
2243
|
+
const { rows: ambiguous } = await client.query(`${mappingCTE}
|
|
2244
|
+
SELECT oo.child_workflow_id
|
|
2245
|
+
FROM mapping m
|
|
2246
|
+
JOIN "${this.schemaName}".operation_outputs oo
|
|
2247
|
+
ON ${this.scoped('oo', ambiguityParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step
|
|
2248
|
+
WHERE oo.child_workflow_id = ANY(${replaced})
|
|
2249
|
+
GROUP BY oo.child_workflow_id
|
|
2250
|
+
HAVING COUNT(DISTINCT ${childKey}) > 1
|
|
2251
|
+
LIMIT 1`, ambiguityParams);
|
|
2252
|
+
if (ambiguous.length > 0) {
|
|
2253
|
+
throw new error_1.DBOSError(`Can't replace child workflow ${ambiguous[0].child_workflow_id}: the forked steps refer to children with that ID under more than one shard key`);
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
1797
2256
|
// Build the child_workflow_id expression, applying replacements if provided.
|
|
2257
|
+
const ooParams = copyParams();
|
|
1798
2258
|
let childWfExpr = 'oo.child_workflow_id';
|
|
1799
|
-
const ooParams = [...mappingParams];
|
|
1800
2259
|
if (options.replacementChildren && Object.keys(options.replacementChildren).length > 0) {
|
|
1801
2260
|
const whenClauses = [];
|
|
1802
2261
|
for (const [oldId, newId] of Object.entries(options.replacementChildren)) {
|
|
1803
|
-
whenClauses.push(`WHEN oo.child_workflow_id = $${
|
|
1804
|
-
ooParams.push(oldId, newId);
|
|
1805
|
-
mIdx += 2;
|
|
2262
|
+
whenClauses.push(`WHEN oo.child_workflow_id = $${ooParams.push(oldId)} THEN $${ooParams.push(newId)}::text`);
|
|
1806
2263
|
}
|
|
1807
2264
|
childWfExpr = `CASE ${whenClauses.join(' ')} ELSE oo.child_workflow_id END`;
|
|
1808
2265
|
}
|
|
2266
|
+
const ooKeys = keyed(ooParams);
|
|
1809
2267
|
// Copy operation outputs
|
|
1810
2268
|
await client.query(`${mappingCTE}
|
|
1811
2269
|
INSERT INTO "${this.schemaName}".operation_outputs
|
|
1812
|
-
(workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp)
|
|
1813
|
-
SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
2270
|
+
(${ooKeys.names}workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? ', child_shard_key' : ''})
|
|
2271
|
+
SELECT ${ooKeys.exprs}m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint${childKey !== undefined ? `, ${childKey}` : ''}
|
|
1814
2272
|
FROM mapping m
|
|
1815
2273
|
JOIN "${this.schemaName}".operation_outputs oo
|
|
1816
|
-
ON oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
|
|
2274
|
+
ON ${this.scoped('oo', ooParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
|
|
1817
2275
|
// Copy streams
|
|
2276
|
+
const streamParams = copyParams();
|
|
2277
|
+
const streamKeys = keyed(streamParams);
|
|
1818
2278
|
await client.query(`${mappingCTE}
|
|
1819
2279
|
INSERT INTO "${this.schemaName}".streams
|
|
1820
|
-
(workflow_uuid, key, value, serialization, "offset", function_id)
|
|
1821
|
-
SELECT m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
|
|
2280
|
+
(${streamKeys.names}workflow_uuid, key, value, serialization, "offset", function_id)
|
|
2281
|
+
SELECT ${streamKeys.exprs}m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
|
|
1822
2282
|
FROM mapping m
|
|
1823
2283
|
JOIN "${this.schemaName}".streams s
|
|
1824
|
-
ON s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`,
|
|
2284
|
+
ON ${this.scoped('s', streamParams)} AND s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`, streamParams);
|
|
1825
2285
|
// Copy events history
|
|
2286
|
+
const historyParams = copyParams();
|
|
2287
|
+
const historyKeys = keyed(historyParams);
|
|
1826
2288
|
await client.query(`${mappingCTE}
|
|
1827
2289
|
INSERT INTO "${this.schemaName}".workflow_events_history
|
|
1828
|
-
(workflow_uuid, function_id, key, value, serialization)
|
|
1829
|
-
SELECT m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
|
|
2290
|
+
(${historyKeys.names}workflow_uuid, function_id, key, value, serialization)
|
|
2291
|
+
SELECT ${historyKeys.exprs}m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
|
|
1830
2292
|
FROM mapping m
|
|
1831
2293
|
JOIN "${this.schemaName}".workflow_events_history weh
|
|
1832
|
-
ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`,
|
|
2294
|
+
ON ${this.scoped('weh', historyParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`, historyParams);
|
|
1833
2295
|
// Copy only the latest version of each event using a window function
|
|
2296
|
+
const eventParams = copyParams();
|
|
2297
|
+
const eventKeys = keyed(eventParams);
|
|
1834
2298
|
await client.query(`${mappingCTE}
|
|
1835
2299
|
INSERT INTO "${this.schemaName}".workflow_events
|
|
1836
|
-
(workflow_uuid, key, value, serialization)
|
|
1837
|
-
SELECT ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
|
|
2300
|
+
(${eventKeys.names}workflow_uuid, key, value, serialization)
|
|
2301
|
+
SELECT ${eventKeys.exprs}ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
|
|
1838
2302
|
FROM (
|
|
1839
2303
|
SELECT m.fork_id AS workflow_uuid, weh.key, weh.value, weh.serialization,
|
|
1840
|
-
ROW_NUMBER() OVER (PARTITION BY
|
|
2304
|
+
ROW_NUMBER() OVER (PARTITION BY m.fork_id, weh.key ORDER BY weh.function_id DESC) AS rn
|
|
1841
2305
|
FROM mapping m
|
|
1842
2306
|
JOIN "${this.schemaName}".workflow_events_history weh
|
|
1843
|
-
ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
|
|
2307
|
+
ON ${this.scoped('weh', eventParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
|
|
1844
2308
|
) ranked
|
|
1845
|
-
WHERE ranked.rn = 1`,
|
|
2309
|
+
WHERE ranked.rn = 1`, eventParams);
|
|
1846
2310
|
}
|
|
1847
2311
|
await client.query('COMMIT');
|
|
1848
2312
|
return forkedWorkflowIDs;
|
|
@@ -1965,27 +2429,36 @@ class SystemDatabase {
|
|
|
1965
2429
|
}
|
|
1966
2430
|
/** Cancellation check for polling waits: goes through the limiter so readers cannot starve the pool. */
|
|
1967
2431
|
async checkIfCanceledLimited(workflowID) {
|
|
1968
|
-
|
|
2432
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
2433
|
+
const params = [];
|
|
2434
|
+
const target = this.ownKey(workflowID, undefined, () => this.wf(undefined, workflowID, params));
|
|
2435
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${target}`, params));
|
|
2436
|
+
if (rows[0]?.status === workflow_1.StatusString.CANCELLED) {
|
|
2437
|
+
throw new error_1.DBOSWorkflowCancelledError(workflowID);
|
|
2438
|
+
}
|
|
1969
2439
|
}
|
|
1970
2440
|
watchForCancellation(workflowID) {
|
|
1971
2441
|
const controller = new AbortController();
|
|
1972
2442
|
let state = 'idle';
|
|
2443
|
+
// The running workflow's own key, resolved now for the poller that reads it later outside this context.
|
|
2444
|
+
const shardKey = this.ownKey(workflowID, undefined, () => this.keyHint(workflowID));
|
|
2445
|
+
const waiterKey = this.ownKey(workflowID, undefined, () => this.waiter(workflowID, ''));
|
|
1973
2446
|
const start = () => {
|
|
1974
2447
|
state = 'watching';
|
|
1975
2448
|
if (this.#destroyed)
|
|
1976
2449
|
return;
|
|
1977
|
-
let watchers = this.#cancelWatchers.get(
|
|
2450
|
+
let watchers = this.#cancelWatchers.get(waiterKey);
|
|
1978
2451
|
if (watchers === undefined) {
|
|
1979
|
-
watchers = new Set();
|
|
1980
|
-
this.#cancelWatchers.set(
|
|
2452
|
+
watchers = { workflowID, shardKey, controllers: new Set() };
|
|
2453
|
+
this.#cancelWatchers.set(waiterKey, watchers);
|
|
1981
2454
|
}
|
|
1982
|
-
watchers.add(controller);
|
|
1983
|
-
this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => this.#runCancelPoller());
|
|
2455
|
+
watchers.controllers.add(controller);
|
|
2456
|
+
this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => (0, shard_keys_1.withClearedShardKeyState)(() => this.#runCancelPoller()));
|
|
1984
2457
|
};
|
|
1985
2458
|
const stop = () => {
|
|
1986
|
-
const watchers = this.#cancelWatchers.get(
|
|
1987
|
-
if (watchers?.delete(controller) && watchers.size === 0) {
|
|
1988
|
-
this.#cancelWatchers.delete(
|
|
2459
|
+
const watchers = this.#cancelWatchers.get(waiterKey);
|
|
2460
|
+
if (watchers?.controllers.delete(controller) && watchers.controllers.size === 0) {
|
|
2461
|
+
this.#cancelWatchers.delete(waiterKey);
|
|
1989
2462
|
if (this.#cancelWatchers.size === 0)
|
|
1990
2463
|
this.#cancelPollerWake?.();
|
|
1991
2464
|
}
|
|
@@ -2024,15 +2497,36 @@ class SystemDatabase {
|
|
|
2024
2497
|
this.#cancelPollerLoop = undefined;
|
|
2025
2498
|
}
|
|
2026
2499
|
async #abortCancelledWatchers() {
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2500
|
+
// One read per shard key, each bound to its own workflows.
|
|
2501
|
+
const byKey = new Map();
|
|
2502
|
+
for (const { workflowID, shardKey } of this.#cancelWatchers.values()) {
|
|
2503
|
+
const ids = byKey.get(shardKey) ?? [];
|
|
2504
|
+
ids.push(workflowID);
|
|
2505
|
+
byKey.set(shardKey, ids);
|
|
2506
|
+
}
|
|
2507
|
+
for (const [shardKey, workflowIDs] of byKey) {
|
|
2508
|
+
// Each key's read on its own, so one key's failure doesn't hide another key's cancellations.
|
|
2509
|
+
let rows;
|
|
2510
|
+
try {
|
|
2511
|
+
({ rows } = await (0, shard_keys_1.bindShardKeys)(workflowIDs.map((id) => [id, shardKey]), () => {
|
|
2512
|
+
const params = [workflow_1.StatusString.CANCELLED];
|
|
2513
|
+
const ids = this.wfIn(undefined, workflowIDs, params);
|
|
2514
|
+
return this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${ids} AND status = $1`, params));
|
|
2515
|
+
}));
|
|
2516
|
+
}
|
|
2517
|
+
catch (e) {
|
|
2518
|
+
this.logger.warn(`Error polling status for cancellable steps: ${String(e)}`);
|
|
2031
2519
|
continue;
|
|
2032
|
-
|
|
2033
|
-
const
|
|
2034
|
-
for (const
|
|
2035
|
-
|
|
2520
|
+
}
|
|
2521
|
+
const cancelled = new Set(rows.map((row) => row.workflow_uuid));
|
|
2522
|
+
for (const [waiterKey, watchers] of this.#cancelWatchers) {
|
|
2523
|
+
if (watchers.shardKey !== shardKey || !cancelled.has(watchers.workflowID))
|
|
2524
|
+
continue;
|
|
2525
|
+
this.#cancelWatchers.delete(waiterKey);
|
|
2526
|
+
const reason = new error_1.DBOSWorkflowCancelledError(watchers.workflowID);
|
|
2527
|
+
for (const controller of watchers.controllers)
|
|
2528
|
+
controller.abort(reason);
|
|
2529
|
+
}
|
|
2036
2530
|
}
|
|
2037
2531
|
}
|
|
2038
2532
|
// A missing row normally means the workflow has not been inserted yet, so
|
|
@@ -2044,21 +2538,23 @@ class SystemDatabase {
|
|
|
2044
2538
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2045
2539
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2046
2540
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
2047
|
-
// Record the durable timeout deadline once before polling.
|
|
2541
|
+
// Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
|
|
2048
2542
|
// first call and reads back the same value on recovery, so it never changes across iterations.
|
|
2049
2543
|
if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
|
|
2050
|
-
finishTime = await this
|
|
2544
|
+
finishTime = await this.durableSleepInternal(callerID, timerFuncID, timeoutms);
|
|
2051
2545
|
}
|
|
2052
2546
|
while (true) {
|
|
2053
2547
|
if (callerID)
|
|
2054
2548
|
await this.checkIfCanceledLimited(callerID);
|
|
2055
2549
|
let rows;
|
|
2056
2550
|
try {
|
|
2551
|
+
const params = [];
|
|
2552
|
+
const target = this.wf('ws', workflowID, params);
|
|
2057
2553
|
({ rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT ws.status, COALESCE(wo.output, ws.output) AS output,
|
|
2058
2554
|
COALESCE(wo.error, ws.error) AS error, ws.serialization
|
|
2059
2555
|
FROM "${this.schemaName}".workflow_status ws
|
|
2060
|
-
LEFT JOIN "${this.schemaName}".workflow_output wo ON
|
|
2061
|
-
WHERE
|
|
2556
|
+
LEFT JOIN "${this.schemaName}".workflow_output wo ON ${this.wfJoin('ws', 'wo')}
|
|
2557
|
+
WHERE ${target}`, params)));
|
|
2062
2558
|
}
|
|
2063
2559
|
catch (e) {
|
|
2064
2560
|
const err = e;
|
|
@@ -2095,21 +2591,27 @@ class SystemDatabase {
|
|
|
2095
2591
|
}
|
|
2096
2592
|
}
|
|
2097
2593
|
async awaitFirstWorkflowId(workflowIds, callerID, pollingIntervalMs) {
|
|
2098
|
-
const placeholders = workflowIds.map((_, i) => `$${i + 1}`).join(', ');
|
|
2099
2594
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
2100
2595
|
while (true) {
|
|
2101
2596
|
if (callerID)
|
|
2102
2597
|
await this.checkIfCanceledLimited(callerID);
|
|
2103
|
-
const
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
LIMIT 1`, workflowIds));
|
|
2107
|
-
if (rows.length > 0) {
|
|
2108
|
-
return rows[0].workflow_uuid;
|
|
2598
|
+
const first = await this.checkFirstWorkflowId(workflowIds);
|
|
2599
|
+
if (first !== undefined) {
|
|
2600
|
+
return first;
|
|
2109
2601
|
}
|
|
2110
2602
|
await (0, utils_1.sleepms)(pollIntervalMs);
|
|
2111
2603
|
}
|
|
2112
2604
|
}
|
|
2605
|
+
/** The first of these workflows found complete, if any. */
|
|
2606
|
+
async checkFirstWorkflowId(workflowIds) {
|
|
2607
|
+
const params = [];
|
|
2608
|
+
const ids = this.wfIn(undefined, workflowIds, params);
|
|
2609
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
2610
|
+
WHERE ${ids}
|
|
2611
|
+
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
|
|
2612
|
+
LIMIT 1`, params));
|
|
2613
|
+
return rows[0]?.workflow_uuid;
|
|
2614
|
+
}
|
|
2113
2615
|
async awaitWorkflowIds(workflowIds, callerID, pollingIntervalMs) {
|
|
2114
2616
|
const remainingWorkflowIds = new Set(workflowIds);
|
|
2115
2617
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
@@ -2117,11 +2619,8 @@ class SystemDatabase {
|
|
|
2117
2619
|
const currentWorkflowIds = [...remainingWorkflowIds];
|
|
2118
2620
|
if (callerID)
|
|
2119
2621
|
await this.checkIfCanceledLimited(callerID);
|
|
2120
|
-
const
|
|
2121
|
-
|
|
2122
|
-
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, [currentWorkflowIds]));
|
|
2123
|
-
for (const row of rows) {
|
|
2124
|
-
remainingWorkflowIds.delete(row.workflow_uuid);
|
|
2622
|
+
for (const workflowID of await this.checkCompletedWorkflowIds(currentWorkflowIds)) {
|
|
2623
|
+
remainingWorkflowIds.delete(workflowID);
|
|
2125
2624
|
}
|
|
2126
2625
|
if (remainingWorkflowIds.size === 0) {
|
|
2127
2626
|
return;
|
|
@@ -2129,9 +2628,18 @@ class SystemDatabase {
|
|
|
2129
2628
|
await (0, utils_1.sleepms)(pollIntervalMs);
|
|
2130
2629
|
}
|
|
2131
2630
|
}
|
|
2631
|
+
/** Those of these workflows found complete. */
|
|
2632
|
+
async checkCompletedWorkflowIds(workflowIds) {
|
|
2633
|
+
const params = [];
|
|
2634
|
+
const ids = this.wfIn(undefined, workflowIds, params);
|
|
2635
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
2636
|
+
WHERE ${ids}
|
|
2637
|
+
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, params));
|
|
2638
|
+
return rows.map((row) => row.workflow_uuid);
|
|
2639
|
+
}
|
|
2132
2640
|
// ==================== Sleep ====================
|
|
2133
2641
|
async durableSleepms(workflowID, functionID, durationMS) {
|
|
2134
|
-
const endTime = await this
|
|
2642
|
+
const endTime = await this.durableSleepInternal(workflowID, functionID, durationMS, true);
|
|
2135
2643
|
while (Date.now() < endTime) {
|
|
2136
2644
|
await (0, utils_1.sleepms)(Math.min(endTime - Date.now(), utils_1.sleepConfig.maxTimeoutMS));
|
|
2137
2645
|
}
|
|
@@ -2139,31 +2647,28 @@ class SystemDatabase {
|
|
|
2139
2647
|
}
|
|
2140
2648
|
// ==================== Messaging ====================
|
|
2141
2649
|
nullTopic = '__null__topic__';
|
|
2142
|
-
async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey
|
|
2143
|
-
|
|
2144
|
-
|
|
2650
|
+
async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey,
|
|
2651
|
+
// The destination's shard key; required on a sharded system database.
|
|
2652
|
+
shardKey) {
|
|
2145
2653
|
const startTime = Date.now();
|
|
2146
2654
|
let missing;
|
|
2147
2655
|
const client = await this.connect();
|
|
2148
2656
|
try {
|
|
2149
2657
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
2150
2658
|
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
|
|
2151
|
-
await
|
|
2152
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2153
|
-
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
2659
|
+
await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2154
2660
|
return undefined;
|
|
2155
2661
|
});
|
|
2156
2662
|
await client.query('COMMIT');
|
|
2157
2663
|
}
|
|
2158
2664
|
catch (error) {
|
|
2159
2665
|
await client.query('ROLLBACK');
|
|
2160
|
-
|
|
2161
|
-
if (err.code === '23503') {
|
|
2666
|
+
if (error instanceof error_1.DBOSNonExistentWorkflowError) {
|
|
2162
2667
|
// Foreign key constraint violation (only expected for the INSERT query)
|
|
2163
|
-
missing =
|
|
2668
|
+
missing = error;
|
|
2164
2669
|
}
|
|
2165
2670
|
else {
|
|
2166
|
-
throw
|
|
2671
|
+
throw error;
|
|
2167
2672
|
}
|
|
2168
2673
|
}
|
|
2169
2674
|
finally {
|
|
@@ -2176,22 +2681,22 @@ class SystemDatabase {
|
|
|
2176
2681
|
}
|
|
2177
2682
|
}
|
|
2178
2683
|
/** Runs on `client` if given, joining its transaction; otherwise on the pool with retries. */
|
|
2179
|
-
async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client) {
|
|
2684
|
+
async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client, shardKey) {
|
|
2180
2685
|
if (client !== undefined) {
|
|
2181
|
-
return await this
|
|
2686
|
+
return await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2182
2687
|
}
|
|
2183
|
-
return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey);
|
|
2688
|
+
return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2184
2689
|
}
|
|
2185
2690
|
/** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
|
|
2186
|
-
async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
|
|
2691
|
+
async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2187
2692
|
const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
|
|
2188
2693
|
const client = await this.connect();
|
|
2189
2694
|
try {
|
|
2190
|
-
await this
|
|
2191
|
-
await this
|
|
2695
|
+
await this.inTransaction(client, async () => {
|
|
2696
|
+
await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2192
2697
|
// After the insert, in the order the recorded send locks, so they cannot deadlock.
|
|
2193
2698
|
if (ownerXid !== undefined) {
|
|
2194
|
-
await this
|
|
2699
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
2195
2700
|
}
|
|
2196
2701
|
});
|
|
2197
2702
|
}
|
|
@@ -2199,17 +2704,23 @@ class SystemDatabase {
|
|
|
2199
2704
|
client.release();
|
|
2200
2705
|
}
|
|
2201
2706
|
}
|
|
2202
|
-
async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey) {
|
|
2203
|
-
return await this
|
|
2707
|
+
async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2708
|
+
return await this.sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2204
2709
|
}
|
|
2205
|
-
async
|
|
2710
|
+
async sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2206
2711
|
topic = topic ?? this.nullTopic;
|
|
2207
|
-
//
|
|
2712
|
+
// An idempotency key is scoped per destination: suffix it with the recipient's workflow ID.
|
|
2208
2713
|
const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
|
|
2714
|
+
const values = {
|
|
2715
|
+
...this.wfValues(destinationID, shardKey),
|
|
2716
|
+
destination_uuid: destinationID,
|
|
2717
|
+
topic,
|
|
2718
|
+
message,
|
|
2719
|
+
serialization,
|
|
2720
|
+
message_uuid: messageUUID,
|
|
2721
|
+
};
|
|
2209
2722
|
try {
|
|
2210
|
-
await
|
|
2211
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2212
|
-
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
2723
|
+
await this.insertRows(db, 'notifications', Object.keys(values), [Object.values(values)], `ON CONFLICT (${this.keyConflict('message_uuid')}) DO NOTHING`);
|
|
2213
2724
|
}
|
|
2214
2725
|
catch (error) {
|
|
2215
2726
|
const err = error;
|
|
@@ -2233,10 +2744,10 @@ class SystemDatabase {
|
|
|
2233
2744
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2234
2745
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2235
2746
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
|
|
2236
|
-
// Record the durable timeout deadline once before polling.
|
|
2747
|
+
// Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
|
|
2237
2748
|
// first call and reads back the same value on recovery, so it never changes across iterations.
|
|
2238
2749
|
if (timeoutms) {
|
|
2239
|
-
finishTime = await this
|
|
2750
|
+
finishTime = await this.durableSleepInternal(workflowID, timeoutFunctionID, timeoutms);
|
|
2240
2751
|
}
|
|
2241
2752
|
while (true) {
|
|
2242
2753
|
// register the key with the global notifications listener.
|
|
@@ -2244,12 +2755,14 @@ class SystemDatabase {
|
|
|
2244
2755
|
const messagePromise = new Promise((resolve) => {
|
|
2245
2756
|
resolveNotification = resolve;
|
|
2246
2757
|
});
|
|
2247
|
-
const payload =
|
|
2758
|
+
const payload = this.waiter(workflowID, topic);
|
|
2248
2759
|
const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
|
|
2249
2760
|
try {
|
|
2250
2761
|
await this.checkIfCanceledLimited(workflowID);
|
|
2251
2762
|
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
2252
|
-
const
|
|
2763
|
+
const params = [topic];
|
|
2764
|
+
const target = this.wf(undefined, workflowID, params, 'destination_uuid');
|
|
2765
|
+
const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE ${target} AND topic=$1 AND consumed = false;`, params))).rows;
|
|
2253
2766
|
if (initRecvRows.length !== 0)
|
|
2254
2767
|
break;
|
|
2255
2768
|
const ct = Date.now();
|
|
@@ -2276,21 +2789,22 @@ class SystemDatabase {
|
|
|
2276
2789
|
const client = await this.connect();
|
|
2277
2790
|
try {
|
|
2278
2791
|
await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
|
|
2792
|
+
const consumeParams = [topic, functionID];
|
|
2279
2793
|
const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
|
|
2280
|
-
SET consumed = true, consumed_by_function_id = $
|
|
2281
|
-
WHERE
|
|
2282
|
-
AND topic = $
|
|
2794
|
+
SET consumed = true, consumed_by_function_id = $2
|
|
2795
|
+
WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
|
|
2796
|
+
AND topic = $1
|
|
2283
2797
|
AND consumed = false
|
|
2284
2798
|
AND message_uuid = (
|
|
2285
2799
|
SELECT message_uuid
|
|
2286
2800
|
FROM "${this.schemaName}".notifications
|
|
2287
|
-
WHERE
|
|
2288
|
-
AND topic = $
|
|
2801
|
+
WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
|
|
2802
|
+
AND topic = $1
|
|
2289
2803
|
AND consumed = false
|
|
2290
2804
|
ORDER BY created_at_epoch_ms ASC
|
|
2291
2805
|
LIMIT 1
|
|
2292
2806
|
)
|
|
2293
|
-
RETURNING notifications.message, notifications.serialization;`,
|
|
2807
|
+
RETURNING notifications.message, notifications.serialization;`, consumeParams)).rows;
|
|
2294
2808
|
if (finalRecvRows.length > 0) {
|
|
2295
2809
|
message = finalRecvRows[0].message;
|
|
2296
2810
|
serialization = finalRecvRows[0].serialization;
|
|
@@ -2319,16 +2833,19 @@ class SystemDatabase {
|
|
|
2319
2833
|
// Only a real write (not a replay) should wake readers.
|
|
2320
2834
|
let didWrite = false;
|
|
2321
2835
|
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SETEVENT, workflowID, functionID, async () => {
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2836
|
+
const keys = this.wfValues(workflowID);
|
|
2837
|
+
const keyColumns = Object.keys(keys).map((c) => `${c}, `);
|
|
2838
|
+
const keyPlaceholders = (from) => Object.keys(keys).map((_, i) => `$${from + i}, `);
|
|
2839
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_events (${keyColumns.join('')}workflow_uuid, key, value, serialization)
|
|
2840
|
+
VALUES (${keyPlaceholders(5).join('')}$1, $2, $3, $4)
|
|
2841
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'key')})
|
|
2325
2842
|
DO UPDATE SET value = $3, serialization = $4
|
|
2326
|
-
RETURNING workflow_uuid;`, [workflowID, key, message, serialization]);
|
|
2843
|
+
RETURNING workflow_uuid;`, [workflowID, key, message, serialization, ...Object.values(keys)]);
|
|
2327
2844
|
// Also write to the immutable history table for fork support
|
|
2328
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (workflow_uuid, function_id, key, value, serialization)
|
|
2329
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2330
|
-
ON CONFLICT (workflow_uuid, function_id, key)
|
|
2331
|
-
DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization]);
|
|
2845
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (${keyColumns.join('')}workflow_uuid, function_id, key, value, serialization)
|
|
2846
|
+
VALUES (${keyPlaceholders(6).join('')}$1, $2, $3, $4, $5)
|
|
2847
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id', 'key')})
|
|
2848
|
+
DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization, ...Object.values(keys)]);
|
|
2332
2849
|
didWrite = true;
|
|
2333
2850
|
return undefined;
|
|
2334
2851
|
});
|
|
@@ -2362,15 +2879,15 @@ class SystemDatabase {
|
|
|
2362
2879
|
// Get the return the value. if it's in the DB, otherwise return null.
|
|
2363
2880
|
let value = null;
|
|
2364
2881
|
let valueSer = null;
|
|
2365
|
-
const payloadKey =
|
|
2882
|
+
const payloadKey = this.waiter(workflowID, key);
|
|
2366
2883
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2367
2884
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2368
2885
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
|
|
2369
2886
|
// If we have a callerWorkflow, we want a durable sleep, otherwise, not. Record the durable
|
|
2370
|
-
// timeout deadline once before polling.
|
|
2887
|
+
// timeout deadline once before polling. durableSleepInternal persists it on the first call and reads
|
|
2371
2888
|
// back the same value on recovery, so it never changes across iterations.
|
|
2372
2889
|
if (callerWorkflow && timeoutms) {
|
|
2373
|
-
finishTime = await this
|
|
2890
|
+
finishTime = await this.durableSleepInternal(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms);
|
|
2374
2891
|
}
|
|
2375
2892
|
// Register the key with the global notifications listener first... we do not want to look in the DB first
|
|
2376
2893
|
// or that would cause a timing hole.
|
|
@@ -2384,9 +2901,11 @@ class SystemDatabase {
|
|
|
2384
2901
|
if (callerWorkflow?.workflowID)
|
|
2385
2902
|
await this.checkIfCanceledLimited(callerWorkflow?.workflowID);
|
|
2386
2903
|
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
2904
|
+
const params = [key];
|
|
2905
|
+
const target = this.wf(undefined, workflowID, params);
|
|
2387
2906
|
const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT key, value, serialization
|
|
2388
2907
|
FROM "${this.schemaName}".workflow_events
|
|
2389
|
-
WHERE
|
|
2908
|
+
WHERE ${target} AND key=$1;`, params))).rows;
|
|
2390
2909
|
if (initRecvRows.length > 0) {
|
|
2391
2910
|
value = initRecvRows[0].value;
|
|
2392
2911
|
valueSer = initRecvRows[0].serialization;
|
|
@@ -2425,15 +2944,12 @@ class SystemDatabase {
|
|
|
2425
2944
|
try {
|
|
2426
2945
|
while (true) {
|
|
2427
2946
|
try {
|
|
2428
|
-
await this
|
|
2947
|
+
await this.inTransaction(client, async () => {
|
|
2429
2948
|
// Derives the first unused offset inside the insert; two writers can still pick the same one.
|
|
2430
|
-
await
|
|
2431
|
-
SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
|
|
2432
|
-
FROM "${this.schemaName}".streams s
|
|
2433
|
-
WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
|
|
2949
|
+
await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
|
|
2434
2950
|
// After the insert, in the order the workflow-level writes lock, so they cannot deadlock.
|
|
2435
2951
|
if (ownerXid !== undefined) {
|
|
2436
|
-
await this
|
|
2952
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
2437
2953
|
}
|
|
2438
2954
|
});
|
|
2439
2955
|
}
|
|
@@ -2467,10 +2983,7 @@ class SystemDatabase {
|
|
|
2467
2983
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
2468
2984
|
await this.#runAndRecordResult(client, functionName, workflowID, functionID, async () => {
|
|
2469
2985
|
// Derives the first unused offset inside the insert; two writers can still pick the same one.
|
|
2470
|
-
await
|
|
2471
|
-
SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
|
|
2472
|
-
FROM "${this.schemaName}".streams s
|
|
2473
|
-
WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
|
|
2986
|
+
await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
|
|
2474
2987
|
didWrite = true;
|
|
2475
2988
|
return undefined;
|
|
2476
2989
|
});
|
|
@@ -2502,6 +3015,15 @@ class SystemDatabase {
|
|
|
2502
3015
|
client.release();
|
|
2503
3016
|
}
|
|
2504
3017
|
}
|
|
3018
|
+
/** Append a value to a stream, at the first unused offset, derived inside the insert. */
|
|
3019
|
+
async #insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization) {
|
|
3020
|
+
const params = [key, serializedValue, functionID, serialization, workflowID];
|
|
3021
|
+
const keyColumns = this.wfColumns(workflowID, params);
|
|
3022
|
+
await client.query(`INSERT INTO "${this.schemaName}".streams (${keyColumns.map(([c]) => `${c}, `).join('')}workflow_uuid, key, value, "offset", function_id, serialization)
|
|
3023
|
+
SELECT ${keyColumns.map(([, e]) => `${e}, `).join('')}$5::text, $1::text, $2::text, COALESCE(MAX(s."offset"), -1) + 1, $3::int, $4::text
|
|
3024
|
+
FROM "${this.schemaName}".streams s
|
|
3025
|
+
WHERE ${this.wf('s', workflowID, params)} AND s.key = $1`, params);
|
|
3026
|
+
}
|
|
2505
3027
|
async closeStreamFromWorkflow(workflowID, functionID, key) {
|
|
2506
3028
|
await this.writeStreamFromWorkflow(workflowID, functionID, key, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name(), exports.DBOS_FUNCNAME_CLOSESTREAM);
|
|
2507
3029
|
}
|
|
@@ -2511,13 +3033,15 @@ class SystemDatabase {
|
|
|
2511
3033
|
// Read the value at `offset` and the workflow's status in one query: status null = no such workflow, value undefined = nothing at that offset.
|
|
2512
3034
|
async readStreamValue(workflowID, key, offset) {
|
|
2513
3035
|
// LEFT JOIN so a workflow with nothing at offset still reports its status (single PK lookup); under the poll limiter, inside @dbRetry so the permit frees across backoff.
|
|
3036
|
+
const params = [key, offset];
|
|
3037
|
+
const target = this.wf('ws', workflowID, params);
|
|
2514
3038
|
const result = await this.#pollWithLimiter(() => this.pool.query(
|
|
2515
3039
|
// "offset" is a reserved word, so alias it (stream_offset) to read it back plainly.
|
|
2516
3040
|
`SELECT ws.status AS status, s.value AS value, s.serialization AS serialization, s."offset" AS stream_offset
|
|
2517
3041
|
FROM "${this.schemaName}".workflow_status ws
|
|
2518
3042
|
LEFT OUTER JOIN "${this.schemaName}".streams s
|
|
2519
|
-
ON s
|
|
2520
|
-
WHERE
|
|
3043
|
+
ON ${this.wfJoin('s', 'ws')} AND s.key = $1 AND s."offset" = $2
|
|
3044
|
+
WHERE ${target}`, params));
|
|
2521
3045
|
if (result.rows.length === 0) {
|
|
2522
3046
|
return { status: null, value: undefined };
|
|
2523
3047
|
}
|
|
@@ -2654,7 +3178,7 @@ class SystemDatabase {
|
|
|
2654
3178
|
// Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v3, so cost scales with the number of partitions rather than the backlog depth.
|
|
2655
3179
|
const params = [queueName, workflow_1.StatusString.ENQUEUED];
|
|
2656
3180
|
// Only partitions this application can actually dequeue from.
|
|
2657
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3181
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2658
3182
|
const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
|
|
2659
3183
|
(SELECT MIN(queue_partition_key) AS pk
|
|
2660
3184
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -2671,7 +3195,14 @@ class SystemDatabase {
|
|
|
2671
3195
|
}
|
|
2672
3196
|
async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0,
|
|
2673
3197
|
// Written to every claimed row: the dispatches of this claim own their executions.
|
|
2674
|
-
ownerXid = null) {
|
|
3198
|
+
ownerXid = null, isLatestVersion) {
|
|
3199
|
+
const { claimedIDs } = await this.claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount, partitionLocalRunningCount, ownerXid, isLatestVersion);
|
|
3200
|
+
return claimedIDs;
|
|
3201
|
+
}
|
|
3202
|
+
/** Claim this queue's next workflows, plus the limit that left no room to claim any (undefined if every limit had room). */
|
|
3203
|
+
async claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0, ownerXid = null, isLatestVersion) {
|
|
3204
|
+
// Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
|
|
3205
|
+
const startTimeMs = Date.now();
|
|
2675
3206
|
const claimedIDs = [];
|
|
2676
3207
|
const partitionParams = queuePartitionKey !== undefined ? [queuePartitionKey] : [];
|
|
2677
3208
|
// Shares a concurrency or rate limit budget with other executors.
|
|
@@ -2681,6 +3212,10 @@ class SystemDatabase {
|
|
|
2681
3212
|
queue.partitionRateLimit !== undefined;
|
|
2682
3213
|
// Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
|
|
2683
3214
|
const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
|
|
3215
|
+
// A sharded database keeps the version row under another key, which the claim's transaction cannot span.
|
|
3216
|
+
if (isLatestVersion === undefined && this.sharded) {
|
|
3217
|
+
isLatestVersion = await this.isLatestVersion(appVersion);
|
|
3218
|
+
}
|
|
2684
3219
|
const client = await this.connect();
|
|
2685
3220
|
try {
|
|
2686
3221
|
// Default to READ COMMITTED except with a budget shared across executors
|
|
@@ -2694,7 +3229,7 @@ class SystemDatabase {
|
|
|
2694
3229
|
const rateLimitRemaining = async (rateLimit, partitionScoped) => {
|
|
2695
3230
|
const params = [queue.name, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED, rateLimit.periodSec * 1000];
|
|
2696
3231
|
// Count only what this application would dequeue, matching the select below.
|
|
2697
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3232
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2698
3233
|
const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
|
|
2699
3234
|
const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
2700
3235
|
WHERE queue_name = $1
|
|
@@ -2713,7 +3248,7 @@ class SystemDatabase {
|
|
|
2713
3248
|
*/
|
|
2714
3249
|
const pendingCount = async (partitionScoped) => {
|
|
2715
3250
|
const params = [queue.name, workflow_1.StatusString.PENDING];
|
|
2716
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3251
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2717
3252
|
const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
|
|
2718
3253
|
const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
2719
3254
|
WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
|
|
@@ -2721,27 +3256,38 @@ class SystemDatabase {
|
|
|
2721
3256
|
};
|
|
2722
3257
|
// Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
|
|
2723
3258
|
let maxTasks = Infinity;
|
|
3259
|
+
// The first limit found with no room, which the caller reads as why nothing was claimed.
|
|
3260
|
+
let stopReason = undefined;
|
|
3261
|
+
const limit = (room, reason) => {
|
|
3262
|
+
maxTasks = Math.min(maxTasks, room);
|
|
3263
|
+
if (maxTasks <= 0)
|
|
3264
|
+
stopReason ??= reason;
|
|
3265
|
+
};
|
|
2724
3266
|
if (queue.workerConcurrency !== undefined) {
|
|
2725
3267
|
// Use the in-memory registry for this worker's running count — avoids a DB round trip.
|
|
2726
|
-
|
|
3268
|
+
limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
|
|
2727
3269
|
}
|
|
2728
3270
|
if (queue.partitionWorkerConcurrency !== undefined) {
|
|
2729
|
-
|
|
3271
|
+
limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
|
|
2730
3272
|
}
|
|
2731
3273
|
if (maxTasks <= 0) {
|
|
2732
3274
|
await client.query('COMMIT');
|
|
2733
|
-
return claimedIDs;
|
|
3275
|
+
return { claimedIDs, stopReason };
|
|
2734
3276
|
}
|
|
2735
3277
|
if (queue.rateLimit !== undefined) {
|
|
2736
3278
|
// Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
|
|
2737
|
-
|
|
3279
|
+
limit(await rateLimitRemaining(queue.rateLimit, false), 'limiter');
|
|
3280
|
+
if (maxTasks <= 0) {
|
|
3281
|
+
await client.query('COMMIT');
|
|
3282
|
+
return { claimedIDs, stopReason };
|
|
3283
|
+
}
|
|
2738
3284
|
}
|
|
2739
3285
|
if (queue.partitionRateLimit !== undefined) {
|
|
2740
|
-
|
|
3286
|
+
limit(await rateLimitRemaining(queue.partitionRateLimit, true), 'partition_limiter');
|
|
2741
3287
|
}
|
|
2742
3288
|
if (maxTasks <= 0) {
|
|
2743
3289
|
await client.query('COMMIT');
|
|
2744
|
-
return claimedIDs;
|
|
3290
|
+
return { claimedIDs, stopReason };
|
|
2745
3291
|
}
|
|
2746
3292
|
if (queue.concurrency !== undefined) {
|
|
2747
3293
|
// Global concurrency still requires a DB query since other workers may be running workflows too.
|
|
@@ -2749,23 +3295,26 @@ class SystemDatabase {
|
|
|
2749
3295
|
if (totalRunningTasks > queue.concurrency) {
|
|
2750
3296
|
this.logger.warn(`Total running tasks (${totalRunningTasks}) exceeds the global concurrency limit (${queue.concurrency})`);
|
|
2751
3297
|
}
|
|
2752
|
-
|
|
3298
|
+
limit(Math.max(0, queue.concurrency - totalRunningTasks), 'concurrency');
|
|
3299
|
+
if (maxTasks <= 0) {
|
|
3300
|
+
await client.query('COMMIT');
|
|
3301
|
+
return { claimedIDs, stopReason };
|
|
3302
|
+
}
|
|
2753
3303
|
}
|
|
2754
3304
|
if (queue.partitionConcurrency !== undefined) {
|
|
2755
3305
|
const partitionRunningTasks = await pendingCount(true);
|
|
2756
3306
|
if (partitionRunningTasks > queue.partitionConcurrency) {
|
|
2757
3307
|
this.logger.warn(`Total running tasks (${partitionRunningTasks}) on partition ${queuePartitionKey} of queue ${queue.name} exceeds the partition concurrency limit (${queue.partitionConcurrency})`);
|
|
2758
3308
|
}
|
|
2759
|
-
|
|
3309
|
+
limit(Math.max(0, queue.partitionConcurrency - partitionRunningTasks), 'partition_concurrency');
|
|
2760
3310
|
}
|
|
2761
3311
|
// Return immediately if there are no available tasks due to flow control limits
|
|
2762
3312
|
if (maxTasks <= 0) {
|
|
2763
3313
|
await client.query('COMMIT');
|
|
2764
|
-
return claimedIDs;
|
|
3314
|
+
return { claimedIDs, stopReason };
|
|
2765
3315
|
}
|
|
2766
3316
|
// Retrieve the first max_tasks workflows in the queue.
|
|
2767
|
-
|
|
2768
|
-
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
3317
|
+
isLatestVersion ??= await this.isLatestVersion(appVersion, client);
|
|
2769
3318
|
const versionClause = isLatestVersion
|
|
2770
3319
|
? '(application_version = $3 OR application_version IS NULL)'
|
|
2771
3320
|
: 'application_version = $3';
|
|
@@ -2775,13 +3324,17 @@ class SystemDatabase {
|
|
|
2775
3324
|
const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
|
|
2776
3325
|
const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
|
|
2777
3326
|
const selectScope = this.appNameFilter('application_name', this.appName, selectParams);
|
|
3327
|
+
const keyScope = this.scoped(undefined, selectParams);
|
|
3328
|
+
const claimGate = this.claimGate(queue.name, queuePartitionKey, selectParams);
|
|
2778
3329
|
const selectQuery = `
|
|
2779
3330
|
SELECT workflow_uuid
|
|
2780
3331
|
FROM "${this.schemaName}".workflow_status
|
|
2781
|
-
WHERE
|
|
3332
|
+
WHERE ${keyScope}
|
|
3333
|
+
AND status = $1
|
|
2782
3334
|
AND queue_name = $2
|
|
2783
3335
|
AND ${versionClause}
|
|
2784
3336
|
AND ${selectScope}
|
|
3337
|
+
AND ${claimGate}
|
|
2785
3338
|
${queuePartitionKey !== undefined ? 'AND queue_partition_key = $4' : ''}
|
|
2786
3339
|
ORDER BY priority ASC, created_at ASC
|
|
2787
3340
|
${limitClause}
|
|
@@ -2800,35 +3353,36 @@ class SystemDatabase {
|
|
|
2800
3353
|
executorID,
|
|
2801
3354
|
appVersion,
|
|
2802
3355
|
queue.rateLimit !== undefined || queue.partitionRateLimit !== undefined,
|
|
2803
|
-
workflowIDs,
|
|
2804
3356
|
workflow_1.StatusString.ENQUEUED,
|
|
2805
3357
|
// Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
|
|
2806
3358
|
this.appName ?? null,
|
|
2807
3359
|
ownerXid,
|
|
3360
|
+
startTimeMs,
|
|
2808
3361
|
];
|
|
3362
|
+
const claimedIDsFilter = this.wfIn(undefined, workflowIDs, updateParams);
|
|
2809
3363
|
// Re-check ownership alongside status, as the partitioned claim guard does.
|
|
2810
3364
|
const claimScope = this.appNameFilter('application_name', this.appName, updateParams);
|
|
2811
3365
|
// RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
|
|
2812
3366
|
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2813
3367
|
SET status = $1,
|
|
2814
|
-
owner_xid = $
|
|
3368
|
+
owner_xid = $7,
|
|
2815
3369
|
executor_id = $2,
|
|
2816
3370
|
application_version = $3,
|
|
2817
3371
|
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2818
3372
|
rate_limited = $4,
|
|
2819
|
-
application_name = COALESCE(application_name, $
|
|
3373
|
+
application_name = COALESCE(application_name, $6),
|
|
2820
3374
|
recovery_attempts = recovery_attempts + 1,
|
|
2821
3375
|
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2822
3376
|
workflow_deadline_epoch_ms = CASE
|
|
2823
3377
|
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
2824
3378
|
CASE
|
|
2825
|
-
WHEN workflow_timeout_ms > 9223372036854775807 -
|
|
3379
|
+
WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
|
|
2826
3380
|
THEN 9223372036854775807
|
|
2827
|
-
ELSE
|
|
3381
|
+
ELSE $8::bigint + workflow_timeout_ms
|
|
2828
3382
|
END
|
|
2829
3383
|
ELSE workflow_deadline_epoch_ms
|
|
2830
3384
|
END
|
|
2831
|
-
WHERE
|
|
3385
|
+
WHERE ${claimedIDsFilter} AND status = $5 AND ${claimScope}
|
|
2832
3386
|
RETURNING workflow_uuid`, updateParams);
|
|
2833
3387
|
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
2834
3388
|
for (const id of workflowIDs) {
|
|
@@ -2846,14 +3400,16 @@ class SystemDatabase {
|
|
|
2846
3400
|
client.release();
|
|
2847
3401
|
}
|
|
2848
3402
|
// Return the IDs of all functions we marked started
|
|
2849
|
-
return claimedIDs;
|
|
3403
|
+
return { claimedIDs };
|
|
2850
3404
|
}
|
|
2851
3405
|
/** Max heads admitted per sweep: bounds dispatch, not the partition walk; an unconstrained sweep takes the lowest keys, so higher keys can wait under sustained load. */
|
|
2852
3406
|
partitionedDequeueSweepCap = 8192;
|
|
2853
3407
|
/** Dequeue each partition's head-of-line workflow in one transaction, at most {@link partitionedDequeueSweepCap} per sweep; only valid for partition-concurrency-1 queues with no queue-wide limit. */
|
|
2854
3408
|
async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity,
|
|
2855
3409
|
// Written to every claimed row: the dispatches of this claim own their executions.
|
|
2856
|
-
ownerXid = null) {
|
|
3410
|
+
ownerXid = null, isLatestVersion) {
|
|
3411
|
+
// Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
|
|
3412
|
+
const startTimeMs = Date.now();
|
|
2857
3413
|
if (queue.partitionConcurrency !== 1 ||
|
|
2858
3414
|
queue.concurrency !== undefined ||
|
|
2859
3415
|
queue.rateLimit !== undefined ||
|
|
@@ -2861,11 +3417,14 @@ class SystemDatabase {
|
|
|
2861
3417
|
throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
|
|
2862
3418
|
}
|
|
2863
3419
|
// partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
|
|
3420
|
+
// A sharded database keeps the version row under another key, which the claim's transaction cannot span.
|
|
3421
|
+
if (isLatestVersion === undefined && this.sharded) {
|
|
3422
|
+
isLatestVersion = await this.isLatestVersion(appVersion);
|
|
3423
|
+
}
|
|
2864
3424
|
const client = await this.connect();
|
|
2865
3425
|
try {
|
|
2866
3426
|
await client.query('BEGIN');
|
|
2867
|
-
|
|
2868
|
-
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
3427
|
+
isLatestVersion ??= await this.isLatestVersion(appVersion, client);
|
|
2869
3428
|
const versionClause = (n) => isLatestVersion
|
|
2870
3429
|
? `(application_version = $${n} OR application_version IS NULL)`
|
|
2871
3430
|
: `application_version = $${n}`;
|
|
@@ -2880,7 +3439,8 @@ class SystemDatabase {
|
|
|
2880
3439
|
workflow_1.StatusString.PENDING,
|
|
2881
3440
|
sweepLimit,
|
|
2882
3441
|
];
|
|
2883
|
-
const candidateScope = this.appNameFilter('application_name', this.appName, candidateParams)
|
|
3442
|
+
const candidateScope = `${this.scoped(undefined, candidateParams)} AND ${this.appNameFilter('application_name', this.appName, candidateParams)}`;
|
|
3443
|
+
const probeScope = this.scoped(undefined, candidateParams);
|
|
2884
3444
|
// Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
|
|
2885
3445
|
const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
|
|
2886
3446
|
(SELECT MIN(queue_partition_key) AS pk
|
|
@@ -2897,11 +3457,11 @@ class SystemDatabase {
|
|
|
2897
3457
|
SELECT partitions.pk
|
|
2898
3458
|
FROM partitions
|
|
2899
3459
|
WHERE partitions.pk IS NOT NULL
|
|
2900
|
-
--
|
|
3460
|
+
-- No application filter by design: a mutual-exclusion probe must block on any owner's row.
|
|
2901
3461
|
AND NOT EXISTS (
|
|
2902
3462
|
SELECT 1
|
|
2903
3463
|
FROM "${this.schemaName}".workflow_status
|
|
2904
|
-
WHERE queue_name = $1 AND status = $4
|
|
3464
|
+
WHERE ${probeScope} AND queue_name = $1 AND status = $4
|
|
2905
3465
|
AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
|
|
2906
3466
|
)
|
|
2907
3467
|
ORDER BY ${sweepOrder}
|
|
@@ -2930,18 +3490,19 @@ class SystemDatabase {
|
|
|
2930
3490
|
}
|
|
2931
3491
|
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
|
|
2932
3492
|
// Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
|
|
2933
|
-
const claimGuard = (ids, status, name, version, scope) =>
|
|
3493
|
+
const claimGuard = (ids, status, name, version, scope) => `${ids}
|
|
2934
3494
|
AND status = $${status}
|
|
2935
3495
|
AND queue_name = $${name}
|
|
2936
3496
|
AND queue_partition_key IS NOT NULL
|
|
2937
3497
|
AND ${versionClause(version)}
|
|
2938
3498
|
AND ${scope}`;
|
|
2939
|
-
const lockedParams = [
|
|
3499
|
+
const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
|
|
3500
|
+
const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
|
|
2940
3501
|
const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
|
|
2941
3502
|
// Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
|
|
2942
3503
|
const lockedResult = await client.query(`SELECT workflow_uuid
|
|
2943
3504
|
FROM "${this.schemaName}".workflow_status
|
|
2944
|
-
WHERE ${claimGuard(1, 2, 3,
|
|
3505
|
+
WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
|
|
2945
3506
|
FOR UPDATE SKIP LOCKED`, lockedParams);
|
|
2946
3507
|
const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
|
|
2947
3508
|
// Preserve partition order for submission.
|
|
@@ -2953,36 +3514,37 @@ class SystemDatabase {
|
|
|
2953
3514
|
const flipParams = [
|
|
2954
3515
|
workflow_1.StatusString.PENDING,
|
|
2955
3516
|
executorID,
|
|
2956
|
-
claimIDs,
|
|
2957
3517
|
workflow_1.StatusString.ENQUEUED,
|
|
2958
3518
|
queue.name,
|
|
2959
3519
|
appVersion,
|
|
2960
3520
|
// Claim the row, as the unpartitioned dequeue does.
|
|
2961
3521
|
this.appName ?? null,
|
|
2962
3522
|
ownerXid,
|
|
3523
|
+
startTimeMs,
|
|
2963
3524
|
];
|
|
3525
|
+
const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
|
|
2964
3526
|
const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
|
|
2965
3527
|
// Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
|
|
2966
3528
|
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2967
3529
|
SET status = $1,
|
|
2968
|
-
owner_xid = $
|
|
3530
|
+
owner_xid = $7,
|
|
2969
3531
|
executor_id = $2,
|
|
2970
|
-
application_version = $
|
|
3532
|
+
application_version = $5,
|
|
2971
3533
|
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2972
3534
|
rate_limited = FALSE,
|
|
2973
|
-
application_name = COALESCE(application_name, $
|
|
3535
|
+
application_name = COALESCE(application_name, $6),
|
|
2974
3536
|
recovery_attempts = recovery_attempts + 1,
|
|
2975
3537
|
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2976
3538
|
workflow_deadline_epoch_ms = CASE
|
|
2977
3539
|
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
2978
3540
|
CASE
|
|
2979
|
-
WHEN workflow_timeout_ms > 9223372036854775807 -
|
|
3541
|
+
WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
|
|
2980
3542
|
THEN 9223372036854775807
|
|
2981
|
-
ELSE
|
|
3543
|
+
ELSE $8::bigint + workflow_timeout_ms
|
|
2982
3544
|
END
|
|
2983
3545
|
ELSE workflow_deadline_epoch_ms
|
|
2984
3546
|
END
|
|
2985
|
-
WHERE ${claimGuard(3, 4, 5,
|
|
3547
|
+
WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
|
|
2986
3548
|
RETURNING workflow_uuid`, flipParams);
|
|
2987
3549
|
await client.query('COMMIT');
|
|
2988
3550
|
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
@@ -3001,14 +3563,15 @@ class SystemDatabase {
|
|
|
3001
3563
|
}
|
|
3002
3564
|
}
|
|
3003
3565
|
// ==================== Queries & Maintenance ====================
|
|
3004
|
-
|
|
3005
|
-
|
|
3566
|
+
/** With `keyedIDs`, the listed workflow IDs are addressed by their shard keys, as a batch read of known workflows. */
|
|
3567
|
+
async listWorkflows(input, options = {}) {
|
|
3568
|
+
return await this.#listWorkflows(input, options);
|
|
3006
3569
|
}
|
|
3007
3570
|
/**
|
|
3008
3571
|
* Unretried body of {@link listWorkflows}. An ID-keyed read runs on `client` when given, so a caller
|
|
3009
3572
|
* holding a transaction never waits on a second pool connection; the caller owns the retry.
|
|
3010
3573
|
*/
|
|
3011
|
-
async #listWorkflows(input, client) {
|
|
3574
|
+
async #listWorkflows(input, options = {}, client) {
|
|
3012
3575
|
const schemaName = this.schemaName;
|
|
3013
3576
|
const selectColumns = [
|
|
3014
3577
|
'workflow_uuid',
|
|
@@ -3050,16 +3613,23 @@ class SystemDatabase {
|
|
|
3050
3613
|
const payloadJoins = [];
|
|
3051
3614
|
if (input.loadInput) {
|
|
3052
3615
|
payloadColumns.push('COALESCE(wi.inputs, workflow_status.inputs) AS inputs');
|
|
3053
|
-
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON wi
|
|
3616
|
+
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON ${this.wfJoin('wi', 'workflow_status')}`);
|
|
3054
3617
|
}
|
|
3055
3618
|
if (input.loadOutput) {
|
|
3056
3619
|
payloadColumns.push('COALESCE(wo.output, workflow_status.output) AS output');
|
|
3057
3620
|
payloadColumns.push('COALESCE(wo.error, workflow_status.error) AS error');
|
|
3058
|
-
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON wo
|
|
3621
|
+
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON ${this.wfJoin('wo', 'workflow_status')}`);
|
|
3059
3622
|
}
|
|
3060
3623
|
if (input.loadInput || input.loadOutput) {
|
|
3061
3624
|
selectColumns.push('serialization');
|
|
3062
3625
|
}
|
|
3626
|
+
const keyColumns = [];
|
|
3627
|
+
const keyColumn = this.keyColumn('workflow_status');
|
|
3628
|
+
if (keyColumn !== undefined)
|
|
3629
|
+
keyColumns.push(`${keyColumn} AS shard_key`);
|
|
3630
|
+
const parentKeyColumn = this.keyColumn('workflow_status', 'parent_shard_key');
|
|
3631
|
+
if (parentKeyColumn !== undefined)
|
|
3632
|
+
keyColumns.push(`${parentKeyColumn} AS parent_shard_key`);
|
|
3063
3633
|
input.sortDesc = input.sortDesc ?? false; // By default, sort in ascending order
|
|
3064
3634
|
// Build WHERE clauses
|
|
3065
3635
|
const whereClauses = [];
|
|
@@ -3102,6 +3672,18 @@ class SystemDatabase {
|
|
|
3102
3672
|
whereClauses.push(idKeyed
|
|
3103
3673
|
? this.appNameFilter('application_name', input.applicationName, params)
|
|
3104
3674
|
: this.observabilityFilter('application_name', input.applicationName, params));
|
|
3675
|
+
// An empty key list matches nothing, rather than every key.
|
|
3676
|
+
if (input.shardKey !== undefined && input.shardKey !== null) {
|
|
3677
|
+
const keys = Array.isArray(input.shardKey) ? input.shardKey : [input.shardKey];
|
|
3678
|
+
whereClauses.push(this.keyIn('workflow_status', keys, params));
|
|
3679
|
+
}
|
|
3680
|
+
if (input.parentShardKey !== undefined && input.parentShardKey !== null) {
|
|
3681
|
+
const keys = Array.isArray(input.parentShardKey) ? input.parentShardKey : [input.parentShardKey];
|
|
3682
|
+
whereClauses.push(this.keyIn('workflow_status', keys, params, 'parent_shard_key'));
|
|
3683
|
+
}
|
|
3684
|
+
if (options.keyedIDs && input.workflowIDs) {
|
|
3685
|
+
whereClauses.push(this.wfIn('workflow_status', input.workflowIDs, params));
|
|
3686
|
+
}
|
|
3105
3687
|
paramCounter = params.length + 1;
|
|
3106
3688
|
if (input.workflow_id_prefix) {
|
|
3107
3689
|
if (Array.isArray(input.workflow_id_prefix)) {
|
|
@@ -3116,7 +3698,7 @@ class SystemDatabase {
|
|
|
3116
3698
|
paramCounter++;
|
|
3117
3699
|
}
|
|
3118
3700
|
}
|
|
3119
|
-
if (input.workflowIDs) {
|
|
3701
|
+
if (input.workflowIDs && !options.keyedIDs) {
|
|
3120
3702
|
const placeholders = input.workflowIDs.map((_, i) => `$${paramCounter + i}`).join(', ');
|
|
3121
3703
|
whereClauses.push(`workflow_status.workflow_uuid IN (${placeholders})`);
|
|
3122
3704
|
params.push(...input.workflowIDs);
|
|
@@ -3198,7 +3780,7 @@ class SystemDatabase {
|
|
|
3198
3780
|
params.push(input.offset);
|
|
3199
3781
|
paramCounter++;
|
|
3200
3782
|
}
|
|
3201
|
-
const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns].join(', ');
|
|
3783
|
+
const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns, ...keyColumns].join(', ');
|
|
3202
3784
|
const query = `
|
|
3203
3785
|
SELECT ${projection}
|
|
3204
3786
|
FROM "${schemaName}".workflow_status
|
|
@@ -3218,23 +3800,16 @@ class SystemDatabase {
|
|
|
3218
3800
|
async createSchedule(schedule, client) {
|
|
3219
3801
|
const q = client ?? this.pool;
|
|
3220
3802
|
const owner = await this.#resolveRowOwner(q, 'workflow_schedules', 'schedule_name', schedule.scheduleName, schedule.applicationName, 'Schedule');
|
|
3803
|
+
const values = {
|
|
3804
|
+
...this.globalValues(),
|
|
3805
|
+
...this.scheduleValues(schedule),
|
|
3806
|
+
...scheduleRow(schedule, owner),
|
|
3807
|
+
};
|
|
3221
3808
|
try {
|
|
3222
|
-
await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
3223
|
-
(
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
schedule.scheduleName,
|
|
3227
|
-
schedule.workflowName,
|
|
3228
|
-
schedule.workflowClassName,
|
|
3229
|
-
schedule.schedule,
|
|
3230
|
-
schedule.status,
|
|
3231
|
-
schedule.context,
|
|
3232
|
-
schedule.lastFiredAt,
|
|
3233
|
-
schedule.automaticBackfill,
|
|
3234
|
-
schedule.cronTimezone,
|
|
3235
|
-
schedule.queueName,
|
|
3236
|
-
owner ?? null,
|
|
3237
|
-
]);
|
|
3809
|
+
await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
|
|
3810
|
+
VALUES (${Object.keys(values)
|
|
3811
|
+
.map((_, i) => `$${i + 1}`)
|
|
3812
|
+
.join(', ')})`, Object.values(values));
|
|
3238
3813
|
}
|
|
3239
3814
|
catch (e) {
|
|
3240
3815
|
if (isPgDatabaseError(e) && e.code === '23505') {
|
|
@@ -3281,31 +3856,37 @@ class SystemDatabase {
|
|
|
3281
3856
|
// Unset scopes to this application, as on every other observability query.
|
|
3282
3857
|
conditions.push(this.observabilityFilter('application_name', filters?.applicationName, params));
|
|
3283
3858
|
paramIdx = params.length + 1;
|
|
3859
|
+
conditions.push(this.global(undefined, params));
|
|
3284
3860
|
const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
|
|
3285
|
-
const result = await q.query(`SELECT ${
|
|
3861
|
+
const result = await q.query(`SELECT ${this.#scheduleColumns()}
|
|
3286
3862
|
FROM "${this.schemaName}".workflow_schedules${where}
|
|
3287
3863
|
ORDER BY schedule_name`, params);
|
|
3288
3864
|
return result.rows.map((row) => mapWorkflowSchedule(row));
|
|
3289
3865
|
}
|
|
3290
3866
|
async getSchedule(name, client) {
|
|
3291
3867
|
const q = client ?? this.pool;
|
|
3292
|
-
const
|
|
3868
|
+
const params = [name];
|
|
3869
|
+
const result = await q.query(`SELECT ${this.#scheduleColumns()}
|
|
3293
3870
|
FROM "${this.schemaName}".workflow_schedules
|
|
3294
|
-
WHERE schedule_name = $1`,
|
|
3871
|
+
WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
|
|
3295
3872
|
if (result.rows.length === 0)
|
|
3296
3873
|
return null;
|
|
3297
3874
|
return mapWorkflowSchedule(result.rows[0]);
|
|
3298
3875
|
}
|
|
3876
|
+
/** The columns a schedule is read back with, its workflows' shard key among them on a sharded database. */
|
|
3877
|
+
#scheduleColumns() {
|
|
3878
|
+
const keyColumn = this.scheduleKeyColumn();
|
|
3879
|
+
return keyColumn !== undefined ? `${SCHEDULE_COLUMNS}, ${keyColumn}` : SCHEDULE_COLUMNS;
|
|
3880
|
+
}
|
|
3299
3881
|
async deleteSchedule(name, client) {
|
|
3300
3882
|
const q = client ?? this.pool;
|
|
3301
|
-
|
|
3883
|
+
const params = [name];
|
|
3884
|
+
await q.query(`DELETE FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
|
|
3302
3885
|
}
|
|
3303
3886
|
async setScheduleStatus(name, status, client) {
|
|
3304
3887
|
const q = client ?? this.pool;
|
|
3305
|
-
|
|
3306
|
-
|
|
3307
|
-
name,
|
|
3308
|
-
]);
|
|
3888
|
+
const params = [status, name];
|
|
3889
|
+
await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET status = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
|
|
3309
3890
|
}
|
|
3310
3891
|
async updateSchedule(name, updates, client) {
|
|
3311
3892
|
const q = client ?? this.pool;
|
|
@@ -3328,22 +3909,22 @@ class SystemDatabase {
|
|
|
3328
3909
|
}
|
|
3329
3910
|
if (setClauses.length === 0) {
|
|
3330
3911
|
// Nothing to change, but still surface a missing schedule as an error.
|
|
3331
|
-
const
|
|
3332
|
-
|
|
3333
|
-
]);
|
|
3912
|
+
const existingParams = [name];
|
|
3913
|
+
const existing = await q.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, existingParams)} AND schedule_name = $1`, existingParams);
|
|
3334
3914
|
if (existing.rows.length === 0) {
|
|
3335
3915
|
throw new error_1.DBOSError(`Schedule '${name}' not found`);
|
|
3336
3916
|
}
|
|
3337
3917
|
return;
|
|
3338
3918
|
}
|
|
3339
3919
|
params.push(name);
|
|
3340
|
-
const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx}`, params);
|
|
3920
|
+
const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx} AND ${this.global(undefined, params)}`, params);
|
|
3341
3921
|
if (result.rowCount === 0) {
|
|
3342
3922
|
throw new error_1.DBOSError(`Schedule '${name}' not found`);
|
|
3343
3923
|
}
|
|
3344
3924
|
}
|
|
3345
3925
|
async updateLastFiredAt(name, lastFiredAt) {
|
|
3346
|
-
|
|
3926
|
+
const params = [lastFiredAt, name];
|
|
3927
|
+
await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
|
|
3347
3928
|
}
|
|
3348
3929
|
async applySchedules(schedules) {
|
|
3349
3930
|
const client = await this.connect();
|
|
@@ -3351,12 +3932,21 @@ class SystemDatabase {
|
|
|
3351
3932
|
await client.query('BEGIN');
|
|
3352
3933
|
for (const sched of schedules) {
|
|
3353
3934
|
const owner = await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
3935
|
+
const scheduleKeys = this.scheduleValues(sched);
|
|
3936
|
+
const values = {
|
|
3937
|
+
...this.globalValues(),
|
|
3938
|
+
...scheduleKeys,
|
|
3939
|
+
...scheduleRow(sched, owner),
|
|
3940
|
+
};
|
|
3354
3941
|
// Upsert on schedule_name; on conflict, preserve schedule_id and runtime state (status, last_fired_at) and update only the declared definition fields, so an unchanged re-apply is a no-op.
|
|
3355
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
3356
|
-
(
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3942
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
|
|
3943
|
+
VALUES (${Object.keys(values)
|
|
3944
|
+
.map((_, i) => `$${i + 1}`)
|
|
3945
|
+
.join(', ')})
|
|
3946
|
+
ON CONFLICT (${this.keyConflict('schedule_name')}) DO UPDATE SET
|
|
3947
|
+
${Object.keys(scheduleKeys)
|
|
3948
|
+
.map((c) => `${c} = EXCLUDED.${c}, `)
|
|
3949
|
+
.join('')}workflow_name = EXCLUDED.workflow_name,
|
|
3360
3950
|
workflow_class_name = EXCLUDED.workflow_class_name,
|
|
3361
3951
|
schedule = EXCLUDED.schedule,
|
|
3362
3952
|
context = EXCLUDED.context,
|
|
@@ -3364,20 +3954,7 @@ class SystemDatabase {
|
|
|
3364
3954
|
cron_timezone = EXCLUDED.cron_timezone,
|
|
3365
3955
|
queue_name = EXCLUDED.queue_name,
|
|
3366
3956
|
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it took.
|
|
3367
|
-
application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`,
|
|
3368
|
-
sched.scheduleId,
|
|
3369
|
-
sched.scheduleName,
|
|
3370
|
-
sched.workflowName,
|
|
3371
|
-
sched.workflowClassName,
|
|
3372
|
-
sched.schedule,
|
|
3373
|
-
sched.status,
|
|
3374
|
-
sched.context,
|
|
3375
|
-
sched.lastFiredAt,
|
|
3376
|
-
sched.automaticBackfill,
|
|
3377
|
-
sched.cronTimezone,
|
|
3378
|
-
sched.queueName,
|
|
3379
|
-
owner ?? null,
|
|
3380
|
-
]);
|
|
3957
|
+
application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, Object.values(values));
|
|
3381
3958
|
// Read back, since the guard above is silent about why it declined to claim.
|
|
3382
3959
|
await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
3383
3960
|
}
|
|
@@ -3402,14 +3979,23 @@ class SystemDatabase {
|
|
|
3402
3979
|
try {
|
|
3403
3980
|
await client.query('BEGIN');
|
|
3404
3981
|
// Claim a pre-upgrade row in place, so the version is not recreated or retimed.
|
|
3982
|
+
const claimParams = [owner ?? null, versionName];
|
|
3405
3983
|
const claimed = await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3406
3984
|
SET application_name = $1
|
|
3407
|
-
WHERE version_name = $2 AND application_name IS NULL`,
|
|
3985
|
+
WHERE ${this.global(undefined, claimParams)} AND version_name = $2 AND application_name IS NULL`, claimParams);
|
|
3408
3986
|
if ((claimed.rowCount ?? 0) === 0) {
|
|
3987
|
+
const values = {
|
|
3988
|
+
...this.globalValues(),
|
|
3989
|
+
version_id: (0, crypto_1.randomUUID)(),
|
|
3990
|
+
version_name: versionName,
|
|
3991
|
+
application_name: owner ?? null,
|
|
3992
|
+
};
|
|
3409
3993
|
// Targetless DO NOTHING: names no arbiter, so it survives version_name's uniqueness being dropped while still absorbing a concurrent registrar.
|
|
3410
|
-
await client.query(`INSERT INTO "${this.schemaName}".application_versions (
|
|
3411
|
-
VALUES ($
|
|
3412
|
-
|
|
3994
|
+
await client.query(`INSERT INTO "${this.schemaName}".application_versions (${Object.keys(values).join(', ')})
|
|
3995
|
+
VALUES (${Object.keys(values)
|
|
3996
|
+
.map((_, i) => `$${i + 1}`)
|
|
3997
|
+
.join(', ')})
|
|
3998
|
+
ON CONFLICT DO NOTHING`, Object.values(values));
|
|
3413
3999
|
}
|
|
3414
4000
|
// Read back, since the writes above are silent about why they declined to claim.
|
|
3415
4001
|
await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
|
|
@@ -3440,7 +4026,7 @@ class SystemDatabase {
|
|
|
3440
4026
|
params.push(resolved);
|
|
3441
4027
|
await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3442
4028
|
SET version_timestamp = $1, application_name = ${resolved === undefined ? 'application_name' : '$3'}
|
|
3443
|
-
WHERE version_name = $2 AND ${scope}`, params);
|
|
4029
|
+
WHERE ${this.global(undefined, params)} AND version_name = $2 AND ${scope}`, params);
|
|
3444
4030
|
await client.query('COMMIT');
|
|
3445
4031
|
}
|
|
3446
4032
|
catch (e) {
|
|
@@ -3453,7 +4039,7 @@ class SystemDatabase {
|
|
|
3453
4039
|
}
|
|
3454
4040
|
async listApplicationVersions() {
|
|
3455
4041
|
const params = [];
|
|
3456
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
4042
|
+
const scope = `${this.global(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
3457
4043
|
const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
|
|
3458
4044
|
FROM "${this.schemaName}".application_versions
|
|
3459
4045
|
WHERE ${scope}
|
|
@@ -3473,9 +4059,10 @@ class SystemDatabase {
|
|
|
3473
4059
|
}
|
|
3474
4060
|
// ==================== Queues ====================
|
|
3475
4061
|
async getQueue(name) {
|
|
4062
|
+
const params = [name];
|
|
3476
4063
|
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3477
4064
|
FROM "${this.schemaName}".queues
|
|
3478
|
-
WHERE name = $1`,
|
|
4065
|
+
WHERE ${this.global(undefined, params)} AND name = $1`, params);
|
|
3479
4066
|
return rows.length === 0 ? null : queueRecordFromRow(rows[0]);
|
|
3480
4067
|
}
|
|
3481
4068
|
/**
|
|
@@ -3484,14 +4071,15 @@ class SystemDatabase {
|
|
|
3484
4071
|
*/
|
|
3485
4072
|
async listQueues(applicationName) {
|
|
3486
4073
|
const params = [];
|
|
3487
|
-
const scope = this.observabilityFilter('application_name', applicationName, params)
|
|
4074
|
+
const scope = `${this.global(undefined, params)} AND ${this.observabilityFilter('application_name', applicationName, params)}`;
|
|
3488
4075
|
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3489
4076
|
FROM "${this.schemaName}".queues
|
|
3490
4077
|
WHERE ${scope}`, params);
|
|
3491
4078
|
return rows.map(queueRecordFromRow);
|
|
3492
4079
|
}
|
|
3493
4080
|
async deleteQueue(name) {
|
|
3494
|
-
|
|
4081
|
+
const params = [name];
|
|
4082
|
+
await this.pool.query(`DELETE FROM "${this.schemaName}".queues WHERE ${this.global(undefined, params)} AND name = $1`, params);
|
|
3495
4083
|
}
|
|
3496
4084
|
async updateQueue(name, fields) {
|
|
3497
4085
|
const setClauses = [];
|
|
@@ -3507,7 +4095,7 @@ class SystemDatabase {
|
|
|
3507
4095
|
setClauses.push(`"updated_at" = $${idx++}`);
|
|
3508
4096
|
params.push(Date.now());
|
|
3509
4097
|
params.push(name);
|
|
3510
|
-
await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx}`, params);
|
|
4098
|
+
await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx} AND ${this.global(undefined, params)}`, params);
|
|
3511
4099
|
}
|
|
3512
4100
|
/** Returns true iff this call inserted a new row (i.e. the queue did not
|
|
3513
4101
|
* previously exist). False if the row already existed, regardless of
|
|
@@ -3515,7 +4103,7 @@ class SystemDatabase {
|
|
|
3515
4103
|
async upsertQueue(record, updateExisting) {
|
|
3516
4104
|
const now = Date.now();
|
|
3517
4105
|
const onConflict = updateExisting
|
|
3518
|
-
? `ON CONFLICT (name) DO UPDATE SET
|
|
4106
|
+
? `ON CONFLICT (${this.keyConflict('name')}) DO UPDATE SET
|
|
3519
4107
|
concurrency = EXCLUDED.concurrency,
|
|
3520
4108
|
worker_concurrency = EXCLUDED.worker_concurrency,
|
|
3521
4109
|
rate_limit_max = EXCLUDED.rate_limit_max,
|
|
@@ -3530,21 +4118,27 @@ class SystemDatabase {
|
|
|
3530
4118
|
updated_at = EXCLUDED.updated_at,
|
|
3531
4119
|
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it just took.
|
|
3532
4120
|
application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
|
|
3533
|
-
: `ON CONFLICT (name) DO NOTHING`;
|
|
4121
|
+
: `ON CONFLICT (${this.keyConflict('name')}) DO NOTHING`;
|
|
3534
4122
|
const owner = record.applicationName ?? this.appName;
|
|
3535
4123
|
const client = await this.connect();
|
|
3536
4124
|
try {
|
|
3537
4125
|
await client.query('BEGIN');
|
|
3538
|
-
const
|
|
4126
|
+
const existedParams = [record.name];
|
|
4127
|
+
const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE ${this.global(undefined, existedParams)} AND name = $1`, existedParams);
|
|
3539
4128
|
// A name collision is a conflict in every mode: the name is the queue's address.
|
|
3540
4129
|
const resolvedOwner = await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
4130
|
+
const keys = this.globalValues();
|
|
3541
4131
|
await client.query(`INSERT INTO "${this.schemaName}".queues
|
|
3542
|
-
(
|
|
4132
|
+
(${Object.keys(keys)
|
|
4133
|
+
.map((c) => `${c}, `)
|
|
4134
|
+
.join('')}name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
|
|
3543
4135
|
priority_enabled, partition_queue, partition_concurrency, partition_worker_concurrency,
|
|
3544
4136
|
partition_rate_limit_max, partition_rate_limit_period_sec,
|
|
3545
4137
|
polling_interval_sec, updated_at, application_name)
|
|
3546
4138
|
-- priority_enabled is vestigial: every queue dispatches in priority order.
|
|
3547
|
-
VALUES ($
|
|
4139
|
+
VALUES (${Object.keys(keys)
|
|
4140
|
+
.map((_, i) => `$${14 + i}, `)
|
|
4141
|
+
.join('')}$1, $2, $3, $4, $5, TRUE, $6, $7, $8, $9, $10, $11, $12, $13)
|
|
3548
4142
|
${onConflict}`, [
|
|
3549
4143
|
record.name,
|
|
3550
4144
|
record.concurrency,
|
|
@@ -3559,6 +4153,7 @@ class SystemDatabase {
|
|
|
3559
4153
|
record.pollingIntervalSec,
|
|
3560
4154
|
now,
|
|
3561
4155
|
resolvedOwner ?? null,
|
|
4156
|
+
...Object.values(keys),
|
|
3562
4157
|
]);
|
|
3563
4158
|
// Read back, since the guard above is silent about why it declined to claim.
|
|
3564
4159
|
await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
@@ -3668,6 +4263,9 @@ class SystemDatabase {
|
|
|
3668
4263
|
params.push(statuses);
|
|
3669
4264
|
where = `${where} AND status = ANY($${params.length})`;
|
|
3670
4265
|
}
|
|
4266
|
+
else {
|
|
4267
|
+
where = `${where} AND ${this.global(undefined, params)}`;
|
|
4268
|
+
}
|
|
3671
4269
|
const res = await client.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${where}`, params);
|
|
3672
4270
|
return res.rowCount ?? 0;
|
|
3673
4271
|
};
|
|
@@ -3692,84 +4290,64 @@ class SystemDatabase {
|
|
|
3692
4290
|
}
|
|
3693
4291
|
// ==================== Internal ====================
|
|
3694
4292
|
async insertWorkflowStatus(client, initStatus, creatorXid) {
|
|
4293
|
+
const values = {
|
|
4294
|
+
...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
|
|
4295
|
+
workflow_uuid: initStatus.workflowUUID,
|
|
4296
|
+
status: initStatus.status,
|
|
4297
|
+
name: initStatus.workflowName,
|
|
4298
|
+
// For cross-language compatibility, these variables MUST be NULL in the database when not set
|
|
4299
|
+
class_name: initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
|
|
4300
|
+
config_name: initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
|
|
4301
|
+
queue_name: initStatus.queueName ?? null,
|
|
4302
|
+
authenticated_user: initStatus.authenticatedUser,
|
|
4303
|
+
assumed_role: initStatus.assumedRole,
|
|
4304
|
+
authenticated_roles: JSON.stringify(initStatus.authenticatedRoles),
|
|
4305
|
+
executor_id: initStatus.executorId,
|
|
4306
|
+
application_version: initStatus.applicationVersion ?? null,
|
|
4307
|
+
application_id: initStatus.applicationID,
|
|
4308
|
+
recovery_attempts: initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
|
|
4309
|
+
workflow_timeout_ms: initStatus.timeoutMS ?? null,
|
|
4310
|
+
workflow_deadline_epoch_ms: initStatus.deadlineEpochMS ?? null,
|
|
4311
|
+
// Legacy column: the payload lives in workflow_input.
|
|
4312
|
+
inputs: null,
|
|
4313
|
+
deduplication_id: initStatus.deduplicationID ?? null,
|
|
4314
|
+
priority: initStatus.priority,
|
|
4315
|
+
queue_partition_key: initStatus.queuePartitionKey ?? null,
|
|
4316
|
+
forked_from: initStatus.forkedFrom ?? null,
|
|
4317
|
+
parent_workflow_id: initStatus.parentWorkflowID ?? null,
|
|
4318
|
+
...this.parentValues(initStatus.parentWorkflowID),
|
|
4319
|
+
serialization: initStatus.serialization,
|
|
4320
|
+
creator_xid: creatorXid,
|
|
4321
|
+
delay_until_epoch_ms: initStatus.delayUntilEpochMS ?? null,
|
|
4322
|
+
attributes: initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
|
|
4323
|
+
schedule_name: initStatus.scheduleName ?? null,
|
|
4324
|
+
debounce_deadline_epoch_ms: initStatus.debounceDeadlineEpochMS ?? null,
|
|
4325
|
+
is_debounced: initStatus.isDebounced ?? false,
|
|
4326
|
+
application_name: initStatus.applicationName ?? null,
|
|
4327
|
+
// A direct start runs at once, so its creator token also owns the execution.
|
|
4328
|
+
owner_xid: initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
|
|
4329
|
+
};
|
|
4330
|
+
const columns = Object.keys(values);
|
|
4331
|
+
const inputValues = {
|
|
4332
|
+
...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
|
|
4333
|
+
workflow_uuid: initStatus.workflowUUID,
|
|
4334
|
+
inputs: initStatus.input ?? null,
|
|
4335
|
+
};
|
|
4336
|
+
const inputColumns = Object.keys(inputValues);
|
|
3695
4337
|
try {
|
|
3696
|
-
const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
name,
|
|
3700
|
-
class_name,
|
|
3701
|
-
config_name,
|
|
3702
|
-
queue_name,
|
|
3703
|
-
authenticated_user,
|
|
3704
|
-
assumed_role,
|
|
3705
|
-
authenticated_roles,
|
|
3706
|
-
executor_id,
|
|
3707
|
-
application_version,
|
|
3708
|
-
application_id,
|
|
3709
|
-
recovery_attempts,
|
|
3710
|
-
workflow_timeout_ms,
|
|
3711
|
-
workflow_deadline_epoch_ms,
|
|
3712
|
-
inputs,
|
|
3713
|
-
deduplication_id,
|
|
3714
|
-
priority,
|
|
3715
|
-
queue_partition_key,
|
|
3716
|
-
forked_from,
|
|
3717
|
-
parent_workflow_id,
|
|
3718
|
-
serialization,
|
|
3719
|
-
creator_xid,
|
|
3720
|
-
delay_until_epoch_ms,
|
|
3721
|
-
attributes,
|
|
3722
|
-
schedule_name,
|
|
3723
|
-
debounce_deadline_epoch_ms,
|
|
3724
|
-
is_debounced,
|
|
3725
|
-
application_name,
|
|
3726
|
-
owner_xid
|
|
3727
|
-
) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)
|
|
3728
|
-
ON CONFLICT (workflow_uuid)
|
|
4338
|
+
const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
|
|
4339
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
4340
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')})
|
|
3729
4341
|
-- A no-op update, so an existing row comes back unchanged for the caller to inspect.
|
|
3730
4342
|
DO UPDATE SET creator_xid = workflow_status.creator_xid
|
|
3731
|
-
RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`,
|
|
3732
|
-
initStatus.workflowUUID,
|
|
3733
|
-
initStatus.status,
|
|
3734
|
-
initStatus.workflowName,
|
|
3735
|
-
// For cross-language compatibility, these variables MUST be NULL in the database when not set
|
|
3736
|
-
initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
|
|
3737
|
-
initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
|
|
3738
|
-
initStatus.queueName ?? null,
|
|
3739
|
-
initStatus.authenticatedUser,
|
|
3740
|
-
initStatus.assumedRole,
|
|
3741
|
-
JSON.stringify(initStatus.authenticatedRoles),
|
|
3742
|
-
initStatus.executorId,
|
|
3743
|
-
initStatus.applicationVersion ?? null,
|
|
3744
|
-
initStatus.applicationID,
|
|
3745
|
-
initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
|
|
3746
|
-
initStatus.timeoutMS ?? null,
|
|
3747
|
-
initStatus.deadlineEpochMS ?? null,
|
|
3748
|
-
// Legacy column: the payload lives in workflow_input.
|
|
3749
|
-
null,
|
|
3750
|
-
initStatus.deduplicationID ?? null,
|
|
3751
|
-
initStatus.priority,
|
|
3752
|
-
initStatus.queuePartitionKey ?? null,
|
|
3753
|
-
initStatus.forkedFrom ?? null,
|
|
3754
|
-
initStatus.parentWorkflowID ?? null,
|
|
3755
|
-
initStatus.serialization,
|
|
3756
|
-
creatorXid,
|
|
3757
|
-
initStatus.delayUntilEpochMS ?? null,
|
|
3758
|
-
initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
|
|
3759
|
-
initStatus.scheduleName ?? null,
|
|
3760
|
-
initStatus.debounceDeadlineEpochMS ?? null,
|
|
3761
|
-
initStatus.isDebounced ?? false,
|
|
3762
|
-
initStatus.applicationName ?? null,
|
|
3763
|
-
// A direct start runs at once, so its creator token also owns the execution.
|
|
3764
|
-
initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
|
|
3765
|
-
]);
|
|
4343
|
+
RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`, Object.values(values));
|
|
3766
4344
|
if (rows.length === 0) {
|
|
3767
4345
|
throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
|
|
3768
4346
|
}
|
|
3769
4347
|
// Two statements, not a data-modifying CTE: at scale the CTE costs more than the round trip it saves.
|
|
3770
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (
|
|
3771
|
-
VALUES ($1,
|
|
3772
|
-
ON CONFLICT (workflow_uuid) DO NOTHING`,
|
|
4348
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (${inputColumns.join(', ')})
|
|
4349
|
+
VALUES (${inputColumns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
4350
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`, Object.values(inputValues));
|
|
3773
4351
|
const ret = rows[0];
|
|
3774
4352
|
ret.class_name = ret.class_name ?? '';
|
|
3775
4353
|
ret.config_name = ret.config_name ?? '';
|
|
@@ -3785,16 +4363,17 @@ class SystemDatabase {
|
|
|
3785
4363
|
}
|
|
3786
4364
|
}
|
|
3787
4365
|
async getWorkflowStatusValue(client, workflowID) {
|
|
3788
|
-
const
|
|
4366
|
+
const params = [];
|
|
4367
|
+
const { rows } = await client.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
|
|
3789
4368
|
return rows.length === 0 ? undefined : rows[0].status;
|
|
3790
4369
|
}
|
|
3791
4370
|
async updateWorkflowStatus(client, workflowID, status, options = {}) {
|
|
3792
4371
|
// Use SQL now() so updated_at and completed_at (when set together) are
|
|
3793
4372
|
// computed in the same statement against the same clock.
|
|
3794
4373
|
const nowMsExpr = `(EXTRACT(EPOCH FROM now()) * 1000)::bigint`;
|
|
3795
|
-
let setClause = `SET status=$
|
|
3796
|
-
|
|
3797
|
-
|
|
4374
|
+
let setClause = `SET status=$1, updated_at=${nowMsExpr}`;
|
|
4375
|
+
const args = [status];
|
|
4376
|
+
let whereClause = `WHERE ${this.wf(undefined, workflowID, args)}`;
|
|
3798
4377
|
const update = options.update ?? {};
|
|
3799
4378
|
if (update.resetRecoveryAttempts) {
|
|
3800
4379
|
setClause += `, recovery_attempts = 0`;
|
|
@@ -3853,27 +4432,31 @@ class SystemDatabase {
|
|
|
3853
4432
|
async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
3854
4433
|
const ownerXid = options.ownerChecked ? undefined : (0, context_1.currentOwnerXid)(workflowID);
|
|
3855
4434
|
if (ownerXid !== undefined) {
|
|
3856
|
-
await this
|
|
3857
|
-
}
|
|
4435
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
4436
|
+
}
|
|
4437
|
+
const values = {
|
|
4438
|
+
...this.wfValues(workflowID),
|
|
4439
|
+
workflow_uuid: workflowID,
|
|
4440
|
+
function_id: functionID,
|
|
4441
|
+
output: options.output ?? null,
|
|
4442
|
+
error: options.error ?? null,
|
|
4443
|
+
function_name: functionName,
|
|
4444
|
+
child_workflow_id: options.childWorkflowID ?? null,
|
|
4445
|
+
...(options.childValues ?? this.childValues(options.childWorkflowID)),
|
|
4446
|
+
started_at_epoch_ms: startTimeEpochMs,
|
|
4447
|
+
completed_at_epoch_ms: endTimeEpochMs,
|
|
4448
|
+
serialization: options.serialization ?? null,
|
|
4449
|
+
// Mirrors the parent: only the running application records its steps.
|
|
4450
|
+
application_name: this.appName ?? null,
|
|
4451
|
+
};
|
|
4452
|
+
const columns = Object.keys(values);
|
|
3858
4453
|
try {
|
|
3859
4454
|
const out = await client.query(`INSERT INTO "${this.schemaName}".operation_outputs
|
|
3860
|
-
(
|
|
3861
|
-
VALUES ($
|
|
3862
|
-
ON CONFLICT (workflow_uuid, function_id) DO UPDATE
|
|
4455
|
+
(${columns.join(', ')}, retention_timestamp)
|
|
4456
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')}, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
|
|
4457
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id')}) DO UPDATE
|
|
3863
4458
|
SET completed_at_epoch_ms = operation_outputs.completed_at_epoch_ms
|
|
3864
|
-
RETURNING completed_at_epoch_ms;`,
|
|
3865
|
-
workflowID,
|
|
3866
|
-
functionID,
|
|
3867
|
-
options.output ?? null,
|
|
3868
|
-
options.error ?? null,
|
|
3869
|
-
functionName,
|
|
3870
|
-
options.childWorkflowID ?? null,
|
|
3871
|
-
startTimeEpochMs,
|
|
3872
|
-
endTimeEpochMs,
|
|
3873
|
-
options.serialization ?? null,
|
|
3874
|
-
// Mirrors the parent: only the running application records its steps.
|
|
3875
|
-
this.appName ?? null,
|
|
3876
|
-
]);
|
|
4459
|
+
RETURNING completed_at_epoch_ms;`, Object.values(values));
|
|
3877
4460
|
// A row already there with another completion time is this execution's own doing
|
|
3878
4461
|
// (a retry carries the same time), so it is nondeterminism, not a duplicate.
|
|
3879
4462
|
if (checkConflict &&
|
|
@@ -3890,7 +4473,7 @@ class SystemDatabase {
|
|
|
3890
4473
|
}
|
|
3891
4474
|
}
|
|
3892
4475
|
/** Run `fn` in a READ COMMITTED transaction on `client`, so an ownership check holds its row lock until the write commits. */
|
|
3893
|
-
async
|
|
4476
|
+
async inTransaction(client, fn) {
|
|
3894
4477
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
3895
4478
|
let result;
|
|
3896
4479
|
try {
|
|
@@ -3905,22 +4488,26 @@ class SystemDatabase {
|
|
|
3905
4488
|
return result;
|
|
3906
4489
|
}
|
|
3907
4490
|
/** Throw DBOSWorkflowConflictError unless ownerXid still owns the workflow; the row lock holds until commit, so no hand-off lands in between. */
|
|
3908
|
-
async
|
|
3909
|
-
const
|
|
4491
|
+
async checkOwner(client, workflowID, ownerXid) {
|
|
4492
|
+
const params = [];
|
|
4493
|
+
const { rows } = await client.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)} FOR NO KEY UPDATE`, params);
|
|
3910
4494
|
if (rows[0]?.owner_xid !== ownerXid) {
|
|
3911
4495
|
throw new error_1.DBOSWorkflowConflictError(workflowID);
|
|
3912
4496
|
}
|
|
3913
4497
|
}
|
|
3914
4498
|
/** The workflow's current ownership token; null if unowned or missing. */
|
|
3915
4499
|
async getWorkflowOwner(workflowID) {
|
|
3916
|
-
const
|
|
4500
|
+
const params = [];
|
|
4501
|
+
const { rows } = await this.pool.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
|
|
3917
4502
|
return rows[0]?.owner_xid ?? null;
|
|
3918
4503
|
}
|
|
3919
4504
|
async #getOperationResultAndThrowIfCancelled(client, workflowID, functionID) {
|
|
3920
4505
|
await this.#checkIfCanceled(client, workflowID);
|
|
3921
|
-
const
|
|
4506
|
+
const childKey = this.keyColumn(undefined, 'child_shard_key');
|
|
4507
|
+
const params = [functionID];
|
|
4508
|
+
const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name, serialization${childKey !== undefined ? `, ${childKey}` : ''}
|
|
3922
4509
|
FROM "${this.schemaName}".operation_outputs
|
|
3923
|
-
WHERE
|
|
4510
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
|
|
3924
4511
|
if (rows.length === 0) {
|
|
3925
4512
|
return undefined;
|
|
3926
4513
|
}
|
|
@@ -3929,6 +4516,7 @@ class SystemDatabase {
|
|
|
3929
4516
|
output: rows[0].output,
|
|
3930
4517
|
error: rows[0].error,
|
|
3931
4518
|
childWorkflowID: rows[0].child_workflow_id,
|
|
4519
|
+
childShardKey: rows[0].child_shard_key ?? undefined,
|
|
3932
4520
|
functionName: rows[0].function_name,
|
|
3933
4521
|
// Return serialization so recv/getEvent replay deserializes with the stored format, not the default.
|
|
3934
4522
|
serialization: rows[0].serialization,
|
|
@@ -3966,7 +4554,11 @@ class SystemDatabase {
|
|
|
3966
4554
|
// been cancelled.
|
|
3967
4555
|
// For an actual sleep, completed_at is the wake deadline so the step's duration reflects the
|
|
3968
4556
|
// sleep; a timeout marker records zero duration since its deadline may never be reached.
|
|
3969
|
-
async
|
|
4557
|
+
async durableSleepInternal(workflowID, functionID, durationMS, recordCompletionAtDeadline = false) {
|
|
4558
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
4559
|
+
return await this.ownKey(workflowID, undefined, () => this.#durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline));
|
|
4560
|
+
}
|
|
4561
|
+
async #durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline) {
|
|
3970
4562
|
const startTimeMs = Date.now();
|
|
3971
4563
|
// Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
|
|
3972
4564
|
const endTimeMs = startTimeMs + Math.ceil(durationMS);
|
|
@@ -3979,7 +4571,7 @@ class SystemDatabase {
|
|
|
3979
4571
|
}
|
|
3980
4572
|
return JSON.parse(res.output);
|
|
3981
4573
|
}
|
|
3982
|
-
await this
|
|
4574
|
+
await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
|
|
3983
4575
|
output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
|
|
3984
4576
|
serialization: serialization_1.DBOSPortableJSON.name(),
|
|
3985
4577
|
}));
|
|
@@ -4159,7 +4751,7 @@ __decorate([
|
|
|
4159
4751
|
__decorate([
|
|
4160
4752
|
dbRetry(),
|
|
4161
4753
|
__metadata("design:type", Function),
|
|
4162
|
-
__metadata("design:paramtypes", [String, Number]),
|
|
4754
|
+
__metadata("design:paramtypes", [String, Number, String]),
|
|
4163
4755
|
__metadata("design:returntype", Promise)
|
|
4164
4756
|
], SystemDatabase.prototype, "getOperationResultAndThrowIfCancelled", null);
|
|
4165
4757
|
__decorate([
|
|
@@ -4219,9 +4811,9 @@ __decorate([
|
|
|
4219
4811
|
__decorate([
|
|
4220
4812
|
dbRetry(),
|
|
4221
4813
|
__metadata("design:type", Function),
|
|
4222
|
-
__metadata("design:paramtypes", [Array
|
|
4814
|
+
__metadata("design:paramtypes", [Array]),
|
|
4223
4815
|
__metadata("design:returntype", Promise)
|
|
4224
|
-
], SystemDatabase.prototype, "
|
|
4816
|
+
], SystemDatabase.prototype, "deleteWorkflowsInternal", null);
|
|
4225
4817
|
__decorate([
|
|
4226
4818
|
dbRetry()
|
|
4227
4819
|
// A missing row normally means the workflow has not been inserted yet, so
|
|
@@ -4255,19 +4847,19 @@ __decorate([
|
|
|
4255
4847
|
__decorate([
|
|
4256
4848
|
dbRetry(),
|
|
4257
4849
|
__metadata("design:type", Function),
|
|
4258
|
-
__metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String]),
|
|
4850
|
+
__metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String, String]),
|
|
4259
4851
|
__metadata("design:returntype", Promise)
|
|
4260
4852
|
], SystemDatabase.prototype, "send", null);
|
|
4261
4853
|
__decorate([
|
|
4262
4854
|
dbRetry(),
|
|
4263
4855
|
__metadata("design:type", Function),
|
|
4264
|
-
__metadata("design:paramtypes", [String, String, Object, Object, Object, String]),
|
|
4856
|
+
__metadata("design:paramtypes", [String, String, Object, Object, Object, String, String]),
|
|
4265
4857
|
__metadata("design:returntype", Promise)
|
|
4266
4858
|
], SystemDatabase.prototype, "sendFromStep", null);
|
|
4267
4859
|
__decorate([
|
|
4268
4860
|
dbRetry(),
|
|
4269
4861
|
__metadata("design:type", Function),
|
|
4270
|
-
__metadata("design:paramtypes", [String, Object, Object, Object, String]),
|
|
4862
|
+
__metadata("design:paramtypes", [String, Object, Object, Object, String, String]),
|
|
4271
4863
|
__metadata("design:returntype", Promise)
|
|
4272
4864
|
], SystemDatabase.prototype, "sendDirectStandalone", null);
|
|
4273
4865
|
__decorate([
|
|
@@ -4321,7 +4913,7 @@ __decorate([
|
|
|
4321
4913
|
__decorate([
|
|
4322
4914
|
dbRetry(),
|
|
4323
4915
|
__metadata("design:type", Function),
|
|
4324
|
-
__metadata("design:paramtypes", [Object]),
|
|
4916
|
+
__metadata("design:paramtypes", [Object, Object]),
|
|
4325
4917
|
__metadata("design:returntype", Promise)
|
|
4326
4918
|
], SystemDatabase.prototype, "listWorkflows", null);
|
|
4327
4919
|
__decorate([
|