@dbos-inc/dbos-sdk 5.3.12-preview → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/auto_upgrade.d.ts +27 -0
- package/dist/src/auto_upgrade.d.ts.map +1 -0
- package/dist/src/auto_upgrade.js +84 -0
- package/dist/src/auto_upgrade.js.map +1 -0
- package/dist/src/cli/cli.js +56 -13
- package/dist/src/cli/cli.js.map +1 -1
- package/dist/src/client.d.ts +20 -10
- package/dist/src/client.d.ts.map +1 -1
- package/dist/src/client.js +76 -44
- package/dist/src/client.js.map +1 -1
- package/dist/src/config.d.ts +1 -0
- package/dist/src/config.d.ts.map +1 -1
- package/dist/src/config.js +12 -0
- package/dist/src/config.js.map +1 -1
- package/dist/src/context.d.ts +6 -0
- package/dist/src/context.d.ts.map +1 -1
- package/dist/src/context.js +15 -1
- package/dist/src/context.js.map +1 -1
- package/dist/src/datasource.d.ts +2 -0
- package/dist/src/datasource.d.ts.map +1 -1
- package/dist/src/datasource.js +7 -1
- package/dist/src/datasource.js.map +1 -1
- package/dist/src/dbos-executor.d.ts +39 -4
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js +89 -56
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts +60 -11
- package/dist/src/dbos.d.ts.map +1 -1
- package/dist/src/dbos.js +240 -79
- package/dist/src/dbos.js.map +1 -1
- package/dist/src/enqueue_options.d.ts +2 -0
- package/dist/src/enqueue_options.d.ts.map +1 -1
- package/dist/src/enqueue_options.js +3 -0
- package/dist/src/enqueue_options.js.map +1 -1
- package/dist/src/enqueue_workflow.d.ts.map +1 -1
- package/dist/src/enqueue_workflow.js +11 -5
- package/dist/src/enqueue_workflow.js.map +1 -1
- package/dist/src/enterprise.d.ts +67 -2
- package/dist/src/enterprise.d.ts.map +1 -1
- package/dist/src/enterprise.js +185 -9
- package/dist/src/enterprise.js.map +1 -1
- package/dist/src/eventreceiver.d.ts +2 -0
- package/dist/src/eventreceiver.d.ts.map +1 -1
- package/dist/src/eventreceiver.js +7 -1
- package/dist/src/eventreceiver.js.map +1 -1
- package/dist/src/index.d.ts +2 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/scheduler/scheduler.d.ts +5 -0
- package/dist/src/scheduler/scheduler.d.ts.map +1 -1
- package/dist/src/scheduler/scheduler.js +14 -4
- package/dist/src/scheduler/scheduler.js.map +1 -1
- package/dist/src/shard_keys.d.ts +13 -0
- package/dist/src/shard_keys.d.ts.map +1 -0
- package/dist/src/shard_keys.js +55 -0
- package/dist/src/shard_keys.js.map +1 -0
- package/dist/src/streams.d.ts +2 -0
- package/dist/src/streams.d.ts.map +1 -1
- package/dist/src/streams.js +5 -2
- package/dist/src/streams.js.map +1 -1
- package/dist/src/sysdb_migrations/internal/migrations.js +16 -0
- package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
- package/dist/src/system_database.d.ts +143 -9
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +1134 -519
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/wfqueue.d.ts +1 -2
- package/dist/src/wfqueue.d.ts.map +1 -1
- package/dist/src/wfqueue.js +83 -63
- package/dist/src/wfqueue.js.map +1 -1
- package/dist/src/workflow.d.ts +19 -2
- package/dist/src/workflow.d.ts.map +1 -1
- package/dist/src/workflow.js +14 -9
- package/dist/src/workflow.js.map +1 -1
- package/dist/src/workflow_management.d.ts +6 -3
- package/dist/src/workflow_management.d.ts.map +1 -1
- package/dist/src/workflow_management.js +46 -14
- package/dist/src/workflow_management.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +6 -5
|
@@ -1,15 +1,38 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
2
18
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
19
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
20
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
21
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
22
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
23
|
};
|
|
24
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
25
|
+
if (mod && mod.__esModule) return mod;
|
|
26
|
+
var result = {};
|
|
27
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
28
|
+
__setModuleDefault(result, mod);
|
|
29
|
+
return result;
|
|
30
|
+
};
|
|
8
31
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
32
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
33
|
};
|
|
11
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.SystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
35
|
+
exports.SystemDatabase = exports.createSystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.qualify = exports.holdsShardedSchema = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
13
36
|
const dbos_executor_1 = require("./dbos-executor");
|
|
14
37
|
const pg_1 = require("pg");
|
|
15
38
|
const error_1 = require("./error");
|
|
@@ -18,6 +41,8 @@ const utils_1 = require("./utils");
|
|
|
18
41
|
const context_1 = require("./context");
|
|
19
42
|
const async_hooks_1 = require("async_hooks");
|
|
20
43
|
const crypto_1 = require("crypto");
|
|
44
|
+
const shard_keys_1 = require("./shard_keys");
|
|
45
|
+
const enterprise = __importStar(require("./enterprise"));
|
|
21
46
|
const utils_2 = require("./utils");
|
|
22
47
|
const database_utils_1 = require("./database_utils");
|
|
23
48
|
const migration_runner_1 = require("./sysdb_migrations/migration_runner");
|
|
@@ -70,6 +95,8 @@ function validateObservabilityQueryTimeoutMs(value) {
|
|
|
70
95
|
exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutMs;
|
|
71
96
|
// Bounds how long a frozen or unreachable client can hold system database locks.
|
|
72
97
|
exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = 60_000;
|
|
98
|
+
// How long a background job's lock may sit idle before Postgres ends its holder's session.
|
|
99
|
+
exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = 5000;
|
|
73
100
|
/** The timeout is rendered into a `SET idle_in_transaction_session_timeout`, which takes only an in-range integer. */
|
|
74
101
|
function validateIdleTransactionTimeoutMs(value) {
|
|
75
102
|
if (value !== undefined && (!Number.isFinite(value) || value > MAX_STATEMENT_TIMEOUT_MS)) {
|
|
@@ -157,6 +184,7 @@ async function grantDbosSchemaPermissions(databaseUrl, roleName, logger, schemaN
|
|
|
157
184
|
await client.end();
|
|
158
185
|
}
|
|
159
186
|
}
|
|
187
|
+
exports.grantDbosSchemaPermissions = grantDbosSchemaPermissions;
|
|
160
188
|
/**
|
|
161
189
|
* Check out a connection carrying our own error handler, so a socket death while we hold it is
|
|
162
190
|
* logged instead of thrown at an emitter with no listener. `release()` takes the handler back off,
|
|
@@ -219,6 +247,7 @@ async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = '
|
|
|
219
247
|
}
|
|
220
248
|
const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
|
|
221
249
|
try {
|
|
250
|
+
await refuseShardedSchema(client, schemaName);
|
|
222
251
|
const isCockroach = await isCockroachDB(client);
|
|
223
252
|
await (0, migration_runner_1.runSysMigrationsPg)(client, (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }), schemaName, {
|
|
224
253
|
onWarn: (e) => logger.info(e),
|
|
@@ -242,6 +271,7 @@ exports.migrateSystemDatabase = migrateSystemDatabase;
|
|
|
242
271
|
async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
|
|
243
272
|
const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
|
|
244
273
|
try {
|
|
274
|
+
await refuseShardedSchema(client, schemaName);
|
|
245
275
|
const isCockroach = await isCockroachDB(client);
|
|
246
276
|
const requiredVersion = (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }).length;
|
|
247
277
|
const currentVersion = await (0, migration_runner_1.getCurrentSysDBVersion)(client, schemaName);
|
|
@@ -259,6 +289,41 @@ async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = '
|
|
|
259
289
|
}
|
|
260
290
|
}
|
|
261
291
|
exports.verifySystemDatabase = verifySystemDatabase;
|
|
292
|
+
/** Throw if this schema is sharded: an unsharded handle's statements would span every tenant's rows. */
|
|
293
|
+
async function refuseShardedSchema(client, schemaName) {
|
|
294
|
+
const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
|
|
295
|
+
WHERE table_schema = $1 AND table_name = 'workflow_status' AND column_name = 'shard_key'`, [schemaName]);
|
|
296
|
+
if (rows.length > 0) {
|
|
297
|
+
throw new error_1.DBOSError(`The DBOS system schema ${schemaName} is sharded. Enable sharding to use it: ` +
|
|
298
|
+
'set sharding in your DBOS configuration, or pass sharding: true to DBOSClient.');
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
/** Whether this database holds a sharded DBOS system schema (this one, if named); false if it can't be reached. */
|
|
302
|
+
async function holdsShardedSchema(databaseUrl, schemaName) {
|
|
303
|
+
let client;
|
|
304
|
+
try {
|
|
305
|
+
// Bounded, so an unreachable host fails as fast as the caller's own connection would.
|
|
306
|
+
client = new pg_1.Client({ ...(0, utils_2.getClientConfig)(databaseUrl), connectionTimeoutMillis: 10_000 });
|
|
307
|
+
}
|
|
308
|
+
catch {
|
|
309
|
+
// A malformed URL, for the caller's own connection to report.
|
|
310
|
+
return false;
|
|
311
|
+
}
|
|
312
|
+
client.on('error', () => { });
|
|
313
|
+
try {
|
|
314
|
+
await client.connect();
|
|
315
|
+
const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
|
|
316
|
+
WHERE table_name = 'workflow_status' AND column_name = 'shard_key'${schemaName !== undefined ? ' AND table_schema = $1' : ''} LIMIT 1`, schemaName !== undefined ? [schemaName] : []);
|
|
317
|
+
return rows.length > 0;
|
|
318
|
+
}
|
|
319
|
+
catch {
|
|
320
|
+
return false;
|
|
321
|
+
}
|
|
322
|
+
finally {
|
|
323
|
+
await client.end().catch(() => { });
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
exports.holdsShardedSchema = holdsShardedSchema;
|
|
262
327
|
class NotificationMap {
|
|
263
328
|
map = new Map();
|
|
264
329
|
curCK = 0;
|
|
@@ -300,6 +365,23 @@ function mapVersionInfo(row) {
|
|
|
300
365
|
};
|
|
301
366
|
}
|
|
302
367
|
const SCHEDULE_COLUMNS = 'schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name';
|
|
368
|
+
/** A schedule's own columns, as a new row stores them. */
|
|
369
|
+
function scheduleRow(schedule, owner) {
|
|
370
|
+
return {
|
|
371
|
+
schedule_id: schedule.scheduleId,
|
|
372
|
+
schedule_name: schedule.scheduleName,
|
|
373
|
+
workflow_name: schedule.workflowName,
|
|
374
|
+
workflow_class_name: schedule.workflowClassName,
|
|
375
|
+
schedule: schedule.schedule,
|
|
376
|
+
status: schedule.status,
|
|
377
|
+
context: schedule.context,
|
|
378
|
+
last_fired_at: schedule.lastFiredAt,
|
|
379
|
+
automatic_backfill: schedule.automaticBackfill,
|
|
380
|
+
cron_timezone: schedule.cronTimezone,
|
|
381
|
+
queue_name: schedule.queueName,
|
|
382
|
+
application_name: owner ?? null,
|
|
383
|
+
};
|
|
384
|
+
}
|
|
303
385
|
function mapWorkflowSchedule(row) {
|
|
304
386
|
return {
|
|
305
387
|
scheduleId: row.schedule_id,
|
|
@@ -314,6 +396,7 @@ function mapWorkflowSchedule(row) {
|
|
|
314
396
|
cronTimezone: row.cron_timezone ?? null,
|
|
315
397
|
queueName: row.queue_name ?? null,
|
|
316
398
|
applicationName: row.application_name ?? undefined,
|
|
399
|
+
shardKey: row.workflow_shard_key ?? undefined,
|
|
317
400
|
};
|
|
318
401
|
}
|
|
319
402
|
function mapWorkflowStatus(row) {
|
|
@@ -337,7 +420,10 @@ function mapWorkflowStatus(row) {
|
|
|
337
420
|
recoveryAttempts: Number(row.recovery_attempts),
|
|
338
421
|
input: row.inputs ? row.inputs : null,
|
|
339
422
|
timeoutMS: row.workflow_timeout_ms ? Number(row.workflow_timeout_ms) : undefined,
|
|
340
|
-
|
|
423
|
+
// Cap at the largest exact JS integer: a saturated bigint deadline would otherwise round past the bigint maximum and be rejected when a child workflow inherits it.
|
|
424
|
+
deadlineEpochMS: row.workflow_deadline_epoch_ms
|
|
425
|
+
? Math.min(Number(row.workflow_deadline_epoch_ms), Number.MAX_SAFE_INTEGER)
|
|
426
|
+
: undefined,
|
|
341
427
|
deduplicationID: row.deduplication_id ?? undefined,
|
|
342
428
|
priority: row.priority ?? 0,
|
|
343
429
|
queuePartitionKey: row.queue_partition_key ?? undefined,
|
|
@@ -353,8 +439,16 @@ function mapWorkflowStatus(row) {
|
|
|
353
439
|
debounceDeadlineEpochMS: row.debounce_deadline_epoch_ms ? Number(row.debounce_deadline_epoch_ms) : undefined,
|
|
354
440
|
isDebounced: row.is_debounced ?? false,
|
|
355
441
|
applicationName: row.application_name ?? undefined,
|
|
442
|
+
// Read back only from a sharded system database, whose rows carry them.
|
|
443
|
+
shardKey: row.shard_key ?? undefined,
|
|
444
|
+
parentShardKey: row.parent_shard_key ?? undefined,
|
|
356
445
|
};
|
|
357
446
|
}
|
|
447
|
+
/** `column`, qualified by `alias` if one is given. */
|
|
448
|
+
function qualify(alias, column) {
|
|
449
|
+
return alias ? `${alias}.${column}` : column;
|
|
450
|
+
}
|
|
451
|
+
exports.qualify = qualify;
|
|
358
452
|
// SQLSTATE classes/codes that are generally safe to retry
|
|
359
453
|
// https://www.postgresql.org/docs/current/errcodes-appendix.html
|
|
360
454
|
const RETRY_SQLSTATE_PREFIXES = new Set([
|
|
@@ -524,6 +618,26 @@ function dbRetry(options = {}) {
|
|
|
524
618
|
return descriptor;
|
|
525
619
|
};
|
|
526
620
|
}
|
|
621
|
+
/** Build the system database these options describe. */
|
|
622
|
+
function createSystemDatabase(options) {
|
|
623
|
+
if (options.sharding) {
|
|
624
|
+
const { ShardedSystemDatabase } = enterprise.load(enterprise.SHARDING.description);
|
|
625
|
+
return new ShardedSystemDatabase({
|
|
626
|
+
systemDatabaseUrl: options.systemDatabaseUrl,
|
|
627
|
+
logger: options.logger,
|
|
628
|
+
serializer: options.serializer,
|
|
629
|
+
sysDbPoolSize: options.sysDbPoolSize,
|
|
630
|
+
systemDatabasePool: options.systemDatabasePool,
|
|
631
|
+
schemaName: options.schemaName,
|
|
632
|
+
pollingConcurrency: options.pollingConcurrency,
|
|
633
|
+
appName: options.appName,
|
|
634
|
+
observabilityQueryTimeoutMs: options.observabilityQueryTimeoutMs,
|
|
635
|
+
idleTransactionTimeoutMs: options.idleTransactionTimeoutMs,
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
return new SystemDatabase(options.systemDatabaseUrl, options.logger, options.serializer, options.sysDbPoolSize, options.systemDatabasePool, options.schemaName, options.useListenNotify, options.pollingConcurrency, options.notificationCoalesceMs, options.appName, options.observabilityQueryTimeoutMs, options.idleTransactionTimeoutMs);
|
|
639
|
+
}
|
|
640
|
+
exports.createSystemDatabase = createSystemDatabase;
|
|
527
641
|
/**
|
|
528
642
|
* General notes:
|
|
529
643
|
* The responsibilities of the `SystemDatabase` are to store data for workflows, and
|
|
@@ -592,6 +706,7 @@ class SystemDatabase {
|
|
|
592
706
|
pollLimiter;
|
|
593
707
|
/** Executions running in this process, by workflow ID; a re-dispatch runs alongside its stale execution, which parks at its next write. */
|
|
594
708
|
runningWorkflowMap = new Map();
|
|
709
|
+
// Keyed by waiter key, as one ID can live under several shard keys.
|
|
595
710
|
#cancelWatchers = new Map();
|
|
596
711
|
#cancelPollerLoop = undefined;
|
|
597
712
|
#cancelPollerWake = null;
|
|
@@ -724,25 +839,222 @@ class SystemDatabase {
|
|
|
724
839
|
observabilityFilter(column, value, params) {
|
|
725
840
|
return this.appNameFilter(column, value ?? this.appName, params);
|
|
726
841
|
}
|
|
842
|
+
// ==================== Shard Keys ====================
|
|
843
|
+
// Statements locate rows through these, so a sharded subclass adds its key; each appends to `params` and returns SQL.
|
|
844
|
+
/** Whether this handle routes every statement by shard key. */
|
|
845
|
+
get sharded() {
|
|
846
|
+
return false;
|
|
847
|
+
}
|
|
848
|
+
#rejectShardKey(workflowID, shardKey) {
|
|
849
|
+
if ((shardKey !== undefined && shardKey !== null) || (0, shard_keys_1.boundShardKey)(workflowID) !== undefined) {
|
|
850
|
+
throw new error_1.DBOSError('A shard key was given, but this system database is not sharded');
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
/** Predicate selecting one workflow's rows: those whose `column` is `workflowID`. */
|
|
854
|
+
wf(alias, workflowID, params, column = 'workflow_uuid', shardKey) {
|
|
855
|
+
this.#rejectShardKey(workflowID, shardKey);
|
|
856
|
+
params.push(workflowID);
|
|
857
|
+
return `${qualify(alias, column)} = $${params.length}`;
|
|
858
|
+
}
|
|
859
|
+
/** Predicate selecting several workflows' rows. */
|
|
860
|
+
wfIn(alias, workflowIDs, params) {
|
|
861
|
+
for (const workflowID of workflowIDs) {
|
|
862
|
+
this.#rejectShardKey(workflowID, undefined);
|
|
863
|
+
}
|
|
864
|
+
params.push(workflowIDs);
|
|
865
|
+
return `${qualify(alias, 'workflow_uuid')} = ANY($${params.length}::text[])`;
|
|
866
|
+
}
|
|
867
|
+
/** Condition joining two tables' rows of the same workflow. */
|
|
868
|
+
wfJoin(left, right) {
|
|
869
|
+
return `${left}.workflow_uuid = ${right}.workflow_uuid`;
|
|
870
|
+
}
|
|
871
|
+
/** Extra column values for a new row of this workflow. */
|
|
872
|
+
wfValues(workflowID, shardKey) {
|
|
873
|
+
this.#rejectShardKey(workflowID, shardKey);
|
|
874
|
+
return {};
|
|
875
|
+
}
|
|
876
|
+
/** Extra column values storing the shard key of a new workflow's parent. */
|
|
877
|
+
parentValues(_parentWorkflowID) {
|
|
878
|
+
return {};
|
|
879
|
+
}
|
|
880
|
+
/** Extra column values storing the shard key of the workflow a step refers to. */
|
|
881
|
+
childValues(_childWorkflowID) {
|
|
882
|
+
return {};
|
|
883
|
+
}
|
|
884
|
+
/** Extra (column, expression) pairs for an INSERT ... SELECT of this workflow's rows. */
|
|
885
|
+
wfColumns(_workflowID, _params) {
|
|
886
|
+
return [];
|
|
887
|
+
}
|
|
888
|
+
/** Conflict target for a unique key of a workflow table. */
|
|
889
|
+
keyConflict(...columns) {
|
|
890
|
+
return columns.join(', ');
|
|
891
|
+
}
|
|
892
|
+
/** Insert rows into `table` in one statement, returning the RETURNING rows; a sharded subclass inserts row by row. */
|
|
893
|
+
async insertRows(client, table, columns, rows, suffix = '') {
|
|
894
|
+
if (rows.length === 0)
|
|
895
|
+
return [];
|
|
896
|
+
const params = [];
|
|
897
|
+
const tuples = rows.map((row) => `(${row.map((value) => `$${params.push(value)}`).join(', ')})`);
|
|
898
|
+
const { rows: returned } = await client.query(`INSERT INTO "${this.schemaName}".${table} (${columns.join(', ')}) VALUES ${tuples.join(', ')} ${suffix}`, params);
|
|
899
|
+
return returned;
|
|
900
|
+
}
|
|
901
|
+
/** Predicate a queue claim must also meet, evaluated once before it scans; sharded, it caps claimers per key. */
|
|
902
|
+
claimGate(_queueName, _partitionKey, _params) {
|
|
903
|
+
return 'TRUE';
|
|
904
|
+
}
|
|
905
|
+
/** Predicate confining a statement that names no single workflow to the current shard. */
|
|
906
|
+
scoped(_alias, _params) {
|
|
907
|
+
return 'TRUE';
|
|
908
|
+
}
|
|
909
|
+
/** Extra (column, expression) pairs for an INSERT ... SELECT confined to the current shard. */
|
|
910
|
+
scopedColumns(_params) {
|
|
911
|
+
return [];
|
|
912
|
+
}
|
|
913
|
+
/** Predicate selecting a global table's rows (queues, schedules, versions). */
|
|
914
|
+
global(_alias, _params) {
|
|
915
|
+
return 'TRUE';
|
|
916
|
+
}
|
|
917
|
+
/** Extra column values for a new row of a global table. */
|
|
918
|
+
globalValues() {
|
|
919
|
+
return {};
|
|
920
|
+
}
|
|
921
|
+
/** Predicate selecting rows whose `column` holds one of these shard keys. */
|
|
922
|
+
keyIn(_alias, _shardKeys, _params, _column = 'shard_key') {
|
|
923
|
+
throw new error_1.DBOSError('Filtering by shard key requires a sharded system database');
|
|
924
|
+
}
|
|
925
|
+
/** A shard key column to read back with a row, if this handle has one. */
|
|
926
|
+
keyColumn(_alias, _column = 'shard_key') {
|
|
927
|
+
return undefined;
|
|
928
|
+
}
|
|
929
|
+
/** Extra column values storing the shard key a schedule's workflows run under. */
|
|
930
|
+
scheduleValues(schedule) {
|
|
931
|
+
if (schedule.shardKey !== undefined && schedule.shardKey !== null) {
|
|
932
|
+
throw new error_1.DBOSError('Scheduling under a shard key requires a sharded system database');
|
|
933
|
+
}
|
|
934
|
+
return {};
|
|
935
|
+
}
|
|
936
|
+
/** The column holding the shard key a schedule's workflows run under, if any. */
|
|
937
|
+
scheduleKeyColumn() {
|
|
938
|
+
return undefined;
|
|
939
|
+
}
|
|
940
|
+
/** This workflow's shard key, resolved now for a statement or waiter that needs it later. */
|
|
941
|
+
keyHint(workflowID) {
|
|
942
|
+
this.#rejectShardKey(workflowID, undefined);
|
|
943
|
+
return undefined;
|
|
944
|
+
}
|
|
945
|
+
/** The notification-map key for a workflow's topic or key, distinct per shard key as IDs repeat across keys. */
|
|
946
|
+
waiter(workflowID, name) {
|
|
947
|
+
const key = this.keyHint(workflowID);
|
|
948
|
+
const payload = `${workflowID}::${name}`;
|
|
949
|
+
return key === undefined ? payload : `${[...key].length}:${key}:${payload}`;
|
|
950
|
+
}
|
|
951
|
+
/** Bind a workflow to `key`, else to the running workflow's own key even where a same-ID target is bound. */
|
|
952
|
+
ownKey(workflowID, key, callback) {
|
|
953
|
+
if (key === undefined) {
|
|
954
|
+
const ctx = (0, context_1.getCurrentContextStore)();
|
|
955
|
+
if (ctx !== undefined && ctx.workflowId === workflowID) {
|
|
956
|
+
key = ctx.shardKey;
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
return (0, shard_keys_1.bindShardKeys)([[workflowID, key]], callback);
|
|
960
|
+
}
|
|
961
|
+
/** Run `callback` under this application's lock for a background job, passing whether it was taken. */
|
|
962
|
+
async backgroundJobLock(jobName, callback) {
|
|
963
|
+
// CockroachDB grants every advisory lock, so each process runs every job.
|
|
964
|
+
if (await this.#isCockroach()) {
|
|
965
|
+
return await callback(true);
|
|
966
|
+
}
|
|
967
|
+
// Leading 8 bytes of SHA-256, big-endian signed, as for the retention lock.
|
|
968
|
+
const key = advisoryLockKey(`dbos.background_job.${this.schemaName}.${this.appName ?? ''}.${jobName}`);
|
|
969
|
+
return await this.lockedBackgroundJob(jobName, `SELECT pg_try_advisory_xact_lock($1) AS acquired FROM set_config('idle_in_transaction_session_timeout', $2, true)`, [key.toString(), String(exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS)], callback);
|
|
970
|
+
}
|
|
971
|
+
/** Take a transaction-scoped background job lock with `lockQuery`, which returns `acquired`, and run `callback`. */
|
|
972
|
+
async lockedBackgroundJob(jobName, lockQuery, lockParams, callback) {
|
|
973
|
+
const client = await this.connect();
|
|
974
|
+
try {
|
|
975
|
+
await client.query('BEGIN');
|
|
976
|
+
// The local idle timeout ends a frozen holder's session, releasing its lock.
|
|
977
|
+
const { rows } = await client.query(lockQuery, lockParams);
|
|
978
|
+
return await callback(rows[0]?.acquired === true);
|
|
979
|
+
}
|
|
980
|
+
finally {
|
|
981
|
+
try {
|
|
982
|
+
await client.query('ROLLBACK');
|
|
983
|
+
client.release();
|
|
984
|
+
}
|
|
985
|
+
catch (e) {
|
|
986
|
+
// A session the idle timeout ended has released the lock already.
|
|
987
|
+
this.logger.debug(`Could not release the ${jobName} lock: ${String(e)}`);
|
|
988
|
+
client.release(e instanceof Error ? e : true);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
#cockroach = undefined;
|
|
993
|
+
/** Whether this handle's database is CockroachDB, read on first use. */
|
|
994
|
+
#isCockroach() {
|
|
995
|
+
this.#cockroach ??= (async () => {
|
|
996
|
+
const client = await this.connect();
|
|
997
|
+
try {
|
|
998
|
+
return await isCockroachDB(client);
|
|
999
|
+
}
|
|
1000
|
+
finally {
|
|
1001
|
+
client.release();
|
|
1002
|
+
}
|
|
1003
|
+
})();
|
|
1004
|
+
// A failed read is retried on next use, rather than cached.
|
|
1005
|
+
this.#cockroach.catch(() => {
|
|
1006
|
+
this.#cockroach = undefined;
|
|
1007
|
+
});
|
|
1008
|
+
return this.#cockroach;
|
|
1009
|
+
}
|
|
1010
|
+
/** Scopes a queue's workers dequeue from in turn; one unkeyed scope unless sharded. */
|
|
1011
|
+
dequeueScopes(_queueName) {
|
|
1012
|
+
return Promise.resolve([undefined]);
|
|
1013
|
+
}
|
|
1014
|
+
/** Whether a dequeue failed because another worker holds the rows it needs. */
|
|
1015
|
+
isLockConflict(e) {
|
|
1016
|
+
return isPgDatabaseError(e) && e.code === '55P03';
|
|
1017
|
+
}
|
|
1018
|
+
/** Whether a dequeue failed on a serialization conflict with a concurrent one. */
|
|
1019
|
+
isSerializationConflict(e) {
|
|
1020
|
+
return isPgDatabaseError(e) && e.code === '40001';
|
|
1021
|
+
}
|
|
1022
|
+
/** Whether `appVersion` is this application's latest registered version, or none is registered. */
|
|
1023
|
+
async isLatestVersion(appVersion, client) {
|
|
1024
|
+
// Own plus unclaimed: a named peer's deploy must not demote this one.
|
|
1025
|
+
const latest = await this.#latestApplicationVersionName(client ?? this.pool);
|
|
1026
|
+
return latest === undefined || latest === appVersion;
|
|
1027
|
+
}
|
|
727
1028
|
/**
|
|
728
1029
|
* The version name a dequeue treats as latest: this application's own plus unclaimed
|
|
729
1030
|
* ones, so a named peer's deploy does not demote this one.
|
|
730
1031
|
*/
|
|
731
1032
|
async #latestApplicationVersionName(client) {
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
1033
|
+
return (await this.#latestApplicationVersion(client, this.appName))?.version_name;
|
|
1034
|
+
}
|
|
1035
|
+
async #latestApplicationVersion(client, applicationName) {
|
|
1036
|
+
const columns = 'version_id, version_name, version_timestamp, created_at, application_name';
|
|
1037
|
+
const table = `"${this.schemaName}".application_versions`;
|
|
1038
|
+
const params = applicationName ? [applicationName] : [];
|
|
1039
|
+
const scope = this.global(undefined, params);
|
|
1040
|
+
// Probe each eligible owner separately; an OR filter can scan an unbounded history of newer peer versions.
|
|
1041
|
+
// Keep the constant owner in the ordering so the IS NULL probe uses the composite index too.
|
|
1042
|
+
const query = applicationName
|
|
1043
|
+
? `(SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
1044
|
+
UNION ALL
|
|
1045
|
+
(SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
1046
|
+
ORDER BY version_timestamp DESC LIMIT 1`
|
|
1047
|
+
: `SELECT ${columns} FROM ${table} WHERE ${scope} ORDER BY version_timestamp DESC LIMIT 1`;
|
|
1048
|
+
const { rows } = await client.query(query, params);
|
|
1049
|
+
return rows[0];
|
|
739
1050
|
}
|
|
740
1051
|
/**
|
|
741
1052
|
* The owner to persist when writing a row that may already exist. A nameless writer
|
|
742
1053
|
* leaves the owner intact; a named one collides only with a different name.
|
|
743
1054
|
*/
|
|
744
1055
|
async #resolveRowOwner(client, table, keyColumn, name, owner, kind) {
|
|
745
|
-
const
|
|
1056
|
+
const params = [name];
|
|
1057
|
+
const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${this.global(undefined, params)} AND ${keyColumn} = $1`, params);
|
|
746
1058
|
const current = rows[0]?.application_name ?? null;
|
|
747
1059
|
if (current === null)
|
|
748
1060
|
return owner;
|
|
@@ -759,8 +1071,12 @@ class SystemDatabase {
|
|
|
759
1071
|
}
|
|
760
1072
|
/** Migrates the system database, or, when `runMigrations` is false, verifies it is already migrated. */
|
|
761
1073
|
async init(runMigrations = true) {
|
|
762
|
-
|
|
763
|
-
|
|
1074
|
+
if (runMigrations) {
|
|
1075
|
+
await this.runMigrations();
|
|
1076
|
+
}
|
|
1077
|
+
else {
|
|
1078
|
+
await this.verifyMigrations();
|
|
1079
|
+
}
|
|
764
1080
|
if (this.shouldUseDBNotifications) {
|
|
765
1081
|
await this.#listenForNotifications();
|
|
766
1082
|
// Push coalesced stream and event notifications off the write path.
|
|
@@ -768,6 +1084,32 @@ class SystemDatabase {
|
|
|
768
1084
|
this.#notifierLoop = this.#runNotifier();
|
|
769
1085
|
}
|
|
770
1086
|
}
|
|
1087
|
+
/** Create or migrate the system database. */
|
|
1088
|
+
async runMigrations() {
|
|
1089
|
+
await ensureSystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
|
|
1090
|
+
}
|
|
1091
|
+
/** Throw if this unsharded handle's schema is sharded; an unreachable database is left for its first statement. */
|
|
1092
|
+
async checkShardedSchema() {
|
|
1093
|
+
if (this.sharded)
|
|
1094
|
+
return;
|
|
1095
|
+
let client;
|
|
1096
|
+
try {
|
|
1097
|
+
client = await this.connect();
|
|
1098
|
+
}
|
|
1099
|
+
catch {
|
|
1100
|
+
return;
|
|
1101
|
+
}
|
|
1102
|
+
try {
|
|
1103
|
+
await refuseShardedSchema(client, this.schemaName);
|
|
1104
|
+
}
|
|
1105
|
+
finally {
|
|
1106
|
+
client.release();
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
/** Check the system database is migrated, creating and changing nothing. */
|
|
1110
|
+
async verifyMigrations() {
|
|
1111
|
+
await verifySystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
|
|
1112
|
+
}
|
|
771
1113
|
async destroy() {
|
|
772
1114
|
// Set synchronously, before any await, so no reconnect is scheduled or published after this point.
|
|
773
1115
|
this.#notificationsStopped = true;
|
|
@@ -813,7 +1155,7 @@ class SystemDatabase {
|
|
|
813
1155
|
/** Runs on `client` if given, joining its transaction; otherwise in its own retried transaction. */
|
|
814
1156
|
async initWorkflowStatus(initStatus, creatorXid, client, reusePolicy = 'return-existing') {
|
|
815
1157
|
if (client !== undefined) {
|
|
816
|
-
return await this
|
|
1158
|
+
return await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
817
1159
|
}
|
|
818
1160
|
return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
|
|
819
1161
|
}
|
|
@@ -822,7 +1164,7 @@ class SystemDatabase {
|
|
|
822
1164
|
let shouldCommit = false;
|
|
823
1165
|
try {
|
|
824
1166
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
825
|
-
const result = await this
|
|
1167
|
+
const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
826
1168
|
// If there is an existing DB record and we aren't here to recover it, leave it be.
|
|
827
1169
|
shouldCommit = result.shouldExecuteOnThisExecutor;
|
|
828
1170
|
return result;
|
|
@@ -850,10 +1192,12 @@ class SystemDatabase {
|
|
|
850
1192
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
851
1193
|
const parentOwnerXid = (0, context_1.currentOwnerXid)(parentWorkflowID);
|
|
852
1194
|
if (parentOwnerXid !== undefined) {
|
|
853
|
-
await this
|
|
1195
|
+
await this.checkOwner(client, parentWorkflowID, parentOwnerXid);
|
|
854
1196
|
}
|
|
855
|
-
const result = await this
|
|
856
|
-
await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs,
|
|
1197
|
+
const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
1198
|
+
await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs,
|
|
1199
|
+
// Unkeyed, as only a sharded database records the child's key, through its own override.
|
|
1200
|
+
{ childWorkflowID: initStatus.workflowUUID, ownerChecked: true, childValues: {} });
|
|
857
1201
|
shouldCommit = true;
|
|
858
1202
|
return result;
|
|
859
1203
|
}
|
|
@@ -872,7 +1216,7 @@ class SystemDatabase {
|
|
|
872
1216
|
}
|
|
873
1217
|
}
|
|
874
1218
|
}
|
|
875
|
-
async
|
|
1219
|
+
async initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy) {
|
|
876
1220
|
const resRow = await this.insertWorkflowStatus(client, initStatus, creatorXid);
|
|
877
1221
|
if (reusePolicy === 'reject' && (creatorXid === null || resRow.creator_xid !== creatorXid)) {
|
|
878
1222
|
throw new error_1.DBOSWorkflowIDInUseError(initStatus.workflowUUID, resRow.status, resRow.name);
|
|
@@ -904,6 +1248,7 @@ class SystemDatabase {
|
|
|
904
1248
|
async deadLetterWorkflows(workflowIDs, minRecoveryAttempts) {
|
|
905
1249
|
if (workflowIDs.length === 0)
|
|
906
1250
|
return;
|
|
1251
|
+
const params = [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflow_1.StatusString.PENDING, minRecoveryAttempts];
|
|
907
1252
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
908
1253
|
SET status = $1,
|
|
909
1254
|
owner_xid = NULL,
|
|
@@ -912,17 +1257,18 @@ class SystemDatabase {
|
|
|
912
1257
|
queue_name = NULL,
|
|
913
1258
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
914
1259
|
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
915
|
-
WHERE
|
|
1260
|
+
WHERE ${this.wfIn(undefined, workflowIDs, params)} AND status = $2 AND recovery_attempts >= $3`, params);
|
|
916
1261
|
}
|
|
917
1262
|
/** Highest created_at among still-active rows per partition key, used to seed the in-memory cursor. */
|
|
918
1263
|
async #maxPartitionKeyCreatedAt(keys) {
|
|
919
1264
|
const maxima = new Map();
|
|
920
1265
|
if (keys.length === 0)
|
|
921
1266
|
return maxima;
|
|
1267
|
+
const params = [keys, [workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING]];
|
|
922
1268
|
const { rows } = await this.pool.query(`SELECT queue_partition_key, MAX(created_at) AS max_created_at
|
|
923
1269
|
FROM "${this.schemaName}".workflow_status
|
|
924
|
-
WHERE queue_partition_key = ANY($1) AND status = ANY($2)
|
|
925
|
-
GROUP BY queue_partition_key`,
|
|
1270
|
+
WHERE ${this.scoped(undefined, params)} AND queue_partition_key = ANY($1) AND status = ANY($2)
|
|
1271
|
+
GROUP BY queue_partition_key`, params);
|
|
926
1272
|
for (const row of rows) {
|
|
927
1273
|
if (row.max_created_at !== null) {
|
|
928
1274
|
maxima.set(row.queue_partition_key, Number(row.max_created_at));
|
|
@@ -1028,39 +1374,60 @@ class SystemDatabase {
|
|
|
1028
1374
|
const chunkSize = 500;
|
|
1029
1375
|
for (let start = 0; start < statuses.length; start += chunkSize) {
|
|
1030
1376
|
const chunk = statuses.slice(start, start + chunkSize);
|
|
1031
|
-
const
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1377
|
+
const keyed = chunk.map((status) => ({
|
|
1378
|
+
...this.wfValues(status.workflowUUID, status.shardKey),
|
|
1379
|
+
...this.parentValues(status.parentWorkflowID),
|
|
1380
|
+
}));
|
|
1381
|
+
const rows = chunk.map((status, i) => {
|
|
1036
1382
|
const createdAt = createdAts[start + i];
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1383
|
+
return [
|
|
1384
|
+
...Object.values(keyed[i]),
|
|
1385
|
+
status.workflowUUID,
|
|
1386
|
+
status.status,
|
|
1387
|
+
status.workflowName,
|
|
1388
|
+
// For cross-language compatibility, these MUST be NULL in the database when not set
|
|
1389
|
+
status.workflowClassName === '' ? null : status.workflowClassName,
|
|
1390
|
+
status.workflowConfigName === '' ? null : status.workflowConfigName,
|
|
1391
|
+
status.queueName ?? null,
|
|
1392
|
+
status.authenticatedUser,
|
|
1393
|
+
status.assumedRole,
|
|
1394
|
+
JSON.stringify(status.authenticatedRoles),
|
|
1395
|
+
status.executorId,
|
|
1396
|
+
status.applicationVersion ?? null,
|
|
1397
|
+
status.applicationID,
|
|
1398
|
+
createdAt,
|
|
1399
|
+
0,
|
|
1400
|
+
createdAt,
|
|
1401
|
+
status.timeoutMS ?? null,
|
|
1402
|
+
status.deadlineEpochMS ?? null,
|
|
1403
|
+
// Legacy column: the payload lives in workflow_input.
|
|
1404
|
+
null,
|
|
1405
|
+
null,
|
|
1406
|
+
status.priority,
|
|
1407
|
+
status.queuePartitionKey ?? null,
|
|
1408
|
+
status.parentWorkflowID ?? null,
|
|
1409
|
+
status.serialization,
|
|
1410
|
+
null,
|
|
1411
|
+
status.delayUntilEpochMS ?? null,
|
|
1412
|
+
status.attributes ? JSON.stringify(status.attributes) : null,
|
|
1413
|
+
status.scheduleName ?? null,
|
|
1414
|
+
status.applicationName ?? null,
|
|
1415
|
+
];
|
|
1416
|
+
});
|
|
1417
|
+
const returned = await this.insertRows(client, 'workflow_status', [...Object.keys(keyed[0]), ...columns], rows, `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING RETURNING workflow_uuid`);
|
|
1418
|
+
for (const row of returned) {
|
|
1049
1419
|
inserted.add(row.workflow_uuid);
|
|
1050
1420
|
}
|
|
1051
1421
|
}
|
|
1052
1422
|
for (let start = 0; start < statuses.length; start += chunkSize) {
|
|
1053
1423
|
const chunk = statuses.slice(start, start + chunkSize);
|
|
1054
|
-
const
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
}
|
|
1061
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
|
|
1062
|
-
VALUES ${tuples.join(', ')}
|
|
1063
|
-
ON CONFLICT (workflow_uuid) DO NOTHING`, params);
|
|
1424
|
+
const keyed = chunk.map((status) => this.wfValues(status.workflowUUID, status.shardKey));
|
|
1425
|
+
await this.insertRows(client, 'workflow_input', [...Object.keys(keyed[0]), 'workflow_uuid', 'inputs', 'retention_timestamp'], chunk.map((status, i) => [
|
|
1426
|
+
...Object.values(keyed[i]),
|
|
1427
|
+
status.workflowUUID,
|
|
1428
|
+
status.input,
|
|
1429
|
+
createdAts[start + i],
|
|
1430
|
+
]), `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`);
|
|
1064
1431
|
}
|
|
1065
1432
|
await client.query('COMMIT');
|
|
1066
1433
|
}
|
|
@@ -1074,22 +1441,10 @@ class SystemDatabase {
|
|
|
1074
1441
|
return inserted;
|
|
1075
1442
|
}
|
|
1076
1443
|
async recordWorkflowOutput(workflowID, status, ownerXid) {
|
|
1077
|
-
|
|
1078
|
-
try {
|
|
1079
|
-
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
|
|
1080
|
-
}
|
|
1081
|
-
finally {
|
|
1082
|
-
client.release();
|
|
1083
|
-
}
|
|
1444
|
+
return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
|
|
1084
1445
|
}
|
|
1085
1446
|
async recordWorkflowError(workflowID, status, ownerXid) {
|
|
1086
|
-
|
|
1087
|
-
try {
|
|
1088
|
-
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
|
|
1089
|
-
}
|
|
1090
|
-
finally {
|
|
1091
|
-
client.release();
|
|
1092
|
-
}
|
|
1447
|
+
return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
|
|
1093
1448
|
}
|
|
1094
1449
|
// Record a workflow's terminal outcome (SUCCESS or ERROR), reporting whether
|
|
1095
1450
|
// the write landed. The write applies only to a PENDING row still owned by
|
|
@@ -1100,8 +1455,49 @@ class SystemDatabase {
|
|
|
1100
1455
|
// terminal, handed to another execution (e.g. by a concurrent resume or
|
|
1101
1456
|
// recovery), or deleted; the caller resolves which by awaiting the recorded
|
|
1102
1457
|
// outcome.
|
|
1103
|
-
async #recordWorkflowOutcome(
|
|
1458
|
+
async #recordWorkflowOutcome(workflowID, status, outcome, ownerXid) {
|
|
1104
1459
|
ownerXid ??= (0, context_1.currentOwnerXid)(workflowID);
|
|
1460
|
+
if (!(await this.outcomeInOneStatement())) {
|
|
1461
|
+
return await this.#recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid);
|
|
1462
|
+
}
|
|
1463
|
+
// One autocommit round trip; the upsert reads the UPDATE's rows, so the output lands only with the status.
|
|
1464
|
+
const params = [status, workflow_1.StatusString.PENDING, outcome.output ?? null, outcome.error ?? null];
|
|
1465
|
+
const target = this.wf(undefined, workflowID, params);
|
|
1466
|
+
const owned = ownerXid !== undefined ? `AND owner_xid = $${params.push(ownerXid)}` : '';
|
|
1467
|
+
const keys = this.wfValues(workflowID);
|
|
1468
|
+
const keyColumns = Object.keys(keys);
|
|
1469
|
+
const keyExprs = Object.values(keys).map((value) => `$${params.push(value)}::text`);
|
|
1470
|
+
const workflowParam = `$${params.push(workflowID)}::text`;
|
|
1471
|
+
const { rows } = await this.pool.query(`WITH updated AS (
|
|
1472
|
+
UPDATE "${this.schemaName}".workflow_status
|
|
1473
|
+
SET status = $1,
|
|
1474
|
+
-- As the workflow is complete, remove its deduplication ID; a finished workflow has no owner.
|
|
1475
|
+
deduplication_id = NULL,
|
|
1476
|
+
owner_xid = NULL,
|
|
1477
|
+
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1478
|
+
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1479
|
+
WHERE ${target} AND status = $2 ${owned}
|
|
1480
|
+
RETURNING 1
|
|
1481
|
+
), stored AS (
|
|
1482
|
+
INSERT INTO "${this.schemaName}".workflow_output (${[...keyColumns, 'workflow_uuid', 'output', 'error'].join(', ')})
|
|
1483
|
+
SELECT ${[...keyExprs, workflowParam, '$3::text', '$4::text'].join(', ')}
|
|
1484
|
+
WHERE EXISTS (SELECT 1 FROM updated)
|
|
1485
|
+
-- EXCLUDED is the row this INSERT proposed, so these are the values above.
|
|
1486
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error
|
|
1487
|
+
-- CockroachDB refuses a data-modifying CTE without RETURNING.
|
|
1488
|
+
RETURNING 1
|
|
1489
|
+
)
|
|
1490
|
+
-- Selecting from the CTEs rather than inserting at the top level lets a router see the shard key.
|
|
1491
|
+
SELECT count(*) AS count FROM updated`, params);
|
|
1492
|
+
return Number(rows[0]?.count ?? 0) > 0;
|
|
1493
|
+
}
|
|
1494
|
+
/** Whether a workflow's outcome is written in one statement, rather than a transaction of two. */
|
|
1495
|
+
outcomeInOneStatement() {
|
|
1496
|
+
return Promise.resolve(true);
|
|
1497
|
+
}
|
|
1498
|
+
/** The outcome write as a transaction of two statements, for a backend that cannot run it as one. */
|
|
1499
|
+
async #recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid) {
|
|
1500
|
+
const client = await this.connect();
|
|
1105
1501
|
let committed = false;
|
|
1106
1502
|
try {
|
|
1107
1503
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
@@ -1115,19 +1511,31 @@ class SystemDatabase {
|
|
|
1115
1511
|
// The outcome was not ours to write, so leave no orphan payload.
|
|
1116
1512
|
return false;
|
|
1117
1513
|
}
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1514
|
+
const values = {
|
|
1515
|
+
...this.wfValues(workflowID),
|
|
1516
|
+
workflow_uuid: workflowID,
|
|
1517
|
+
output: outcome.output ?? null,
|
|
1518
|
+
error: outcome.error ?? null,
|
|
1519
|
+
};
|
|
1520
|
+
const columns = Object.keys(values);
|
|
1521
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_output (${columns.join(', ')})
|
|
1522
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
1523
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error`, Object.values(values));
|
|
1121
1524
|
committed = true;
|
|
1122
1525
|
return true;
|
|
1123
1526
|
}
|
|
1124
1527
|
finally {
|
|
1125
|
-
|
|
1126
|
-
|
|
1528
|
+
try {
|
|
1529
|
+
if (committed) {
|
|
1530
|
+
await client.query('COMMIT');
|
|
1531
|
+
}
|
|
1532
|
+
else {
|
|
1533
|
+
// Swallowed: a rollback failure must not mask why we are rolling back.
|
|
1534
|
+
await client.query('ROLLBACK').catch(() => undefined);
|
|
1535
|
+
}
|
|
1127
1536
|
}
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
await client.query('ROLLBACK').catch(() => undefined);
|
|
1537
|
+
finally {
|
|
1538
|
+
client.release();
|
|
1131
1539
|
}
|
|
1132
1540
|
}
|
|
1133
1541
|
}
|
|
@@ -1136,6 +1544,7 @@ class SystemDatabase {
|
|
|
1136
1544
|
const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
|
|
1137
1545
|
// executor_id defaults to "local", so it collides across applications.
|
|
1138
1546
|
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
1547
|
+
const keyColumn = this.keyColumn(undefined);
|
|
1139
1548
|
const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1140
1549
|
SET started_at_epoch_ms = NULL,
|
|
1141
1550
|
status = $1,
|
|
@@ -1146,25 +1555,31 @@ class SystemDatabase {
|
|
|
1146
1555
|
AND executor_id = $4
|
|
1147
1556
|
AND application_version = $5
|
|
1148
1557
|
AND ${scope}
|
|
1149
|
-
RETURNING workflow_uuid`, params);
|
|
1150
|
-
return result.rows.map((row) => row.workflow_uuid);
|
|
1558
|
+
RETURNING workflow_uuid${keyColumn !== undefined ? `, ${keyColumn} AS shard_key` : ''}`, params);
|
|
1559
|
+
return result.rows.map((row) => ({ workflowID: row.workflow_uuid, shardKey: row.shard_key ?? undefined }));
|
|
1151
1560
|
}
|
|
1152
1561
|
async getWorkflowStatus(workflowID, callerID, callerFN) {
|
|
1153
|
-
|
|
1154
|
-
|
|
1562
|
+
// Resolved before binding the caller's own key, which a same-ID target would share.
|
|
1563
|
+
const shardKey = this.keyHint(workflowID);
|
|
1564
|
+
// Inside a workflow this runs on the transaction's client: taking a second pool connection while
|
|
1565
|
+
// holding the first deadlocks the pool once every connection is held by a waiting caller.
|
|
1566
|
+
const funcGetStatus = async (client) => {
|
|
1567
|
+
const statuses = await this.#listWorkflows({ workflowIDs: [workflowID], shardKey }, {}, client);
|
|
1155
1568
|
const status = statuses.find((s) => s.workflowUUID === workflowID);
|
|
1156
1569
|
return status ? JSON.stringify(status) : null;
|
|
1157
1570
|
};
|
|
1158
|
-
if (callerID && callerFN) {
|
|
1159
|
-
const
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1571
|
+
if (callerID && callerFN !== undefined) {
|
|
1572
|
+
const json = await this.ownKey(callerID, undefined, async () => {
|
|
1573
|
+
const client = await this.connect();
|
|
1574
|
+
try {
|
|
1575
|
+
// Check if the operation has been done before for OAOO (only do this inside a workflow).
|
|
1576
|
+
return await this.inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, () => funcGetStatus(client)));
|
|
1577
|
+
}
|
|
1578
|
+
finally {
|
|
1579
|
+
client.release();
|
|
1580
|
+
}
|
|
1581
|
+
});
|
|
1582
|
+
return parseStatus(json);
|
|
1168
1583
|
}
|
|
1169
1584
|
else {
|
|
1170
1585
|
const json = await funcGetStatus();
|
|
@@ -1178,7 +1593,7 @@ class SystemDatabase {
|
|
|
1178
1593
|
statusFetchChunkSize = 500;
|
|
1179
1594
|
// Retried per chunk so a reconnect refetches one chunk, not every chunk before it.
|
|
1180
1595
|
async fetchWorkflowStatusChunk(workflowIDs) {
|
|
1181
|
-
return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false });
|
|
1596
|
+
return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false }, { keyedIDs: true });
|
|
1182
1597
|
}
|
|
1183
1598
|
/** Fetch many statuses in as few round trips as possible. IDs with no row are omitted. */
|
|
1184
1599
|
async getWorkflowStatuses(workflowIDs) {
|
|
@@ -1209,23 +1624,26 @@ class SystemDatabase {
|
|
|
1209
1624
|
}
|
|
1210
1625
|
}
|
|
1211
1626
|
// ==================== Step Results ====================
|
|
1212
|
-
async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1627
|
+
async getOperationResultAndThrowIfCancelled(workflowID, functionID, ownerKey) {
|
|
1628
|
+
return await this.ownKey(workflowID, ownerKey, async () => {
|
|
1629
|
+
const client = await this.connect();
|
|
1630
|
+
try {
|
|
1631
|
+
return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
1632
|
+
}
|
|
1633
|
+
finally {
|
|
1634
|
+
client.release();
|
|
1635
|
+
}
|
|
1636
|
+
});
|
|
1220
1637
|
}
|
|
1221
1638
|
async getAllOperationResults(workflowID, limit, offset, loadOutput = true) {
|
|
1222
1639
|
// Skip the output/error payload columns when the caller does not need them,
|
|
1223
1640
|
// mirroring how listWorkflows skips its payload joins when loadOutput is false.
|
|
1641
|
+
const childKey = this.keyColumn(undefined, 'child_shard_key');
|
|
1224
1642
|
const columns = loadOutput
|
|
1225
1643
|
? '*'
|
|
1226
|
-
:
|
|
1227
|
-
|
|
1228
|
-
|
|
1644
|
+
: `workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? `, ${childKey}` : ''}`;
|
|
1645
|
+
const params = [];
|
|
1646
|
+
let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE ${this.wf(undefined, workflowID, params)} ORDER BY function_id`;
|
|
1229
1647
|
if (limit !== undefined) {
|
|
1230
1648
|
params.push(limit);
|
|
1231
1649
|
query += ` LIMIT $${params.length}`;
|
|
@@ -1238,14 +1656,18 @@ class SystemDatabase {
|
|
|
1238
1656
|
return rows;
|
|
1239
1657
|
}
|
|
1240
1658
|
async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1659
|
+
// Resolved before binding the caller's own key, which a same-ID child would share.
|
|
1660
|
+
const childValues = this.childValues(options.childWorkflowID);
|
|
1661
|
+
await this.ownKey(workflowID, options.ownerKey, async () => {
|
|
1662
|
+
const client = await this.connect();
|
|
1663
|
+
try {
|
|
1664
|
+
await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, { ...options, childValues }));
|
|
1665
|
+
}
|
|
1666
|
+
finally {
|
|
1667
|
+
client.release();
|
|
1668
|
+
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
|
|
1669
|
+
}
|
|
1670
|
+
});
|
|
1249
1671
|
}
|
|
1250
1672
|
/** Checkpoint an operation's failure so a replay re-throws it instead of re-running. */
|
|
1251
1673
|
async recordOperationError(workflowID, functionID, functionName, startTimeEpochMs, error, childWorkflowID) {
|
|
@@ -1319,9 +1741,10 @@ class SystemDatabase {
|
|
|
1319
1741
|
throw new TypeError('functionID must be defined');
|
|
1320
1742
|
patchName = `DBOS.patch-${patchName}`;
|
|
1321
1743
|
const readCheckpointName = async (queryable) => {
|
|
1744
|
+
const params = [functionID];
|
|
1322
1745
|
const { rows } = await queryable.query(`SELECT function_name
|
|
1323
1746
|
FROM "${this.schemaName}".operation_outputs
|
|
1324
|
-
WHERE
|
|
1747
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
|
|
1325
1748
|
return rows.length === 0 ? undefined : rows[0].function_name;
|
|
1326
1749
|
};
|
|
1327
1750
|
if (deprecated) {
|
|
@@ -1336,7 +1759,7 @@ class SystemDatabase {
|
|
|
1336
1759
|
// If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
|
|
1337
1760
|
const client = await this.connect();
|
|
1338
1761
|
try {
|
|
1339
|
-
return await this
|
|
1762
|
+
return await this.inTransaction(client, async () => {
|
|
1340
1763
|
const checkpointName = await readCheckpointName(client);
|
|
1341
1764
|
if (checkpointName !== undefined) {
|
|
1342
1765
|
const matches = checkpointName === patchName;
|
|
@@ -1362,7 +1785,7 @@ class SystemDatabase {
|
|
|
1362
1785
|
let frontier = workflowIDs;
|
|
1363
1786
|
while (frontier.length > 0) {
|
|
1364
1787
|
await this.#cancelWorkflows(frontier);
|
|
1365
|
-
const children = await this
|
|
1788
|
+
const children = await this.getDirectChildren(frontier);
|
|
1366
1789
|
frontier = children.filter((id) => !visited.has(id));
|
|
1367
1790
|
for (const id of frontier) {
|
|
1368
1791
|
visited.add(id);
|
|
@@ -1370,41 +1793,44 @@ class SystemDatabase {
|
|
|
1370
1793
|
}
|
|
1371
1794
|
}
|
|
1372
1795
|
async #cancelWorkflows(workflowIDs) {
|
|
1796
|
+
const params = [workflow_1.StatusString.CANCELLED, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR];
|
|
1373
1797
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1374
1798
|
SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
1375
1799
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1376
1800
|
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1377
|
-
WHERE
|
|
1378
|
-
AND status NOT IN ($
|
|
1801
|
+
WHERE ${this.wfIn(undefined, workflowIDs, params)}
|
|
1802
|
+
AND status NOT IN ($2, $3)`, params);
|
|
1379
1803
|
}
|
|
1380
1804
|
async checkIfCanceled(workflowID) {
|
|
1381
|
-
|
|
1805
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
1806
|
+
await this.ownKey(workflowID, undefined, () => this.#checkIfCanceled(this.pool, workflowID));
|
|
1382
1807
|
}
|
|
1383
1808
|
async resumeWorkflows(workflowIDs, queueName) {
|
|
1384
1809
|
const client = await this.connect();
|
|
1385
1810
|
try {
|
|
1386
1811
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1387
1812
|
// Check existence separately: a zero-row update also means "already complete", a legal no-op.
|
|
1388
|
-
const
|
|
1813
|
+
const existingParams = [];
|
|
1814
|
+
const { rows } = await client.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${this.wfIn(undefined, workflowIDs, existingParams)}`, existingParams);
|
|
1389
1815
|
const existing = new Set(rows.map((r) => r.workflow_uuid));
|
|
1390
1816
|
const missing = workflowIDs.filter((id) => !existing.has(id));
|
|
1391
1817
|
if (missing.length > 0) {
|
|
1392
1818
|
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${missing.join(', ')} does not exist`);
|
|
1393
1819
|
}
|
|
1820
|
+
const updateParams = [
|
|
1821
|
+
workflow_1.StatusString.ENQUEUED,
|
|
1822
|
+
queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1823
|
+
workflow_1.StatusString.SUCCESS,
|
|
1824
|
+
workflow_1.StatusString.ERROR,
|
|
1825
|
+
];
|
|
1394
1826
|
await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1395
1827
|
SET status = $1, owner_xid = NULL, queue_name = $2, recovery_attempts = 0,
|
|
1396
1828
|
workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
|
|
1397
1829
|
started_at_epoch_ms = NULL,
|
|
1398
1830
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1399
1831
|
completed_at = NULL
|
|
1400
|
-
WHERE
|
|
1401
|
-
AND status NOT IN ($
|
|
1402
|
-
workflow_1.StatusString.ENQUEUED,
|
|
1403
|
-
queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1404
|
-
workflowIDs,
|
|
1405
|
-
workflow_1.StatusString.SUCCESS,
|
|
1406
|
-
workflow_1.StatusString.ERROR,
|
|
1407
|
-
]);
|
|
1832
|
+
WHERE ${this.wfIn(undefined, workflowIDs, updateParams)}
|
|
1833
|
+
AND status NOT IN ($3, $4)`, updateParams);
|
|
1408
1834
|
await client.query('COMMIT');
|
|
1409
1835
|
}
|
|
1410
1836
|
catch (e) {
|
|
@@ -1417,16 +1843,18 @@ class SystemDatabase {
|
|
|
1417
1843
|
}
|
|
1418
1844
|
}
|
|
1419
1845
|
async setWorkflowPriority(workflowID, priority) {
|
|
1846
|
+
const params = [priority, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED];
|
|
1420
1847
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1421
1848
|
SET priority = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1422
|
-
WHERE
|
|
1423
|
-
AND status IN ($
|
|
1849
|
+
WHERE ${this.wf(undefined, workflowID, params)}
|
|
1850
|
+
AND status IN ($2, $3)`, params);
|
|
1424
1851
|
}
|
|
1425
1852
|
async setWorkflowDelay(workflowID, delayUntilEpochMS) {
|
|
1853
|
+
const params = [delayUntilEpochMS, workflow_1.StatusString.DELAYED];
|
|
1426
1854
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1427
1855
|
SET delay_until_epoch_ms = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1428
|
-
WHERE
|
|
1429
|
-
AND status = $
|
|
1856
|
+
WHERE ${this.wf(undefined, workflowID, params)}
|
|
1857
|
+
AND status = $2`, params);
|
|
1430
1858
|
}
|
|
1431
1859
|
/**
|
|
1432
1860
|
* Extend an existing debounced DELAYED workflow's delay and update its inputs, atomically.
|
|
@@ -1492,7 +1920,7 @@ class SystemDatabase {
|
|
|
1492
1920
|
if (updated.rows.length > 0) {
|
|
1493
1921
|
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
|
|
1494
1922
|
VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
|
|
1495
|
-
ON CONFLICT (workflow_uuid) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
|
|
1923
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
|
|
1496
1924
|
return {
|
|
1497
1925
|
bouncedWorkflowID: updated.rows[0].workflow_uuid,
|
|
1498
1926
|
holderWorkflowID: null,
|
|
@@ -1502,10 +1930,11 @@ class SystemDatabase {
|
|
|
1502
1930
|
holderApplicationName: null,
|
|
1503
1931
|
};
|
|
1504
1932
|
}
|
|
1505
|
-
//
|
|
1933
|
+
// No application filter, so a holder that blocked the update above is reportable.
|
|
1934
|
+
const holderParams = [params.queueName, params.deduplicationID];
|
|
1506
1935
|
const holder = await client.query(`SELECT workflow_uuid, is_debounced, name, class_name, application_name
|
|
1507
1936
|
FROM "${this.schemaName}".workflow_status
|
|
1508
|
-
WHERE queue_name = $1 AND deduplication_id = $2`,
|
|
1937
|
+
WHERE ${this.scoped(undefined, holderParams)} AND queue_name = $1 AND deduplication_id = $2`, holderParams);
|
|
1509
1938
|
if (holder.rows.length === 0) {
|
|
1510
1939
|
return {
|
|
1511
1940
|
bouncedWorkflowID: null,
|
|
@@ -1526,7 +1955,7 @@ class SystemDatabase {
|
|
|
1526
1955
|
};
|
|
1527
1956
|
}
|
|
1528
1957
|
// Get the immediate (one-level) child workflow IDs for a set of workflows.
|
|
1529
|
-
async
|
|
1958
|
+
async getDirectChildren(workflowIDs) {
|
|
1530
1959
|
if (workflowIDs.length === 0) {
|
|
1531
1960
|
return [];
|
|
1532
1961
|
}
|
|
@@ -1540,7 +1969,7 @@ class SystemDatabase {
|
|
|
1540
1969
|
const descendants = new Set();
|
|
1541
1970
|
let frontier = [workflowID];
|
|
1542
1971
|
while (frontier.length > 0) {
|
|
1543
|
-
const children = await this
|
|
1972
|
+
const children = await this.getDirectChildren(frontier);
|
|
1544
1973
|
frontier = children.filter((id) => !descendants.has(id));
|
|
1545
1974
|
for (const id of frontier) {
|
|
1546
1975
|
descendants.add(id);
|
|
@@ -1548,6 +1977,7 @@ class SystemDatabase {
|
|
|
1548
1977
|
}
|
|
1549
1978
|
return [...descendants];
|
|
1550
1979
|
}
|
|
1980
|
+
/** Delete workflows and all associated data, and with deleteChildren every workflow they started, recursively. */
|
|
1551
1981
|
async deleteWorkflows(workflowIDs, deleteChildren = false) {
|
|
1552
1982
|
const allIds = [...workflowIDs];
|
|
1553
1983
|
if (deleteChildren) {
|
|
@@ -1555,15 +1985,18 @@ class SystemDatabase {
|
|
|
1555
1985
|
allIds.push(...(await this.getWorkflowChildren(wfid)));
|
|
1556
1986
|
}
|
|
1557
1987
|
}
|
|
1988
|
+
await this.deleteWorkflowsInternal(allIds);
|
|
1989
|
+
}
|
|
1990
|
+
async deleteWorkflowsInternal(allIds) {
|
|
1558
1991
|
const client = await this.connect();
|
|
1559
1992
|
try {
|
|
1560
1993
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1561
1994
|
// The payload tables carry no foreign key, so the status delete does not cascade
|
|
1562
1995
|
// into them; without this they survive as orphans.
|
|
1563
|
-
for (const table of ['workflow_input', 'workflow_output', 'operation_outputs']) {
|
|
1564
|
-
|
|
1996
|
+
for (const table of ['workflow_input', 'workflow_output', 'operation_outputs', 'workflow_status']) {
|
|
1997
|
+
const params = [];
|
|
1998
|
+
await client.query(`DELETE FROM "${this.schemaName}".${table} WHERE ${this.wfIn(undefined, allIds, params)}`, params);
|
|
1565
1999
|
}
|
|
1566
|
-
await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [allIds]);
|
|
1567
2000
|
await client.query('COMMIT');
|
|
1568
2001
|
}
|
|
1569
2002
|
catch (e) {
|
|
@@ -1604,7 +2037,8 @@ class SystemDatabase {
|
|
|
1604
2037
|
const client = await this.connect();
|
|
1605
2038
|
try {
|
|
1606
2039
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1607
|
-
const
|
|
2040
|
+
const statusParams = [];
|
|
2041
|
+
const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE ${this.wf(undefined, workflowID, statusParams)}`, statusParams);
|
|
1608
2042
|
if (statusRows.length === 0) {
|
|
1609
2043
|
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
|
|
1610
2044
|
}
|
|
@@ -1615,56 +2049,61 @@ class SystemDatabase {
|
|
|
1615
2049
|
// Roll workflow_events back to the last value published before the cut, using
|
|
1616
2050
|
// workflow_events_history as an undo log, then rewind that history too.
|
|
1617
2051
|
// `publishedPastCut` is the correlated EXISTS both halves key off.
|
|
1618
|
-
const publishedPastCut = `EXISTS (
|
|
2052
|
+
const publishedPastCut = (key, params) => `EXISTS (
|
|
1619
2053
|
SELECT 1 FROM "${schema}".workflow_events_history discarded
|
|
1620
|
-
WHERE discarded
|
|
2054
|
+
WHERE ${this.wf('discarded', workflowID, params)} AND discarded.key = ${key} AND discarded.function_id >= $1
|
|
1621
2055
|
)`;
|
|
1622
2056
|
// First unpublish everything the discarded run touched.
|
|
2057
|
+
const unpublishParams = [startStep];
|
|
1623
2058
|
await client.query(`DELETE FROM "${schema}".workflow_events
|
|
1624
|
-
WHERE
|
|
2059
|
+
WHERE ${this.wf(undefined, workflowID, unpublishParams)}
|
|
2060
|
+
AND ${publishedPastCut('"' + schema + '".workflow_events.key', unpublishParams)}`, unpublishParams);
|
|
1625
2061
|
// Then restore those keys, if any, to the value published below the cut.
|
|
1626
|
-
|
|
1627
|
-
|
|
2062
|
+
const restoreParams = [startStep];
|
|
2063
|
+
const keyColumns = this.wfColumns(workflowID, restoreParams);
|
|
2064
|
+
await client.query(`INSERT INTO "${schema}".workflow_events (${[...keyColumns.map(([c]) => c), 'workflow_uuid', 'key', 'value', 'serialization'].join(', ')})
|
|
2065
|
+
SELECT ${[...keyColumns.map(([, e]) => e), 'surviving.workflow_uuid', 'surviving.key', 'surviving.value', 'surviving.serialization'].join(', ')}
|
|
1628
2066
|
FROM (
|
|
1629
2067
|
SELECT weh.workflow_uuid, weh.key, weh.value, weh.serialization,
|
|
1630
2068
|
ROW_NUMBER() OVER (PARTITION BY weh.key ORDER BY weh.function_id DESC) AS rn
|
|
1631
2069
|
FROM "${schema}".workflow_events_history weh
|
|
1632
|
-
WHERE weh
|
|
1633
|
-
AND ${publishedPastCut
|
|
2070
|
+
WHERE ${this.wf('weh', workflowID, restoreParams)} AND weh.function_id < $1
|
|
2071
|
+
AND ${publishedPastCut('weh.key', restoreParams)}
|
|
1634
2072
|
) surviving
|
|
1635
|
-
WHERE surviving.rn = 1`,
|
|
2073
|
+
WHERE surviving.rn = 1`, restoreParams);
|
|
1636
2074
|
// Clear the streams' close sentinels so the replay can append again.
|
|
2075
|
+
const sentinelParams = [startStep, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name()];
|
|
1637
2076
|
await client.query(`DELETE FROM "${schema}".streams
|
|
1638
|
-
WHERE
|
|
2077
|
+
WHERE ${this.wf(undefined, workflowID, sentinelParams)} AND function_id >= $1 AND value = $2 AND serialization = $3`, sentinelParams);
|
|
1639
2078
|
// Discard the steps and the events history past the cut.
|
|
1640
2079
|
for (const table of ['operation_outputs', 'workflow_events_history']) {
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
startStep,
|
|
1644
|
-
]);
|
|
2080
|
+
const params = [startStep];
|
|
2081
|
+
await client.query(`DELETE FROM "${schema}".${table} WHERE ${this.wf(undefined, workflowID, params)} AND function_id >= $1`, params);
|
|
1645
2082
|
}
|
|
1646
2083
|
// Delete messages consumed or received after the rewind point
|
|
1647
|
-
|
|
1648
|
-
await client.query(`DELETE FROM "${schema}".
|
|
2084
|
+
const messageParams = [startStep];
|
|
2085
|
+
await client.query(`DELETE FROM "${schema}".notifications
|
|
2086
|
+
WHERE ${this.wf(undefined, workflowID, messageParams, 'destination_uuid')}
|
|
2087
|
+
AND (consumed_by_function_id >= $1 OR consumed = false)`, messageParams);
|
|
2088
|
+
const outputParams = [];
|
|
2089
|
+
await client.query(`DELETE FROM "${schema}".workflow_output WHERE ${this.wf(undefined, workflowID, outputParams)}`, outputParams);
|
|
1649
2090
|
// Re-enqueue the workflow. Re-asserting the status we read keeps a workflow that
|
|
1650
2091
|
// moved on underneath us from being resurrected.
|
|
1651
|
-
const setVersion = options.applicationVersion !== undefined ? ', application_version = $6' : '';
|
|
1652
2092
|
const params = [
|
|
1653
2093
|
workflow_1.StatusString.ENQUEUED,
|
|
1654
2094
|
options.queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1655
2095
|
options.queuePartitionKey ?? null,
|
|
1656
|
-
workflowID,
|
|
1657
2096
|
status,
|
|
1658
2097
|
];
|
|
1659
|
-
|
|
1660
|
-
params.push(options.applicationVersion)
|
|
1661
|
-
|
|
2098
|
+
const setVersion = options.applicationVersion !== undefined
|
|
2099
|
+
? `, application_version = $${params.push(options.applicationVersion)}`
|
|
2100
|
+
: '';
|
|
1662
2101
|
const { rowCount } = await client.query(`UPDATE "${schema}".workflow_status
|
|
1663
2102
|
SET status = $1, owner_xid = NULL, queue_name = $2, queue_partition_key = $3, recovery_attempts = 0,
|
|
1664
2103
|
workflow_deadline_epoch_ms = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
1665
2104
|
completed_at = NULL, output = NULL, error = NULL,
|
|
1666
2105
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint${setVersion}
|
|
1667
|
-
WHERE
|
|
2106
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND status = $4`, params);
|
|
1668
2107
|
if (rowCount !== 1) {
|
|
1669
2108
|
throw new error_1.DBOSError(`Workflow ${workflowID} changed status while being rewound; retry the rewind`);
|
|
1670
2109
|
}
|
|
@@ -1694,13 +2133,14 @@ class SystemDatabase {
|
|
|
1694
2133
|
try {
|
|
1695
2134
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1696
2135
|
// Fetch the status of all original workflows inside the transaction.
|
|
2136
|
+
const statusParams = [];
|
|
1697
2137
|
const { rows: statusRows } = await client.query(`SELECT ws.workflow_uuid, ws.name, ws.class_name, ws.config_name, ws.application_id,
|
|
1698
2138
|
ws.authenticated_user, ws.authenticated_roles, ws.assumed_role,
|
|
1699
2139
|
COALESCE(wi.inputs, ws.inputs) AS inputs, ws.serialization,
|
|
1700
2140
|
ws.application_version, ws.attributes, ws.application_name
|
|
1701
2141
|
FROM "${this.schemaName}".workflow_status ws
|
|
1702
|
-
LEFT JOIN "${this.schemaName}".workflow_input wi ON
|
|
1703
|
-
WHERE ws
|
|
2142
|
+
LEFT JOIN "${this.schemaName}".workflow_input wi ON ${this.wfJoin('ws', 'wi')}
|
|
2143
|
+
WHERE ${this.wfIn('ws', originalWorkflowIDs, statusParams)}`, statusParams);
|
|
1704
2144
|
const statusByID = new Map(statusRows.map((r) => [r.workflow_uuid, r]));
|
|
1705
2145
|
for (const wid of originalWorkflowIDs) {
|
|
1706
2146
|
if (!statusByID.has(wid)) {
|
|
@@ -1736,35 +2176,42 @@ class SystemDatabase {
|
|
|
1736
2176
|
forkedWorkflowIDs[i],
|
|
1737
2177
|
statusByID.get(origID).application_name ?? this.appName ?? null,
|
|
1738
2178
|
]));
|
|
1739
|
-
const
|
|
1740
|
-
|
|
1741
|
-
let paramIdx = 1;
|
|
1742
|
-
for (let i = 0; i < originalWorkflowIDs.length; i++) {
|
|
1743
|
-
const origID = originalWorkflowIDs[i];
|
|
2179
|
+
const forkKeys = forkedWorkflowIDs.map((forkID) => this.wfValues(forkID));
|
|
2180
|
+
await this.insertRows(client, 'workflow_status', [...Object.keys(forkKeys[0]), ...insertCols], originalWorkflowIDs.map((origID, i) => {
|
|
1744
2181
|
const forkID = forkedWorkflowIDs[i];
|
|
1745
2182
|
const ws = statusByID.get(origID);
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
2183
|
+
return [
|
|
2184
|
+
...Object.values(forkKeys[i]),
|
|
2185
|
+
forkID,
|
|
2186
|
+
workflow_1.StatusString.ENQUEUED,
|
|
2187
|
+
ws.name,
|
|
2188
|
+
ws.class_name ?? null,
|
|
2189
|
+
ws.config_name ?? null,
|
|
2190
|
+
queueName,
|
|
2191
|
+
ws.authenticated_user,
|
|
2192
|
+
ws.assumed_role,
|
|
2193
|
+
ws.authenticated_roles,
|
|
2194
|
+
options.applicationVersion ?? ws.application_version ?? null,
|
|
2195
|
+
ws.application_id,
|
|
2196
|
+
// Legacy column: the payload lives in workflow_input.
|
|
2197
|
+
null,
|
|
2198
|
+
options.queuePartitionKey ?? null,
|
|
2199
|
+
origID,
|
|
2200
|
+
ws.serialization,
|
|
2201
|
+
ws.attributes ? JSON.stringify(ws.attributes) : null,
|
|
2202
|
+
forkOwners.get(forkID) ?? null,
|
|
2203
|
+
...(options.timeoutMS !== undefined ? [options.timeoutMS] : []),
|
|
2204
|
+
];
|
|
2205
|
+
}));
|
|
2206
|
+
await this.insertRows(client, 'workflow_input', [...Object.keys(forkKeys[0]), 'workflow_uuid', 'inputs'], originalWorkflowIDs.map((origID, i) => [
|
|
2207
|
+
...Object.values(forkKeys[i]),
|
|
2208
|
+
forkedWorkflowIDs[i],
|
|
2209
|
+
statusByID.get(origID).inputs,
|
|
2210
|
+
]));
|
|
1766
2211
|
// Mark all original workflows as having been forked from.
|
|
1767
|
-
|
|
2212
|
+
const forkedFromParams = [];
|
|
2213
|
+
await client.query(`UPDATE "${this.schemaName}".workflow_status SET was_forked_from = TRUE
|
|
2214
|
+
WHERE ${this.wfIn(undefined, originalWorkflowIDs, forkedFromParams)}`, forkedFromParams);
|
|
1768
2215
|
// For workflows with start_step > 0, copy checkpoints/events/streams.
|
|
1769
2216
|
// Build a mapping CTE so each copy is a single SQL statement.
|
|
1770
2217
|
const forkMappings = originalWorkflowIDs
|
|
@@ -1781,55 +2228,85 @@ class SystemDatabase {
|
|
|
1781
2228
|
mIdx += 4;
|
|
1782
2229
|
}
|
|
1783
2230
|
const mappingCTE = `WITH mapping(orig_id, fork_id, start_step, owner) AS (VALUES ${mappingValues.join(', ')})`;
|
|
2231
|
+
// The forks live under their originals' shard key, read from the scope; a copy's params extend the mapping's.
|
|
2232
|
+
const copyParams = () => [...mappingParams];
|
|
2233
|
+
const keyed = (params) => {
|
|
2234
|
+
const columns = this.scopedColumns(params);
|
|
2235
|
+
return { names: columns.map(([c]) => `${c}, `).join(''), exprs: columns.map(([, e]) => `${e}, `).join('') };
|
|
2236
|
+
};
|
|
2237
|
+
// A replacement child is a fork, so it shares the replaced child's key.
|
|
2238
|
+
const childKey = this.keyColumn('oo', 'child_shard_key');
|
|
2239
|
+
if (options.replacementChildren && childKey !== undefined) {
|
|
2240
|
+
// A replacement names a child by ID, which can't pick one of several keys.
|
|
2241
|
+
const ambiguityParams = copyParams();
|
|
2242
|
+
const replaced = `$${ambiguityParams.push(Object.keys(options.replacementChildren))}::text[]`;
|
|
2243
|
+
const { rows: ambiguous } = await client.query(`${mappingCTE}
|
|
2244
|
+
SELECT oo.child_workflow_id
|
|
2245
|
+
FROM mapping m
|
|
2246
|
+
JOIN "${this.schemaName}".operation_outputs oo
|
|
2247
|
+
ON ${this.scoped('oo', ambiguityParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step
|
|
2248
|
+
WHERE oo.child_workflow_id = ANY(${replaced})
|
|
2249
|
+
GROUP BY oo.child_workflow_id
|
|
2250
|
+
HAVING COUNT(DISTINCT ${childKey}) > 1
|
|
2251
|
+
LIMIT 1`, ambiguityParams);
|
|
2252
|
+
if (ambiguous.length > 0) {
|
|
2253
|
+
throw new error_1.DBOSError(`Can't replace child workflow ${ambiguous[0].child_workflow_id}: the forked steps refer to children with that ID under more than one shard key`);
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
1784
2256
|
// Build the child_workflow_id expression, applying replacements if provided.
|
|
2257
|
+
const ooParams = copyParams();
|
|
1785
2258
|
let childWfExpr = 'oo.child_workflow_id';
|
|
1786
|
-
const ooParams = [...mappingParams];
|
|
1787
2259
|
if (options.replacementChildren && Object.keys(options.replacementChildren).length > 0) {
|
|
1788
2260
|
const whenClauses = [];
|
|
1789
2261
|
for (const [oldId, newId] of Object.entries(options.replacementChildren)) {
|
|
1790
|
-
whenClauses.push(`WHEN oo.child_workflow_id = $${
|
|
1791
|
-
ooParams.push(oldId, newId);
|
|
1792
|
-
mIdx += 2;
|
|
2262
|
+
whenClauses.push(`WHEN oo.child_workflow_id = $${ooParams.push(oldId)} THEN $${ooParams.push(newId)}::text`);
|
|
1793
2263
|
}
|
|
1794
2264
|
childWfExpr = `CASE ${whenClauses.join(' ')} ELSE oo.child_workflow_id END`;
|
|
1795
2265
|
}
|
|
2266
|
+
const ooKeys = keyed(ooParams);
|
|
1796
2267
|
// Copy operation outputs
|
|
1797
2268
|
await client.query(`${mappingCTE}
|
|
1798
2269
|
INSERT INTO "${this.schemaName}".operation_outputs
|
|
1799
|
-
(workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp)
|
|
1800
|
-
SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
2270
|
+
(${ooKeys.names}workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? ', child_shard_key' : ''})
|
|
2271
|
+
SELECT ${ooKeys.exprs}m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint${childKey !== undefined ? `, ${childKey}` : ''}
|
|
1801
2272
|
FROM mapping m
|
|
1802
2273
|
JOIN "${this.schemaName}".operation_outputs oo
|
|
1803
|
-
ON oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
|
|
2274
|
+
ON ${this.scoped('oo', ooParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
|
|
1804
2275
|
// Copy streams
|
|
2276
|
+
const streamParams = copyParams();
|
|
2277
|
+
const streamKeys = keyed(streamParams);
|
|
1805
2278
|
await client.query(`${mappingCTE}
|
|
1806
2279
|
INSERT INTO "${this.schemaName}".streams
|
|
1807
|
-
(workflow_uuid, key, value, serialization, "offset", function_id)
|
|
1808
|
-
SELECT m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
|
|
2280
|
+
(${streamKeys.names}workflow_uuid, key, value, serialization, "offset", function_id)
|
|
2281
|
+
SELECT ${streamKeys.exprs}m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
|
|
1809
2282
|
FROM mapping m
|
|
1810
2283
|
JOIN "${this.schemaName}".streams s
|
|
1811
|
-
ON s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`,
|
|
2284
|
+
ON ${this.scoped('s', streamParams)} AND s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`, streamParams);
|
|
1812
2285
|
// Copy events history
|
|
2286
|
+
const historyParams = copyParams();
|
|
2287
|
+
const historyKeys = keyed(historyParams);
|
|
1813
2288
|
await client.query(`${mappingCTE}
|
|
1814
2289
|
INSERT INTO "${this.schemaName}".workflow_events_history
|
|
1815
|
-
(workflow_uuid, function_id, key, value, serialization)
|
|
1816
|
-
SELECT m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
|
|
2290
|
+
(${historyKeys.names}workflow_uuid, function_id, key, value, serialization)
|
|
2291
|
+
SELECT ${historyKeys.exprs}m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
|
|
1817
2292
|
FROM mapping m
|
|
1818
2293
|
JOIN "${this.schemaName}".workflow_events_history weh
|
|
1819
|
-
ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`,
|
|
2294
|
+
ON ${this.scoped('weh', historyParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`, historyParams);
|
|
1820
2295
|
// Copy only the latest version of each event using a window function
|
|
2296
|
+
const eventParams = copyParams();
|
|
2297
|
+
const eventKeys = keyed(eventParams);
|
|
1821
2298
|
await client.query(`${mappingCTE}
|
|
1822
2299
|
INSERT INTO "${this.schemaName}".workflow_events
|
|
1823
|
-
(workflow_uuid, key, value, serialization)
|
|
1824
|
-
SELECT ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
|
|
2300
|
+
(${eventKeys.names}workflow_uuid, key, value, serialization)
|
|
2301
|
+
SELECT ${eventKeys.exprs}ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
|
|
1825
2302
|
FROM (
|
|
1826
2303
|
SELECT m.fork_id AS workflow_uuid, weh.key, weh.value, weh.serialization,
|
|
1827
|
-
ROW_NUMBER() OVER (PARTITION BY
|
|
2304
|
+
ROW_NUMBER() OVER (PARTITION BY m.fork_id, weh.key ORDER BY weh.function_id DESC) AS rn
|
|
1828
2305
|
FROM mapping m
|
|
1829
2306
|
JOIN "${this.schemaName}".workflow_events_history weh
|
|
1830
|
-
ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
|
|
2307
|
+
ON ${this.scoped('weh', eventParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
|
|
1831
2308
|
) ranked
|
|
1832
|
-
WHERE ranked.rn = 1`,
|
|
2309
|
+
WHERE ranked.rn = 1`, eventParams);
|
|
1833
2310
|
}
|
|
1834
2311
|
await client.query('COMMIT');
|
|
1835
2312
|
return forkedWorkflowIDs;
|
|
@@ -1952,27 +2429,36 @@ class SystemDatabase {
|
|
|
1952
2429
|
}
|
|
1953
2430
|
/** Cancellation check for polling waits: goes through the limiter so readers cannot starve the pool. */
|
|
1954
2431
|
async checkIfCanceledLimited(workflowID) {
|
|
1955
|
-
|
|
2432
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
2433
|
+
const params = [];
|
|
2434
|
+
const target = this.ownKey(workflowID, undefined, () => this.wf(undefined, workflowID, params));
|
|
2435
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${target}`, params));
|
|
2436
|
+
if (rows[0]?.status === workflow_1.StatusString.CANCELLED) {
|
|
2437
|
+
throw new error_1.DBOSWorkflowCancelledError(workflowID);
|
|
2438
|
+
}
|
|
1956
2439
|
}
|
|
1957
2440
|
watchForCancellation(workflowID) {
|
|
1958
2441
|
const controller = new AbortController();
|
|
1959
2442
|
let state = 'idle';
|
|
2443
|
+
// The running workflow's own key, resolved now for the poller that reads it later outside this context.
|
|
2444
|
+
const shardKey = this.ownKey(workflowID, undefined, () => this.keyHint(workflowID));
|
|
2445
|
+
const waiterKey = this.ownKey(workflowID, undefined, () => this.waiter(workflowID, ''));
|
|
1960
2446
|
const start = () => {
|
|
1961
2447
|
state = 'watching';
|
|
1962
2448
|
if (this.#destroyed)
|
|
1963
2449
|
return;
|
|
1964
|
-
let watchers = this.#cancelWatchers.get(
|
|
2450
|
+
let watchers = this.#cancelWatchers.get(waiterKey);
|
|
1965
2451
|
if (watchers === undefined) {
|
|
1966
|
-
watchers = new Set();
|
|
1967
|
-
this.#cancelWatchers.set(
|
|
2452
|
+
watchers = { workflowID, shardKey, controllers: new Set() };
|
|
2453
|
+
this.#cancelWatchers.set(waiterKey, watchers);
|
|
1968
2454
|
}
|
|
1969
|
-
watchers.add(controller);
|
|
1970
|
-
this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => this.#runCancelPoller());
|
|
2455
|
+
watchers.controllers.add(controller);
|
|
2456
|
+
this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => (0, shard_keys_1.withClearedShardKeyState)(() => this.#runCancelPoller()));
|
|
1971
2457
|
};
|
|
1972
2458
|
const stop = () => {
|
|
1973
|
-
const watchers = this.#cancelWatchers.get(
|
|
1974
|
-
if (watchers?.delete(controller) && watchers.size === 0) {
|
|
1975
|
-
this.#cancelWatchers.delete(
|
|
2459
|
+
const watchers = this.#cancelWatchers.get(waiterKey);
|
|
2460
|
+
if (watchers?.controllers.delete(controller) && watchers.controllers.size === 0) {
|
|
2461
|
+
this.#cancelWatchers.delete(waiterKey);
|
|
1976
2462
|
if (this.#cancelWatchers.size === 0)
|
|
1977
2463
|
this.#cancelPollerWake?.();
|
|
1978
2464
|
}
|
|
@@ -2011,15 +2497,36 @@ class SystemDatabase {
|
|
|
2011
2497
|
this.#cancelPollerLoop = undefined;
|
|
2012
2498
|
}
|
|
2013
2499
|
async #abortCancelledWatchers() {
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2500
|
+
// One read per shard key, each bound to its own workflows.
|
|
2501
|
+
const byKey = new Map();
|
|
2502
|
+
for (const { workflowID, shardKey } of this.#cancelWatchers.values()) {
|
|
2503
|
+
const ids = byKey.get(shardKey) ?? [];
|
|
2504
|
+
ids.push(workflowID);
|
|
2505
|
+
byKey.set(shardKey, ids);
|
|
2506
|
+
}
|
|
2507
|
+
for (const [shardKey, workflowIDs] of byKey) {
|
|
2508
|
+
// Each key's read on its own, so one key's failure doesn't hide another key's cancellations.
|
|
2509
|
+
let rows;
|
|
2510
|
+
try {
|
|
2511
|
+
({ rows } = await (0, shard_keys_1.bindShardKeys)(workflowIDs.map((id) => [id, shardKey]), () => {
|
|
2512
|
+
const params = [workflow_1.StatusString.CANCELLED];
|
|
2513
|
+
const ids = this.wfIn(undefined, workflowIDs, params);
|
|
2514
|
+
return this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${ids} AND status = $1`, params));
|
|
2515
|
+
}));
|
|
2516
|
+
}
|
|
2517
|
+
catch (e) {
|
|
2518
|
+
this.logger.warn(`Error polling status for cancellable steps: ${String(e)}`);
|
|
2018
2519
|
continue;
|
|
2019
|
-
|
|
2020
|
-
const
|
|
2021
|
-
for (const
|
|
2022
|
-
|
|
2520
|
+
}
|
|
2521
|
+
const cancelled = new Set(rows.map((row) => row.workflow_uuid));
|
|
2522
|
+
for (const [waiterKey, watchers] of this.#cancelWatchers) {
|
|
2523
|
+
if (watchers.shardKey !== shardKey || !cancelled.has(watchers.workflowID))
|
|
2524
|
+
continue;
|
|
2525
|
+
this.#cancelWatchers.delete(waiterKey);
|
|
2526
|
+
const reason = new error_1.DBOSWorkflowCancelledError(watchers.workflowID);
|
|
2527
|
+
for (const controller of watchers.controllers)
|
|
2528
|
+
controller.abort(reason);
|
|
2529
|
+
}
|
|
2023
2530
|
}
|
|
2024
2531
|
}
|
|
2025
2532
|
// A missing row normally means the workflow has not been inserted yet, so
|
|
@@ -2031,21 +2538,23 @@ class SystemDatabase {
|
|
|
2031
2538
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2032
2539
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2033
2540
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
2034
|
-
// Record the durable timeout deadline once before polling.
|
|
2541
|
+
// Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
|
|
2035
2542
|
// first call and reads back the same value on recovery, so it never changes across iterations.
|
|
2036
2543
|
if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
|
|
2037
|
-
finishTime = await this
|
|
2544
|
+
finishTime = await this.durableSleepInternal(callerID, timerFuncID, timeoutms);
|
|
2038
2545
|
}
|
|
2039
2546
|
while (true) {
|
|
2040
2547
|
if (callerID)
|
|
2041
2548
|
await this.checkIfCanceledLimited(callerID);
|
|
2042
2549
|
let rows;
|
|
2043
2550
|
try {
|
|
2551
|
+
const params = [];
|
|
2552
|
+
const target = this.wf('ws', workflowID, params);
|
|
2044
2553
|
({ rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT ws.status, COALESCE(wo.output, ws.output) AS output,
|
|
2045
2554
|
COALESCE(wo.error, ws.error) AS error, ws.serialization
|
|
2046
2555
|
FROM "${this.schemaName}".workflow_status ws
|
|
2047
|
-
LEFT JOIN "${this.schemaName}".workflow_output wo ON
|
|
2048
|
-
WHERE
|
|
2556
|
+
LEFT JOIN "${this.schemaName}".workflow_output wo ON ${this.wfJoin('ws', 'wo')}
|
|
2557
|
+
WHERE ${target}`, params)));
|
|
2049
2558
|
}
|
|
2050
2559
|
catch (e) {
|
|
2051
2560
|
const err = e;
|
|
@@ -2082,21 +2591,27 @@ class SystemDatabase {
|
|
|
2082
2591
|
}
|
|
2083
2592
|
}
|
|
2084
2593
|
async awaitFirstWorkflowId(workflowIds, callerID, pollingIntervalMs) {
|
|
2085
|
-
const placeholders = workflowIds.map((_, i) => `$${i + 1}`).join(', ');
|
|
2086
2594
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
2087
2595
|
while (true) {
|
|
2088
2596
|
if (callerID)
|
|
2089
2597
|
await this.checkIfCanceledLimited(callerID);
|
|
2090
|
-
const
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
LIMIT 1`, workflowIds));
|
|
2094
|
-
if (rows.length > 0) {
|
|
2095
|
-
return rows[0].workflow_uuid;
|
|
2598
|
+
const first = await this.checkFirstWorkflowId(workflowIds);
|
|
2599
|
+
if (first !== undefined) {
|
|
2600
|
+
return first;
|
|
2096
2601
|
}
|
|
2097
2602
|
await (0, utils_1.sleepms)(pollIntervalMs);
|
|
2098
2603
|
}
|
|
2099
2604
|
}
|
|
2605
|
+
/** The first of these workflows found complete, if any. */
|
|
2606
|
+
async checkFirstWorkflowId(workflowIds) {
|
|
2607
|
+
const params = [];
|
|
2608
|
+
const ids = this.wfIn(undefined, workflowIds, params);
|
|
2609
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
2610
|
+
WHERE ${ids}
|
|
2611
|
+
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
|
|
2612
|
+
LIMIT 1`, params));
|
|
2613
|
+
return rows[0]?.workflow_uuid;
|
|
2614
|
+
}
|
|
2100
2615
|
async awaitWorkflowIds(workflowIds, callerID, pollingIntervalMs) {
|
|
2101
2616
|
const remainingWorkflowIds = new Set(workflowIds);
|
|
2102
2617
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
@@ -2104,11 +2619,8 @@ class SystemDatabase {
|
|
|
2104
2619
|
const currentWorkflowIds = [...remainingWorkflowIds];
|
|
2105
2620
|
if (callerID)
|
|
2106
2621
|
await this.checkIfCanceledLimited(callerID);
|
|
2107
|
-
const
|
|
2108
|
-
|
|
2109
|
-
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, [currentWorkflowIds]));
|
|
2110
|
-
for (const row of rows) {
|
|
2111
|
-
remainingWorkflowIds.delete(row.workflow_uuid);
|
|
2622
|
+
for (const workflowID of await this.checkCompletedWorkflowIds(currentWorkflowIds)) {
|
|
2623
|
+
remainingWorkflowIds.delete(workflowID);
|
|
2112
2624
|
}
|
|
2113
2625
|
if (remainingWorkflowIds.size === 0) {
|
|
2114
2626
|
return;
|
|
@@ -2116,9 +2628,18 @@ class SystemDatabase {
|
|
|
2116
2628
|
await (0, utils_1.sleepms)(pollIntervalMs);
|
|
2117
2629
|
}
|
|
2118
2630
|
}
|
|
2631
|
+
/** Those of these workflows found complete. */
|
|
2632
|
+
async checkCompletedWorkflowIds(workflowIds) {
|
|
2633
|
+
const params = [];
|
|
2634
|
+
const ids = this.wfIn(undefined, workflowIds, params);
|
|
2635
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
2636
|
+
WHERE ${ids}
|
|
2637
|
+
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, params));
|
|
2638
|
+
return rows.map((row) => row.workflow_uuid);
|
|
2639
|
+
}
|
|
2119
2640
|
// ==================== Sleep ====================
|
|
2120
2641
|
async durableSleepms(workflowID, functionID, durationMS) {
|
|
2121
|
-
const endTime = await this
|
|
2642
|
+
const endTime = await this.durableSleepInternal(workflowID, functionID, durationMS, true);
|
|
2122
2643
|
while (Date.now() < endTime) {
|
|
2123
2644
|
await (0, utils_1.sleepms)(Math.min(endTime - Date.now(), utils_1.sleepConfig.maxTimeoutMS));
|
|
2124
2645
|
}
|
|
@@ -2126,31 +2647,28 @@ class SystemDatabase {
|
|
|
2126
2647
|
}
|
|
2127
2648
|
// ==================== Messaging ====================
|
|
2128
2649
|
nullTopic = '__null__topic__';
|
|
2129
|
-
async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey
|
|
2130
|
-
|
|
2131
|
-
|
|
2650
|
+
async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey,
|
|
2651
|
+
// The destination's shard key; required on a sharded system database.
|
|
2652
|
+
shardKey) {
|
|
2132
2653
|
const startTime = Date.now();
|
|
2133
2654
|
let missing;
|
|
2134
2655
|
const client = await this.connect();
|
|
2135
2656
|
try {
|
|
2136
2657
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
2137
2658
|
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
|
|
2138
|
-
await
|
|
2139
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2140
|
-
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
2659
|
+
await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2141
2660
|
return undefined;
|
|
2142
2661
|
});
|
|
2143
2662
|
await client.query('COMMIT');
|
|
2144
2663
|
}
|
|
2145
2664
|
catch (error) {
|
|
2146
2665
|
await client.query('ROLLBACK');
|
|
2147
|
-
|
|
2148
|
-
if (err.code === '23503') {
|
|
2666
|
+
if (error instanceof error_1.DBOSNonExistentWorkflowError) {
|
|
2149
2667
|
// Foreign key constraint violation (only expected for the INSERT query)
|
|
2150
|
-
missing =
|
|
2668
|
+
missing = error;
|
|
2151
2669
|
}
|
|
2152
2670
|
else {
|
|
2153
|
-
throw
|
|
2671
|
+
throw error;
|
|
2154
2672
|
}
|
|
2155
2673
|
}
|
|
2156
2674
|
finally {
|
|
@@ -2163,22 +2681,22 @@ class SystemDatabase {
|
|
|
2163
2681
|
}
|
|
2164
2682
|
}
|
|
2165
2683
|
/** Runs on `client` if given, joining its transaction; otherwise on the pool with retries. */
|
|
2166
|
-
async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client) {
|
|
2684
|
+
async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client, shardKey) {
|
|
2167
2685
|
if (client !== undefined) {
|
|
2168
|
-
return await this
|
|
2686
|
+
return await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2169
2687
|
}
|
|
2170
|
-
return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey);
|
|
2688
|
+
return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2171
2689
|
}
|
|
2172
2690
|
/** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
|
|
2173
|
-
async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
|
|
2691
|
+
async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2174
2692
|
const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
|
|
2175
2693
|
const client = await this.connect();
|
|
2176
2694
|
try {
|
|
2177
|
-
await this
|
|
2178
|
-
await this
|
|
2695
|
+
await this.inTransaction(client, async () => {
|
|
2696
|
+
await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2179
2697
|
// After the insert, in the order the recorded send locks, so they cannot deadlock.
|
|
2180
2698
|
if (ownerXid !== undefined) {
|
|
2181
|
-
await this
|
|
2699
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
2182
2700
|
}
|
|
2183
2701
|
});
|
|
2184
2702
|
}
|
|
@@ -2186,17 +2704,23 @@ class SystemDatabase {
|
|
|
2186
2704
|
client.release();
|
|
2187
2705
|
}
|
|
2188
2706
|
}
|
|
2189
|
-
async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey) {
|
|
2190
|
-
return await this
|
|
2707
|
+
async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2708
|
+
return await this.sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2191
2709
|
}
|
|
2192
|
-
async
|
|
2710
|
+
async sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2193
2711
|
topic = topic ?? this.nullTopic;
|
|
2194
|
-
//
|
|
2712
|
+
// An idempotency key is scoped per destination: suffix it with the recipient's workflow ID.
|
|
2195
2713
|
const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
|
|
2714
|
+
const values = {
|
|
2715
|
+
...this.wfValues(destinationID, shardKey),
|
|
2716
|
+
destination_uuid: destinationID,
|
|
2717
|
+
topic,
|
|
2718
|
+
message,
|
|
2719
|
+
serialization,
|
|
2720
|
+
message_uuid: messageUUID,
|
|
2721
|
+
};
|
|
2196
2722
|
try {
|
|
2197
|
-
await
|
|
2198
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2199
|
-
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
2723
|
+
await this.insertRows(db, 'notifications', Object.keys(values), [Object.values(values)], `ON CONFLICT (${this.keyConflict('message_uuid')}) DO NOTHING`);
|
|
2200
2724
|
}
|
|
2201
2725
|
catch (error) {
|
|
2202
2726
|
const err = error;
|
|
@@ -2220,10 +2744,10 @@ class SystemDatabase {
|
|
|
2220
2744
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2221
2745
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2222
2746
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
|
|
2223
|
-
// Record the durable timeout deadline once before polling.
|
|
2747
|
+
// Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
|
|
2224
2748
|
// first call and reads back the same value on recovery, so it never changes across iterations.
|
|
2225
2749
|
if (timeoutms) {
|
|
2226
|
-
finishTime = await this
|
|
2750
|
+
finishTime = await this.durableSleepInternal(workflowID, timeoutFunctionID, timeoutms);
|
|
2227
2751
|
}
|
|
2228
2752
|
while (true) {
|
|
2229
2753
|
// register the key with the global notifications listener.
|
|
@@ -2231,12 +2755,14 @@ class SystemDatabase {
|
|
|
2231
2755
|
const messagePromise = new Promise((resolve) => {
|
|
2232
2756
|
resolveNotification = resolve;
|
|
2233
2757
|
});
|
|
2234
|
-
const payload =
|
|
2758
|
+
const payload = this.waiter(workflowID, topic);
|
|
2235
2759
|
const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
|
|
2236
2760
|
try {
|
|
2237
2761
|
await this.checkIfCanceledLimited(workflowID);
|
|
2238
2762
|
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
2239
|
-
const
|
|
2763
|
+
const params = [topic];
|
|
2764
|
+
const target = this.wf(undefined, workflowID, params, 'destination_uuid');
|
|
2765
|
+
const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE ${target} AND topic=$1 AND consumed = false;`, params))).rows;
|
|
2240
2766
|
if (initRecvRows.length !== 0)
|
|
2241
2767
|
break;
|
|
2242
2768
|
const ct = Date.now();
|
|
@@ -2263,21 +2789,22 @@ class SystemDatabase {
|
|
|
2263
2789
|
const client = await this.connect();
|
|
2264
2790
|
try {
|
|
2265
2791
|
await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
|
|
2792
|
+
const consumeParams = [topic, functionID];
|
|
2266
2793
|
const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
|
|
2267
|
-
SET consumed = true, consumed_by_function_id = $
|
|
2268
|
-
WHERE
|
|
2269
|
-
AND topic = $
|
|
2794
|
+
SET consumed = true, consumed_by_function_id = $2
|
|
2795
|
+
WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
|
|
2796
|
+
AND topic = $1
|
|
2270
2797
|
AND consumed = false
|
|
2271
2798
|
AND message_uuid = (
|
|
2272
2799
|
SELECT message_uuid
|
|
2273
2800
|
FROM "${this.schemaName}".notifications
|
|
2274
|
-
WHERE
|
|
2275
|
-
AND topic = $
|
|
2801
|
+
WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
|
|
2802
|
+
AND topic = $1
|
|
2276
2803
|
AND consumed = false
|
|
2277
2804
|
ORDER BY created_at_epoch_ms ASC
|
|
2278
2805
|
LIMIT 1
|
|
2279
2806
|
)
|
|
2280
|
-
RETURNING notifications.message, notifications.serialization;`,
|
|
2807
|
+
RETURNING notifications.message, notifications.serialization;`, consumeParams)).rows;
|
|
2281
2808
|
if (finalRecvRows.length > 0) {
|
|
2282
2809
|
message = finalRecvRows[0].message;
|
|
2283
2810
|
serialization = finalRecvRows[0].serialization;
|
|
@@ -2306,16 +2833,19 @@ class SystemDatabase {
|
|
|
2306
2833
|
// Only a real write (not a replay) should wake readers.
|
|
2307
2834
|
let didWrite = false;
|
|
2308
2835
|
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SETEVENT, workflowID, functionID, async () => {
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2836
|
+
const keys = this.wfValues(workflowID);
|
|
2837
|
+
const keyColumns = Object.keys(keys).map((c) => `${c}, `);
|
|
2838
|
+
const keyPlaceholders = (from) => Object.keys(keys).map((_, i) => `$${from + i}, `);
|
|
2839
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_events (${keyColumns.join('')}workflow_uuid, key, value, serialization)
|
|
2840
|
+
VALUES (${keyPlaceholders(5).join('')}$1, $2, $3, $4)
|
|
2841
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'key')})
|
|
2312
2842
|
DO UPDATE SET value = $3, serialization = $4
|
|
2313
|
-
RETURNING workflow_uuid;`, [workflowID, key, message, serialization]);
|
|
2843
|
+
RETURNING workflow_uuid;`, [workflowID, key, message, serialization, ...Object.values(keys)]);
|
|
2314
2844
|
// Also write to the immutable history table for fork support
|
|
2315
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (workflow_uuid, function_id, key, value, serialization)
|
|
2316
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2317
|
-
ON CONFLICT (workflow_uuid, function_id, key)
|
|
2318
|
-
DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization]);
|
|
2845
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (${keyColumns.join('')}workflow_uuid, function_id, key, value, serialization)
|
|
2846
|
+
VALUES (${keyPlaceholders(6).join('')}$1, $2, $3, $4, $5)
|
|
2847
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id', 'key')})
|
|
2848
|
+
DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization, ...Object.values(keys)]);
|
|
2319
2849
|
didWrite = true;
|
|
2320
2850
|
return undefined;
|
|
2321
2851
|
});
|
|
@@ -2349,15 +2879,15 @@ class SystemDatabase {
|
|
|
2349
2879
|
// Get the return the value. if it's in the DB, otherwise return null.
|
|
2350
2880
|
let value = null;
|
|
2351
2881
|
let valueSer = null;
|
|
2352
|
-
const payloadKey =
|
|
2882
|
+
const payloadKey = this.waiter(workflowID, key);
|
|
2353
2883
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2354
2884
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2355
2885
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
|
|
2356
2886
|
// If we have a callerWorkflow, we want a durable sleep, otherwise, not. Record the durable
|
|
2357
|
-
// timeout deadline once before polling.
|
|
2887
|
+
// timeout deadline once before polling. durableSleepInternal persists it on the first call and reads
|
|
2358
2888
|
// back the same value on recovery, so it never changes across iterations.
|
|
2359
2889
|
if (callerWorkflow && timeoutms) {
|
|
2360
|
-
finishTime = await this
|
|
2890
|
+
finishTime = await this.durableSleepInternal(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms);
|
|
2361
2891
|
}
|
|
2362
2892
|
// Register the key with the global notifications listener first... we do not want to look in the DB first
|
|
2363
2893
|
// or that would cause a timing hole.
|
|
@@ -2371,9 +2901,11 @@ class SystemDatabase {
|
|
|
2371
2901
|
if (callerWorkflow?.workflowID)
|
|
2372
2902
|
await this.checkIfCanceledLimited(callerWorkflow?.workflowID);
|
|
2373
2903
|
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
2904
|
+
const params = [key];
|
|
2905
|
+
const target = this.wf(undefined, workflowID, params);
|
|
2374
2906
|
const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT key, value, serialization
|
|
2375
2907
|
FROM "${this.schemaName}".workflow_events
|
|
2376
|
-
WHERE
|
|
2908
|
+
WHERE ${target} AND key=$1;`, params))).rows;
|
|
2377
2909
|
if (initRecvRows.length > 0) {
|
|
2378
2910
|
value = initRecvRows[0].value;
|
|
2379
2911
|
valueSer = initRecvRows[0].serialization;
|
|
@@ -2412,15 +2944,12 @@ class SystemDatabase {
|
|
|
2412
2944
|
try {
|
|
2413
2945
|
while (true) {
|
|
2414
2946
|
try {
|
|
2415
|
-
await this
|
|
2947
|
+
await this.inTransaction(client, async () => {
|
|
2416
2948
|
// Derives the first unused offset inside the insert; two writers can still pick the same one.
|
|
2417
|
-
await
|
|
2418
|
-
SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
|
|
2419
|
-
FROM "${this.schemaName}".streams s
|
|
2420
|
-
WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
|
|
2949
|
+
await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
|
|
2421
2950
|
// After the insert, in the order the workflow-level writes lock, so they cannot deadlock.
|
|
2422
2951
|
if (ownerXid !== undefined) {
|
|
2423
|
-
await this
|
|
2952
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
2424
2953
|
}
|
|
2425
2954
|
});
|
|
2426
2955
|
}
|
|
@@ -2454,10 +2983,7 @@ class SystemDatabase {
|
|
|
2454
2983
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
2455
2984
|
await this.#runAndRecordResult(client, functionName, workflowID, functionID, async () => {
|
|
2456
2985
|
// Derives the first unused offset inside the insert; two writers can still pick the same one.
|
|
2457
|
-
await
|
|
2458
|
-
SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
|
|
2459
|
-
FROM "${this.schemaName}".streams s
|
|
2460
|
-
WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
|
|
2986
|
+
await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
|
|
2461
2987
|
didWrite = true;
|
|
2462
2988
|
return undefined;
|
|
2463
2989
|
});
|
|
@@ -2489,6 +3015,15 @@ class SystemDatabase {
|
|
|
2489
3015
|
client.release();
|
|
2490
3016
|
}
|
|
2491
3017
|
}
|
|
3018
|
+
/** Append a value to a stream, at the first unused offset, derived inside the insert. */
|
|
3019
|
+
async #insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization) {
|
|
3020
|
+
const params = [key, serializedValue, functionID, serialization, workflowID];
|
|
3021
|
+
const keyColumns = this.wfColumns(workflowID, params);
|
|
3022
|
+
await client.query(`INSERT INTO "${this.schemaName}".streams (${keyColumns.map(([c]) => `${c}, `).join('')}workflow_uuid, key, value, "offset", function_id, serialization)
|
|
3023
|
+
SELECT ${keyColumns.map(([, e]) => `${e}, `).join('')}$5::text, $1::text, $2::text, COALESCE(MAX(s."offset"), -1) + 1, $3::int, $4::text
|
|
3024
|
+
FROM "${this.schemaName}".streams s
|
|
3025
|
+
WHERE ${this.wf('s', workflowID, params)} AND s.key = $1`, params);
|
|
3026
|
+
}
|
|
2492
3027
|
async closeStreamFromWorkflow(workflowID, functionID, key) {
|
|
2493
3028
|
await this.writeStreamFromWorkflow(workflowID, functionID, key, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name(), exports.DBOS_FUNCNAME_CLOSESTREAM);
|
|
2494
3029
|
}
|
|
@@ -2498,13 +3033,15 @@ class SystemDatabase {
|
|
|
2498
3033
|
// Read the value at `offset` and the workflow's status in one query: status null = no such workflow, value undefined = nothing at that offset.
|
|
2499
3034
|
async readStreamValue(workflowID, key, offset) {
|
|
2500
3035
|
// LEFT JOIN so a workflow with nothing at offset still reports its status (single PK lookup); under the poll limiter, inside @dbRetry so the permit frees across backoff.
|
|
3036
|
+
const params = [key, offset];
|
|
3037
|
+
const target = this.wf('ws', workflowID, params);
|
|
2501
3038
|
const result = await this.#pollWithLimiter(() => this.pool.query(
|
|
2502
3039
|
// "offset" is a reserved word, so alias it (stream_offset) to read it back plainly.
|
|
2503
3040
|
`SELECT ws.status AS status, s.value AS value, s.serialization AS serialization, s."offset" AS stream_offset
|
|
2504
3041
|
FROM "${this.schemaName}".workflow_status ws
|
|
2505
3042
|
LEFT OUTER JOIN "${this.schemaName}".streams s
|
|
2506
|
-
ON s
|
|
2507
|
-
WHERE
|
|
3043
|
+
ON ${this.wfJoin('s', 'ws')} AND s.key = $1 AND s."offset" = $2
|
|
3044
|
+
WHERE ${target}`, params));
|
|
2508
3045
|
if (result.rows.length === 0) {
|
|
2509
3046
|
return { status: null, value: undefined };
|
|
2510
3047
|
}
|
|
@@ -2641,7 +3178,7 @@ class SystemDatabase {
|
|
|
2641
3178
|
// Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v3, so cost scales with the number of partitions rather than the backlog depth.
|
|
2642
3179
|
const params = [queueName, workflow_1.StatusString.ENQUEUED];
|
|
2643
3180
|
// Only partitions this application can actually dequeue from.
|
|
2644
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3181
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2645
3182
|
const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
|
|
2646
3183
|
(SELECT MIN(queue_partition_key) AS pk
|
|
2647
3184
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -2658,7 +3195,14 @@ class SystemDatabase {
|
|
|
2658
3195
|
}
|
|
2659
3196
|
async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0,
|
|
2660
3197
|
// Written to every claimed row: the dispatches of this claim own their executions.
|
|
2661
|
-
ownerXid = null) {
|
|
3198
|
+
ownerXid = null, isLatestVersion) {
|
|
3199
|
+
const { claimedIDs } = await this.claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount, partitionLocalRunningCount, ownerXid, isLatestVersion);
|
|
3200
|
+
return claimedIDs;
|
|
3201
|
+
}
|
|
3202
|
+
/** Claim this queue's next workflows, plus the limit that left no room to claim any (undefined if every limit had room). */
|
|
3203
|
+
async claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0, ownerXid = null, isLatestVersion) {
|
|
3204
|
+
// Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
|
|
3205
|
+
const startTimeMs = Date.now();
|
|
2662
3206
|
const claimedIDs = [];
|
|
2663
3207
|
const partitionParams = queuePartitionKey !== undefined ? [queuePartitionKey] : [];
|
|
2664
3208
|
// Shares a concurrency or rate limit budget with other executors.
|
|
@@ -2668,6 +3212,10 @@ class SystemDatabase {
|
|
|
2668
3212
|
queue.partitionRateLimit !== undefined;
|
|
2669
3213
|
// Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
|
|
2670
3214
|
const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
|
|
3215
|
+
// A sharded database keeps the version row under another key, which the claim's transaction cannot span.
|
|
3216
|
+
if (isLatestVersion === undefined && this.sharded) {
|
|
3217
|
+
isLatestVersion = await this.isLatestVersion(appVersion);
|
|
3218
|
+
}
|
|
2671
3219
|
const client = await this.connect();
|
|
2672
3220
|
try {
|
|
2673
3221
|
// Default to READ COMMITTED except with a budget shared across executors
|
|
@@ -2681,7 +3229,7 @@ class SystemDatabase {
|
|
|
2681
3229
|
const rateLimitRemaining = async (rateLimit, partitionScoped) => {
|
|
2682
3230
|
const params = [queue.name, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED, rateLimit.periodSec * 1000];
|
|
2683
3231
|
// Count only what this application would dequeue, matching the select below.
|
|
2684
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3232
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2685
3233
|
const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
|
|
2686
3234
|
const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
2687
3235
|
WHERE queue_name = $1
|
|
@@ -2700,7 +3248,7 @@ class SystemDatabase {
|
|
|
2700
3248
|
*/
|
|
2701
3249
|
const pendingCount = async (partitionScoped) => {
|
|
2702
3250
|
const params = [queue.name, workflow_1.StatusString.PENDING];
|
|
2703
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3251
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2704
3252
|
const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
|
|
2705
3253
|
const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
2706
3254
|
WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
|
|
@@ -2708,27 +3256,38 @@ class SystemDatabase {
|
|
|
2708
3256
|
};
|
|
2709
3257
|
// Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
|
|
2710
3258
|
let maxTasks = Infinity;
|
|
3259
|
+
// The first limit found with no room, which the caller reads as why nothing was claimed.
|
|
3260
|
+
let stopReason = undefined;
|
|
3261
|
+
const limit = (room, reason) => {
|
|
3262
|
+
maxTasks = Math.min(maxTasks, room);
|
|
3263
|
+
if (maxTasks <= 0)
|
|
3264
|
+
stopReason ??= reason;
|
|
3265
|
+
};
|
|
2711
3266
|
if (queue.workerConcurrency !== undefined) {
|
|
2712
3267
|
// Use the in-memory registry for this worker's running count — avoids a DB round trip.
|
|
2713
|
-
|
|
3268
|
+
limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
|
|
2714
3269
|
}
|
|
2715
3270
|
if (queue.partitionWorkerConcurrency !== undefined) {
|
|
2716
|
-
|
|
3271
|
+
limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
|
|
2717
3272
|
}
|
|
2718
3273
|
if (maxTasks <= 0) {
|
|
2719
3274
|
await client.query('COMMIT');
|
|
2720
|
-
return claimedIDs;
|
|
3275
|
+
return { claimedIDs, stopReason };
|
|
2721
3276
|
}
|
|
2722
3277
|
if (queue.rateLimit !== undefined) {
|
|
2723
3278
|
// Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
|
|
2724
|
-
|
|
3279
|
+
limit(await rateLimitRemaining(queue.rateLimit, false), 'limiter');
|
|
3280
|
+
if (maxTasks <= 0) {
|
|
3281
|
+
await client.query('COMMIT');
|
|
3282
|
+
return { claimedIDs, stopReason };
|
|
3283
|
+
}
|
|
2725
3284
|
}
|
|
2726
3285
|
if (queue.partitionRateLimit !== undefined) {
|
|
2727
|
-
|
|
3286
|
+
limit(await rateLimitRemaining(queue.partitionRateLimit, true), 'partition_limiter');
|
|
2728
3287
|
}
|
|
2729
3288
|
if (maxTasks <= 0) {
|
|
2730
3289
|
await client.query('COMMIT');
|
|
2731
|
-
return claimedIDs;
|
|
3290
|
+
return { claimedIDs, stopReason };
|
|
2732
3291
|
}
|
|
2733
3292
|
if (queue.concurrency !== undefined) {
|
|
2734
3293
|
// Global concurrency still requires a DB query since other workers may be running workflows too.
|
|
@@ -2736,23 +3295,26 @@ class SystemDatabase {
|
|
|
2736
3295
|
if (totalRunningTasks > queue.concurrency) {
|
|
2737
3296
|
this.logger.warn(`Total running tasks (${totalRunningTasks}) exceeds the global concurrency limit (${queue.concurrency})`);
|
|
2738
3297
|
}
|
|
2739
|
-
|
|
3298
|
+
limit(Math.max(0, queue.concurrency - totalRunningTasks), 'concurrency');
|
|
3299
|
+
if (maxTasks <= 0) {
|
|
3300
|
+
await client.query('COMMIT');
|
|
3301
|
+
return { claimedIDs, stopReason };
|
|
3302
|
+
}
|
|
2740
3303
|
}
|
|
2741
3304
|
if (queue.partitionConcurrency !== undefined) {
|
|
2742
3305
|
const partitionRunningTasks = await pendingCount(true);
|
|
2743
3306
|
if (partitionRunningTasks > queue.partitionConcurrency) {
|
|
2744
3307
|
this.logger.warn(`Total running tasks (${partitionRunningTasks}) on partition ${queuePartitionKey} of queue ${queue.name} exceeds the partition concurrency limit (${queue.partitionConcurrency})`);
|
|
2745
3308
|
}
|
|
2746
|
-
|
|
3309
|
+
limit(Math.max(0, queue.partitionConcurrency - partitionRunningTasks), 'partition_concurrency');
|
|
2747
3310
|
}
|
|
2748
3311
|
// Return immediately if there are no available tasks due to flow control limits
|
|
2749
3312
|
if (maxTasks <= 0) {
|
|
2750
3313
|
await client.query('COMMIT');
|
|
2751
|
-
return claimedIDs;
|
|
3314
|
+
return { claimedIDs, stopReason };
|
|
2752
3315
|
}
|
|
2753
3316
|
// Retrieve the first max_tasks workflows in the queue.
|
|
2754
|
-
|
|
2755
|
-
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
3317
|
+
isLatestVersion ??= await this.isLatestVersion(appVersion, client);
|
|
2756
3318
|
const versionClause = isLatestVersion
|
|
2757
3319
|
? '(application_version = $3 OR application_version IS NULL)'
|
|
2758
3320
|
: 'application_version = $3';
|
|
@@ -2762,13 +3324,17 @@ class SystemDatabase {
|
|
|
2762
3324
|
const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
|
|
2763
3325
|
const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
|
|
2764
3326
|
const selectScope = this.appNameFilter('application_name', this.appName, selectParams);
|
|
3327
|
+
const keyScope = this.scoped(undefined, selectParams);
|
|
3328
|
+
const claimGate = this.claimGate(queue.name, queuePartitionKey, selectParams);
|
|
2765
3329
|
const selectQuery = `
|
|
2766
3330
|
SELECT workflow_uuid
|
|
2767
3331
|
FROM "${this.schemaName}".workflow_status
|
|
2768
|
-
WHERE
|
|
3332
|
+
WHERE ${keyScope}
|
|
3333
|
+
AND status = $1
|
|
2769
3334
|
AND queue_name = $2
|
|
2770
3335
|
AND ${versionClause}
|
|
2771
3336
|
AND ${selectScope}
|
|
3337
|
+
AND ${claimGate}
|
|
2772
3338
|
${queuePartitionKey !== undefined ? 'AND queue_partition_key = $4' : ''}
|
|
2773
3339
|
ORDER BY priority ASC, created_at ASC
|
|
2774
3340
|
${limitClause}
|
|
@@ -2787,31 +3353,36 @@ class SystemDatabase {
|
|
|
2787
3353
|
executorID,
|
|
2788
3354
|
appVersion,
|
|
2789
3355
|
queue.rateLimit !== undefined || queue.partitionRateLimit !== undefined,
|
|
2790
|
-
workflowIDs,
|
|
2791
3356
|
workflow_1.StatusString.ENQUEUED,
|
|
2792
3357
|
// Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
|
|
2793
3358
|
this.appName ?? null,
|
|
2794
3359
|
ownerXid,
|
|
3360
|
+
startTimeMs,
|
|
2795
3361
|
];
|
|
3362
|
+
const claimedIDsFilter = this.wfIn(undefined, workflowIDs, updateParams);
|
|
2796
3363
|
// Re-check ownership alongside status, as the partitioned claim guard does.
|
|
2797
3364
|
const claimScope = this.appNameFilter('application_name', this.appName, updateParams);
|
|
2798
3365
|
// RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
|
|
2799
3366
|
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2800
3367
|
SET status = $1,
|
|
2801
|
-
owner_xid = $
|
|
3368
|
+
owner_xid = $7,
|
|
2802
3369
|
executor_id = $2,
|
|
2803
3370
|
application_version = $3,
|
|
2804
3371
|
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2805
3372
|
rate_limited = $4,
|
|
2806
|
-
application_name = COALESCE(application_name, $
|
|
3373
|
+
application_name = COALESCE(application_name, $6),
|
|
2807
3374
|
recovery_attempts = recovery_attempts + 1,
|
|
2808
3375
|
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2809
3376
|
workflow_deadline_epoch_ms = CASE
|
|
2810
|
-
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
|
|
2811
|
-
|
|
3377
|
+
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
3378
|
+
CASE
|
|
3379
|
+
WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
|
|
3380
|
+
THEN 9223372036854775807
|
|
3381
|
+
ELSE $8::bigint + workflow_timeout_ms
|
|
3382
|
+
END
|
|
2812
3383
|
ELSE workflow_deadline_epoch_ms
|
|
2813
3384
|
END
|
|
2814
|
-
WHERE
|
|
3385
|
+
WHERE ${claimedIDsFilter} AND status = $5 AND ${claimScope}
|
|
2815
3386
|
RETURNING workflow_uuid`, updateParams);
|
|
2816
3387
|
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
2817
3388
|
for (const id of workflowIDs) {
|
|
@@ -2829,14 +3400,16 @@ class SystemDatabase {
|
|
|
2829
3400
|
client.release();
|
|
2830
3401
|
}
|
|
2831
3402
|
// Return the IDs of all functions we marked started
|
|
2832
|
-
return claimedIDs;
|
|
3403
|
+
return { claimedIDs };
|
|
2833
3404
|
}
|
|
2834
3405
|
/** Max heads admitted per sweep: bounds dispatch, not the partition walk; an unconstrained sweep takes the lowest keys, so higher keys can wait under sustained load. */
|
|
2835
3406
|
partitionedDequeueSweepCap = 8192;
|
|
2836
3407
|
/** Dequeue each partition's head-of-line workflow in one transaction, at most {@link partitionedDequeueSweepCap} per sweep; only valid for partition-concurrency-1 queues with no queue-wide limit. */
|
|
2837
3408
|
async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity,
|
|
2838
3409
|
// Written to every claimed row: the dispatches of this claim own their executions.
|
|
2839
|
-
ownerXid = null) {
|
|
3410
|
+
ownerXid = null, isLatestVersion) {
|
|
3411
|
+
// Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
|
|
3412
|
+
const startTimeMs = Date.now();
|
|
2840
3413
|
if (queue.partitionConcurrency !== 1 ||
|
|
2841
3414
|
queue.concurrency !== undefined ||
|
|
2842
3415
|
queue.rateLimit !== undefined ||
|
|
@@ -2844,11 +3417,14 @@ class SystemDatabase {
|
|
|
2844
3417
|
throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
|
|
2845
3418
|
}
|
|
2846
3419
|
// partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
|
|
3420
|
+
// A sharded database keeps the version row under another key, which the claim's transaction cannot span.
|
|
3421
|
+
if (isLatestVersion === undefined && this.sharded) {
|
|
3422
|
+
isLatestVersion = await this.isLatestVersion(appVersion);
|
|
3423
|
+
}
|
|
2847
3424
|
const client = await this.connect();
|
|
2848
3425
|
try {
|
|
2849
3426
|
await client.query('BEGIN');
|
|
2850
|
-
|
|
2851
|
-
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
3427
|
+
isLatestVersion ??= await this.isLatestVersion(appVersion, client);
|
|
2852
3428
|
const versionClause = (n) => isLatestVersion
|
|
2853
3429
|
? `(application_version = $${n} OR application_version IS NULL)`
|
|
2854
3430
|
: `application_version = $${n}`;
|
|
@@ -2863,7 +3439,8 @@ class SystemDatabase {
|
|
|
2863
3439
|
workflow_1.StatusString.PENDING,
|
|
2864
3440
|
sweepLimit,
|
|
2865
3441
|
];
|
|
2866
|
-
const candidateScope = this.appNameFilter('application_name', this.appName, candidateParams)
|
|
3442
|
+
const candidateScope = `${this.scoped(undefined, candidateParams)} AND ${this.appNameFilter('application_name', this.appName, candidateParams)}`;
|
|
3443
|
+
const probeScope = this.scoped(undefined, candidateParams);
|
|
2867
3444
|
// Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
|
|
2868
3445
|
const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
|
|
2869
3446
|
(SELECT MIN(queue_partition_key) AS pk
|
|
@@ -2880,11 +3457,11 @@ class SystemDatabase {
|
|
|
2880
3457
|
SELECT partitions.pk
|
|
2881
3458
|
FROM partitions
|
|
2882
3459
|
WHERE partitions.pk IS NOT NULL
|
|
2883
|
-
--
|
|
3460
|
+
-- No application filter by design: a mutual-exclusion probe must block on any owner's row.
|
|
2884
3461
|
AND NOT EXISTS (
|
|
2885
3462
|
SELECT 1
|
|
2886
3463
|
FROM "${this.schemaName}".workflow_status
|
|
2887
|
-
WHERE queue_name = $1 AND status = $4
|
|
3464
|
+
WHERE ${probeScope} AND queue_name = $1 AND status = $4
|
|
2888
3465
|
AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
|
|
2889
3466
|
)
|
|
2890
3467
|
ORDER BY ${sweepOrder}
|
|
@@ -2913,18 +3490,19 @@ class SystemDatabase {
|
|
|
2913
3490
|
}
|
|
2914
3491
|
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
|
|
2915
3492
|
// Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
|
|
2916
|
-
const claimGuard = (ids, status, name, version, scope) =>
|
|
3493
|
+
const claimGuard = (ids, status, name, version, scope) => `${ids}
|
|
2917
3494
|
AND status = $${status}
|
|
2918
3495
|
AND queue_name = $${name}
|
|
2919
3496
|
AND queue_partition_key IS NOT NULL
|
|
2920
3497
|
AND ${versionClause(version)}
|
|
2921
3498
|
AND ${scope}`;
|
|
2922
|
-
const lockedParams = [
|
|
3499
|
+
const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
|
|
3500
|
+
const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
|
|
2923
3501
|
const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
|
|
2924
3502
|
// Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
|
|
2925
3503
|
const lockedResult = await client.query(`SELECT workflow_uuid
|
|
2926
3504
|
FROM "${this.schemaName}".workflow_status
|
|
2927
|
-
WHERE ${claimGuard(1, 2, 3,
|
|
3505
|
+
WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
|
|
2928
3506
|
FOR UPDATE SKIP LOCKED`, lockedParams);
|
|
2929
3507
|
const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
|
|
2930
3508
|
// Preserve partition order for submission.
|
|
@@ -2936,32 +3514,37 @@ class SystemDatabase {
|
|
|
2936
3514
|
const flipParams = [
|
|
2937
3515
|
workflow_1.StatusString.PENDING,
|
|
2938
3516
|
executorID,
|
|
2939
|
-
claimIDs,
|
|
2940
3517
|
workflow_1.StatusString.ENQUEUED,
|
|
2941
3518
|
queue.name,
|
|
2942
3519
|
appVersion,
|
|
2943
3520
|
// Claim the row, as the unpartitioned dequeue does.
|
|
2944
3521
|
this.appName ?? null,
|
|
2945
3522
|
ownerXid,
|
|
3523
|
+
startTimeMs,
|
|
2946
3524
|
];
|
|
3525
|
+
const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
|
|
2947
3526
|
const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
|
|
2948
3527
|
// Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
|
|
2949
3528
|
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2950
3529
|
SET status = $1,
|
|
2951
|
-
owner_xid = $
|
|
3530
|
+
owner_xid = $7,
|
|
2952
3531
|
executor_id = $2,
|
|
2953
|
-
application_version = $
|
|
3532
|
+
application_version = $5,
|
|
2954
3533
|
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2955
3534
|
rate_limited = FALSE,
|
|
2956
|
-
application_name = COALESCE(application_name, $
|
|
3535
|
+
application_name = COALESCE(application_name, $6),
|
|
2957
3536
|
recovery_attempts = recovery_attempts + 1,
|
|
2958
3537
|
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2959
3538
|
workflow_deadline_epoch_ms = CASE
|
|
2960
|
-
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
|
|
2961
|
-
|
|
3539
|
+
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
3540
|
+
CASE
|
|
3541
|
+
WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
|
|
3542
|
+
THEN 9223372036854775807
|
|
3543
|
+
ELSE $8::bigint + workflow_timeout_ms
|
|
3544
|
+
END
|
|
2962
3545
|
ELSE workflow_deadline_epoch_ms
|
|
2963
3546
|
END
|
|
2964
|
-
WHERE ${claimGuard(3, 4, 5,
|
|
3547
|
+
WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
|
|
2965
3548
|
RETURNING workflow_uuid`, flipParams);
|
|
2966
3549
|
await client.query('COMMIT');
|
|
2967
3550
|
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
@@ -2980,7 +3563,15 @@ class SystemDatabase {
|
|
|
2980
3563
|
}
|
|
2981
3564
|
}
|
|
2982
3565
|
// ==================== Queries & Maintenance ====================
|
|
2983
|
-
|
|
3566
|
+
/** With `keyedIDs`, the listed workflow IDs are addressed by their shard keys, as a batch read of known workflows. */
|
|
3567
|
+
async listWorkflows(input, options = {}) {
|
|
3568
|
+
return await this.#listWorkflows(input, options);
|
|
3569
|
+
}
|
|
3570
|
+
/**
|
|
3571
|
+
* Unretried body of {@link listWorkflows}. An ID-keyed read runs on `client` when given, so a caller
|
|
3572
|
+
* holding a transaction never waits on a second pool connection; the caller owns the retry.
|
|
3573
|
+
*/
|
|
3574
|
+
async #listWorkflows(input, options = {}, client) {
|
|
2984
3575
|
const schemaName = this.schemaName;
|
|
2985
3576
|
const selectColumns = [
|
|
2986
3577
|
'workflow_uuid',
|
|
@@ -3022,16 +3613,23 @@ class SystemDatabase {
|
|
|
3022
3613
|
const payloadJoins = [];
|
|
3023
3614
|
if (input.loadInput) {
|
|
3024
3615
|
payloadColumns.push('COALESCE(wi.inputs, workflow_status.inputs) AS inputs');
|
|
3025
|
-
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON wi
|
|
3616
|
+
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON ${this.wfJoin('wi', 'workflow_status')}`);
|
|
3026
3617
|
}
|
|
3027
3618
|
if (input.loadOutput) {
|
|
3028
3619
|
payloadColumns.push('COALESCE(wo.output, workflow_status.output) AS output');
|
|
3029
3620
|
payloadColumns.push('COALESCE(wo.error, workflow_status.error) AS error');
|
|
3030
|
-
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON wo
|
|
3621
|
+
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON ${this.wfJoin('wo', 'workflow_status')}`);
|
|
3031
3622
|
}
|
|
3032
3623
|
if (input.loadInput || input.loadOutput) {
|
|
3033
3624
|
selectColumns.push('serialization');
|
|
3034
3625
|
}
|
|
3626
|
+
const keyColumns = [];
|
|
3627
|
+
const keyColumn = this.keyColumn('workflow_status');
|
|
3628
|
+
if (keyColumn !== undefined)
|
|
3629
|
+
keyColumns.push(`${keyColumn} AS shard_key`);
|
|
3630
|
+
const parentKeyColumn = this.keyColumn('workflow_status', 'parent_shard_key');
|
|
3631
|
+
if (parentKeyColumn !== undefined)
|
|
3632
|
+
keyColumns.push(`${parentKeyColumn} AS parent_shard_key`);
|
|
3035
3633
|
input.sortDesc = input.sortDesc ?? false; // By default, sort in ascending order
|
|
3036
3634
|
// Build WHERE clauses
|
|
3037
3635
|
const whereClauses = [];
|
|
@@ -3069,9 +3667,23 @@ class SystemDatabase {
|
|
|
3069
3667
|
// application, as on every other observability query.
|
|
3070
3668
|
// Falsy, not just undefined: a Conductor request carries an omitted list as JSON null.
|
|
3071
3669
|
const idKeyed = (input.workflowIDs?.length ?? 0) > 0;
|
|
3670
|
+
if (client && !idKeyed)
|
|
3671
|
+
throw new TypeError('listWorkflows takes a client only for an ID-keyed read');
|
|
3072
3672
|
whereClauses.push(idKeyed
|
|
3073
3673
|
? this.appNameFilter('application_name', input.applicationName, params)
|
|
3074
3674
|
: this.observabilityFilter('application_name', input.applicationName, params));
|
|
3675
|
+
// An empty key list matches nothing, rather than every key.
|
|
3676
|
+
if (input.shardKey !== undefined && input.shardKey !== null) {
|
|
3677
|
+
const keys = Array.isArray(input.shardKey) ? input.shardKey : [input.shardKey];
|
|
3678
|
+
whereClauses.push(this.keyIn('workflow_status', keys, params));
|
|
3679
|
+
}
|
|
3680
|
+
if (input.parentShardKey !== undefined && input.parentShardKey !== null) {
|
|
3681
|
+
const keys = Array.isArray(input.parentShardKey) ? input.parentShardKey : [input.parentShardKey];
|
|
3682
|
+
whereClauses.push(this.keyIn('workflow_status', keys, params, 'parent_shard_key'));
|
|
3683
|
+
}
|
|
3684
|
+
if (options.keyedIDs && input.workflowIDs) {
|
|
3685
|
+
whereClauses.push(this.wfIn('workflow_status', input.workflowIDs, params));
|
|
3686
|
+
}
|
|
3075
3687
|
paramCounter = params.length + 1;
|
|
3076
3688
|
if (input.workflow_id_prefix) {
|
|
3077
3689
|
if (Array.isArray(input.workflow_id_prefix)) {
|
|
@@ -3086,7 +3698,7 @@ class SystemDatabase {
|
|
|
3086
3698
|
paramCounter++;
|
|
3087
3699
|
}
|
|
3088
3700
|
}
|
|
3089
|
-
if (input.workflowIDs) {
|
|
3701
|
+
if (input.workflowIDs && !options.keyedIDs) {
|
|
3090
3702
|
const placeholders = input.workflowIDs.map((_, i) => `$${paramCounter + i}`).join(', ');
|
|
3091
3703
|
whereClauses.push(`workflow_status.workflow_uuid IN (${placeholders})`);
|
|
3092
3704
|
params.push(...input.workflowIDs);
|
|
@@ -3168,7 +3780,7 @@ class SystemDatabase {
|
|
|
3168
3780
|
params.push(input.offset);
|
|
3169
3781
|
paramCounter++;
|
|
3170
3782
|
}
|
|
3171
|
-
const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns].join(', ');
|
|
3783
|
+
const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns, ...keyColumns].join(', ');
|
|
3172
3784
|
const query = `
|
|
3173
3785
|
SELECT ${projection}
|
|
3174
3786
|
FROM "${schemaName}".workflow_status
|
|
@@ -3180,31 +3792,24 @@ class SystemDatabase {
|
|
|
3180
3792
|
`;
|
|
3181
3793
|
// An ID-keyed read is bounded by its ID list, so it takes no cap: a status lookup must not fail its own caller.
|
|
3182
3794
|
const result = idKeyed
|
|
3183
|
-
? await this.pool.query(query, params)
|
|
3184
|
-
: await this.observabilityQuery((
|
|
3795
|
+
? await (client ?? this.pool).query(query, params)
|
|
3796
|
+
: await this.observabilityQuery((c) => c.query(query, params));
|
|
3185
3797
|
return result.rows.map(mapWorkflowStatus);
|
|
3186
3798
|
}
|
|
3187
3799
|
// ==================== Scheduling ====================
|
|
3188
3800
|
async createSchedule(schedule, client) {
|
|
3189
3801
|
const q = client ?? this.pool;
|
|
3190
3802
|
const owner = await this.#resolveRowOwner(q, 'workflow_schedules', 'schedule_name', schedule.scheduleName, schedule.applicationName, 'Schedule');
|
|
3803
|
+
const values = {
|
|
3804
|
+
...this.globalValues(),
|
|
3805
|
+
...this.scheduleValues(schedule),
|
|
3806
|
+
...scheduleRow(schedule, owner),
|
|
3807
|
+
};
|
|
3191
3808
|
try {
|
|
3192
|
-
await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
3193
|
-
(
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
schedule.scheduleName,
|
|
3197
|
-
schedule.workflowName,
|
|
3198
|
-
schedule.workflowClassName,
|
|
3199
|
-
schedule.schedule,
|
|
3200
|
-
schedule.status,
|
|
3201
|
-
schedule.context,
|
|
3202
|
-
schedule.lastFiredAt,
|
|
3203
|
-
schedule.automaticBackfill,
|
|
3204
|
-
schedule.cronTimezone,
|
|
3205
|
-
schedule.queueName,
|
|
3206
|
-
owner ?? null,
|
|
3207
|
-
]);
|
|
3809
|
+
await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
|
|
3810
|
+
VALUES (${Object.keys(values)
|
|
3811
|
+
.map((_, i) => `$${i + 1}`)
|
|
3812
|
+
.join(', ')})`, Object.values(values));
|
|
3208
3813
|
}
|
|
3209
3814
|
catch (e) {
|
|
3210
3815
|
if (isPgDatabaseError(e) && e.code === '23505') {
|
|
@@ -3251,31 +3856,37 @@ class SystemDatabase {
|
|
|
3251
3856
|
// Unset scopes to this application, as on every other observability query.
|
|
3252
3857
|
conditions.push(this.observabilityFilter('application_name', filters?.applicationName, params));
|
|
3253
3858
|
paramIdx = params.length + 1;
|
|
3859
|
+
conditions.push(this.global(undefined, params));
|
|
3254
3860
|
const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
|
|
3255
|
-
const result = await q.query(`SELECT ${
|
|
3861
|
+
const result = await q.query(`SELECT ${this.#scheduleColumns()}
|
|
3256
3862
|
FROM "${this.schemaName}".workflow_schedules${where}
|
|
3257
3863
|
ORDER BY schedule_name`, params);
|
|
3258
3864
|
return result.rows.map((row) => mapWorkflowSchedule(row));
|
|
3259
3865
|
}
|
|
3260
3866
|
async getSchedule(name, client) {
|
|
3261
3867
|
const q = client ?? this.pool;
|
|
3262
|
-
const
|
|
3868
|
+
const params = [name];
|
|
3869
|
+
const result = await q.query(`SELECT ${this.#scheduleColumns()}
|
|
3263
3870
|
FROM "${this.schemaName}".workflow_schedules
|
|
3264
|
-
WHERE schedule_name = $1`,
|
|
3871
|
+
WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
|
|
3265
3872
|
if (result.rows.length === 0)
|
|
3266
3873
|
return null;
|
|
3267
3874
|
return mapWorkflowSchedule(result.rows[0]);
|
|
3268
3875
|
}
|
|
3876
|
+
/** The columns a schedule is read back with, its workflows' shard key among them on a sharded database. */
|
|
3877
|
+
#scheduleColumns() {
|
|
3878
|
+
const keyColumn = this.scheduleKeyColumn();
|
|
3879
|
+
return keyColumn !== undefined ? `${SCHEDULE_COLUMNS}, ${keyColumn}` : SCHEDULE_COLUMNS;
|
|
3880
|
+
}
|
|
3269
3881
|
async deleteSchedule(name, client) {
|
|
3270
3882
|
const q = client ?? this.pool;
|
|
3271
|
-
|
|
3883
|
+
const params = [name];
|
|
3884
|
+
await q.query(`DELETE FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
|
|
3272
3885
|
}
|
|
3273
3886
|
async setScheduleStatus(name, status, client) {
|
|
3274
3887
|
const q = client ?? this.pool;
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
name,
|
|
3278
|
-
]);
|
|
3888
|
+
const params = [status, name];
|
|
3889
|
+
await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET status = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
|
|
3279
3890
|
}
|
|
3280
3891
|
async updateSchedule(name, updates, client) {
|
|
3281
3892
|
const q = client ?? this.pool;
|
|
@@ -3298,22 +3909,22 @@ class SystemDatabase {
|
|
|
3298
3909
|
}
|
|
3299
3910
|
if (setClauses.length === 0) {
|
|
3300
3911
|
// Nothing to change, but still surface a missing schedule as an error.
|
|
3301
|
-
const
|
|
3302
|
-
|
|
3303
|
-
]);
|
|
3912
|
+
const existingParams = [name];
|
|
3913
|
+
const existing = await q.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, existingParams)} AND schedule_name = $1`, existingParams);
|
|
3304
3914
|
if (existing.rows.length === 0) {
|
|
3305
3915
|
throw new error_1.DBOSError(`Schedule '${name}' not found`);
|
|
3306
3916
|
}
|
|
3307
3917
|
return;
|
|
3308
3918
|
}
|
|
3309
3919
|
params.push(name);
|
|
3310
|
-
const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx}`, params);
|
|
3920
|
+
const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx} AND ${this.global(undefined, params)}`, params);
|
|
3311
3921
|
if (result.rowCount === 0) {
|
|
3312
3922
|
throw new error_1.DBOSError(`Schedule '${name}' not found`);
|
|
3313
3923
|
}
|
|
3314
3924
|
}
|
|
3315
3925
|
async updateLastFiredAt(name, lastFiredAt) {
|
|
3316
|
-
|
|
3926
|
+
const params = [lastFiredAt, name];
|
|
3927
|
+
await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
|
|
3317
3928
|
}
|
|
3318
3929
|
async applySchedules(schedules) {
|
|
3319
3930
|
const client = await this.connect();
|
|
@@ -3321,12 +3932,21 @@ class SystemDatabase {
|
|
|
3321
3932
|
await client.query('BEGIN');
|
|
3322
3933
|
for (const sched of schedules) {
|
|
3323
3934
|
const owner = await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
3935
|
+
const scheduleKeys = this.scheduleValues(sched);
|
|
3936
|
+
const values = {
|
|
3937
|
+
...this.globalValues(),
|
|
3938
|
+
...scheduleKeys,
|
|
3939
|
+
...scheduleRow(sched, owner),
|
|
3940
|
+
};
|
|
3324
3941
|
// Upsert on schedule_name; on conflict, preserve schedule_id and runtime state (status, last_fired_at) and update only the declared definition fields, so an unchanged re-apply is a no-op.
|
|
3325
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
3326
|
-
(
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3942
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
|
|
3943
|
+
VALUES (${Object.keys(values)
|
|
3944
|
+
.map((_, i) => `$${i + 1}`)
|
|
3945
|
+
.join(', ')})
|
|
3946
|
+
ON CONFLICT (${this.keyConflict('schedule_name')}) DO UPDATE SET
|
|
3947
|
+
${Object.keys(scheduleKeys)
|
|
3948
|
+
.map((c) => `${c} = EXCLUDED.${c}, `)
|
|
3949
|
+
.join('')}workflow_name = EXCLUDED.workflow_name,
|
|
3330
3950
|
workflow_class_name = EXCLUDED.workflow_class_name,
|
|
3331
3951
|
schedule = EXCLUDED.schedule,
|
|
3332
3952
|
context = EXCLUDED.context,
|
|
@@ -3334,20 +3954,7 @@ class SystemDatabase {
|
|
|
3334
3954
|
cron_timezone = EXCLUDED.cron_timezone,
|
|
3335
3955
|
queue_name = EXCLUDED.queue_name,
|
|
3336
3956
|
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it took.
|
|
3337
|
-
application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`,
|
|
3338
|
-
sched.scheduleId,
|
|
3339
|
-
sched.scheduleName,
|
|
3340
|
-
sched.workflowName,
|
|
3341
|
-
sched.workflowClassName,
|
|
3342
|
-
sched.schedule,
|
|
3343
|
-
sched.status,
|
|
3344
|
-
sched.context,
|
|
3345
|
-
sched.lastFiredAt,
|
|
3346
|
-
sched.automaticBackfill,
|
|
3347
|
-
sched.cronTimezone,
|
|
3348
|
-
sched.queueName,
|
|
3349
|
-
owner ?? null,
|
|
3350
|
-
]);
|
|
3957
|
+
application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, Object.values(values));
|
|
3351
3958
|
// Read back, since the guard above is silent about why it declined to claim.
|
|
3352
3959
|
await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
3353
3960
|
}
|
|
@@ -3372,14 +3979,23 @@ class SystemDatabase {
|
|
|
3372
3979
|
try {
|
|
3373
3980
|
await client.query('BEGIN');
|
|
3374
3981
|
// Claim a pre-upgrade row in place, so the version is not recreated or retimed.
|
|
3982
|
+
const claimParams = [owner ?? null, versionName];
|
|
3375
3983
|
const claimed = await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3376
3984
|
SET application_name = $1
|
|
3377
|
-
WHERE version_name = $2 AND application_name IS NULL`,
|
|
3985
|
+
WHERE ${this.global(undefined, claimParams)} AND version_name = $2 AND application_name IS NULL`, claimParams);
|
|
3378
3986
|
if ((claimed.rowCount ?? 0) === 0) {
|
|
3987
|
+
const values = {
|
|
3988
|
+
...this.globalValues(),
|
|
3989
|
+
version_id: (0, crypto_1.randomUUID)(),
|
|
3990
|
+
version_name: versionName,
|
|
3991
|
+
application_name: owner ?? null,
|
|
3992
|
+
};
|
|
3379
3993
|
// Targetless DO NOTHING: names no arbiter, so it survives version_name's uniqueness being dropped while still absorbing a concurrent registrar.
|
|
3380
|
-
await client.query(`INSERT INTO "${this.schemaName}".application_versions (
|
|
3381
|
-
VALUES ($
|
|
3382
|
-
|
|
3994
|
+
await client.query(`INSERT INTO "${this.schemaName}".application_versions (${Object.keys(values).join(', ')})
|
|
3995
|
+
VALUES (${Object.keys(values)
|
|
3996
|
+
.map((_, i) => `$${i + 1}`)
|
|
3997
|
+
.join(', ')})
|
|
3998
|
+
ON CONFLICT DO NOTHING`, Object.values(values));
|
|
3383
3999
|
}
|
|
3384
4000
|
// Read back, since the writes above are silent about why they declined to claim.
|
|
3385
4001
|
await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
|
|
@@ -3410,7 +4026,7 @@ class SystemDatabase {
|
|
|
3410
4026
|
params.push(resolved);
|
|
3411
4027
|
await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3412
4028
|
SET version_timestamp = $1, application_name = ${resolved === undefined ? 'application_name' : '$3'}
|
|
3413
|
-
WHERE version_name = $2 AND ${scope}`, params);
|
|
4029
|
+
WHERE ${this.global(undefined, params)} AND version_name = $2 AND ${scope}`, params);
|
|
3414
4030
|
await client.query('COMMIT');
|
|
3415
4031
|
}
|
|
3416
4032
|
catch (e) {
|
|
@@ -3423,7 +4039,7 @@ class SystemDatabase {
|
|
|
3423
4039
|
}
|
|
3424
4040
|
async listApplicationVersions() {
|
|
3425
4041
|
const params = [];
|
|
3426
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
4042
|
+
const scope = `${this.global(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
3427
4043
|
const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
|
|
3428
4044
|
FROM "${this.schemaName}".application_versions
|
|
3429
4045
|
WHERE ${scope}
|
|
@@ -3435,24 +4051,18 @@ class SystemDatabase {
|
|
|
3435
4051
|
* caller acting for another one — firing its schedule — must name it.
|
|
3436
4052
|
*/
|
|
3437
4053
|
async getLatestApplicationVersion(applicationName) {
|
|
3438
|
-
const
|
|
3439
|
-
|
|
3440
|
-
const scope = this.appNameFilter('application_name', owner, params);
|
|
3441
|
-
const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
|
|
3442
|
-
FROM "${this.schemaName}".application_versions
|
|
3443
|
-
WHERE ${scope}
|
|
3444
|
-
ORDER BY version_timestamp DESC
|
|
3445
|
-
LIMIT 1`, params);
|
|
3446
|
-
if (rows.length === 0) {
|
|
4054
|
+
const latest = await this.#latestApplicationVersion(this.pool, applicationName ?? this.appName);
|
|
4055
|
+
if (latest === undefined) {
|
|
3447
4056
|
throw new error_1.DBOSInitializationError('No application versions found');
|
|
3448
4057
|
}
|
|
3449
|
-
return mapVersionInfo(
|
|
4058
|
+
return mapVersionInfo(latest);
|
|
3450
4059
|
}
|
|
3451
4060
|
// ==================== Queues ====================
|
|
3452
4061
|
async getQueue(name) {
|
|
4062
|
+
const params = [name];
|
|
3453
4063
|
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3454
4064
|
FROM "${this.schemaName}".queues
|
|
3455
|
-
WHERE name = $1`,
|
|
4065
|
+
WHERE ${this.global(undefined, params)} AND name = $1`, params);
|
|
3456
4066
|
return rows.length === 0 ? null : queueRecordFromRow(rows[0]);
|
|
3457
4067
|
}
|
|
3458
4068
|
/**
|
|
@@ -3461,14 +4071,15 @@ class SystemDatabase {
|
|
|
3461
4071
|
*/
|
|
3462
4072
|
async listQueues(applicationName) {
|
|
3463
4073
|
const params = [];
|
|
3464
|
-
const scope = this.observabilityFilter('application_name', applicationName, params)
|
|
4074
|
+
const scope = `${this.global(undefined, params)} AND ${this.observabilityFilter('application_name', applicationName, params)}`;
|
|
3465
4075
|
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3466
4076
|
FROM "${this.schemaName}".queues
|
|
3467
4077
|
WHERE ${scope}`, params);
|
|
3468
4078
|
return rows.map(queueRecordFromRow);
|
|
3469
4079
|
}
|
|
3470
4080
|
async deleteQueue(name) {
|
|
3471
|
-
|
|
4081
|
+
const params = [name];
|
|
4082
|
+
await this.pool.query(`DELETE FROM "${this.schemaName}".queues WHERE ${this.global(undefined, params)} AND name = $1`, params);
|
|
3472
4083
|
}
|
|
3473
4084
|
async updateQueue(name, fields) {
|
|
3474
4085
|
const setClauses = [];
|
|
@@ -3484,7 +4095,7 @@ class SystemDatabase {
|
|
|
3484
4095
|
setClauses.push(`"updated_at" = $${idx++}`);
|
|
3485
4096
|
params.push(Date.now());
|
|
3486
4097
|
params.push(name);
|
|
3487
|
-
await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx}`, params);
|
|
4098
|
+
await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx} AND ${this.global(undefined, params)}`, params);
|
|
3488
4099
|
}
|
|
3489
4100
|
/** Returns true iff this call inserted a new row (i.e. the queue did not
|
|
3490
4101
|
* previously exist). False if the row already existed, regardless of
|
|
@@ -3492,7 +4103,7 @@ class SystemDatabase {
|
|
|
3492
4103
|
async upsertQueue(record, updateExisting) {
|
|
3493
4104
|
const now = Date.now();
|
|
3494
4105
|
const onConflict = updateExisting
|
|
3495
|
-
? `ON CONFLICT (name) DO UPDATE SET
|
|
4106
|
+
? `ON CONFLICT (${this.keyConflict('name')}) DO UPDATE SET
|
|
3496
4107
|
concurrency = EXCLUDED.concurrency,
|
|
3497
4108
|
worker_concurrency = EXCLUDED.worker_concurrency,
|
|
3498
4109
|
rate_limit_max = EXCLUDED.rate_limit_max,
|
|
@@ -3507,21 +4118,27 @@ class SystemDatabase {
|
|
|
3507
4118
|
updated_at = EXCLUDED.updated_at,
|
|
3508
4119
|
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it just took.
|
|
3509
4120
|
application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
|
|
3510
|
-
: `ON CONFLICT (name) DO NOTHING`;
|
|
4121
|
+
: `ON CONFLICT (${this.keyConflict('name')}) DO NOTHING`;
|
|
3511
4122
|
const owner = record.applicationName ?? this.appName;
|
|
3512
4123
|
const client = await this.connect();
|
|
3513
4124
|
try {
|
|
3514
4125
|
await client.query('BEGIN');
|
|
3515
|
-
const
|
|
4126
|
+
const existedParams = [record.name];
|
|
4127
|
+
const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE ${this.global(undefined, existedParams)} AND name = $1`, existedParams);
|
|
3516
4128
|
// A name collision is a conflict in every mode: the name is the queue's address.
|
|
3517
4129
|
const resolvedOwner = await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
4130
|
+
const keys = this.globalValues();
|
|
3518
4131
|
await client.query(`INSERT INTO "${this.schemaName}".queues
|
|
3519
|
-
(
|
|
4132
|
+
(${Object.keys(keys)
|
|
4133
|
+
.map((c) => `${c}, `)
|
|
4134
|
+
.join('')}name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
|
|
3520
4135
|
priority_enabled, partition_queue, partition_concurrency, partition_worker_concurrency,
|
|
3521
4136
|
partition_rate_limit_max, partition_rate_limit_period_sec,
|
|
3522
4137
|
polling_interval_sec, updated_at, application_name)
|
|
3523
4138
|
-- priority_enabled is vestigial: every queue dispatches in priority order.
|
|
3524
|
-
VALUES ($
|
|
4139
|
+
VALUES (${Object.keys(keys)
|
|
4140
|
+
.map((_, i) => `$${14 + i}, `)
|
|
4141
|
+
.join('')}$1, $2, $3, $4, $5, TRUE, $6, $7, $8, $9, $10, $11, $12, $13)
|
|
3525
4142
|
${onConflict}`, [
|
|
3526
4143
|
record.name,
|
|
3527
4144
|
record.concurrency,
|
|
@@ -3536,6 +4153,7 @@ class SystemDatabase {
|
|
|
3536
4153
|
record.pollingIntervalSec,
|
|
3537
4154
|
now,
|
|
3538
4155
|
resolvedOwner ?? null,
|
|
4156
|
+
...Object.values(keys),
|
|
3539
4157
|
]);
|
|
3540
4158
|
// Read back, since the guard above is silent about why it declined to claim.
|
|
3541
4159
|
await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
@@ -3645,6 +4263,9 @@ class SystemDatabase {
|
|
|
3645
4263
|
params.push(statuses);
|
|
3646
4264
|
where = `${where} AND status = ANY($${params.length})`;
|
|
3647
4265
|
}
|
|
4266
|
+
else {
|
|
4267
|
+
where = `${where} AND ${this.global(undefined, params)}`;
|
|
4268
|
+
}
|
|
3648
4269
|
const res = await client.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${where}`, params);
|
|
3649
4270
|
return res.rowCount ?? 0;
|
|
3650
4271
|
};
|
|
@@ -3669,84 +4290,64 @@ class SystemDatabase {
|
|
|
3669
4290
|
}
|
|
3670
4291
|
// ==================== Internal ====================
|
|
3671
4292
|
async insertWorkflowStatus(client, initStatus, creatorXid) {
|
|
4293
|
+
const values = {
|
|
4294
|
+
...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
|
|
4295
|
+
workflow_uuid: initStatus.workflowUUID,
|
|
4296
|
+
status: initStatus.status,
|
|
4297
|
+
name: initStatus.workflowName,
|
|
4298
|
+
// For cross-language compatibility, these variables MUST be NULL in the database when not set
|
|
4299
|
+
class_name: initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
|
|
4300
|
+
config_name: initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
|
|
4301
|
+
queue_name: initStatus.queueName ?? null,
|
|
4302
|
+
authenticated_user: initStatus.authenticatedUser,
|
|
4303
|
+
assumed_role: initStatus.assumedRole,
|
|
4304
|
+
authenticated_roles: JSON.stringify(initStatus.authenticatedRoles),
|
|
4305
|
+
executor_id: initStatus.executorId,
|
|
4306
|
+
application_version: initStatus.applicationVersion ?? null,
|
|
4307
|
+
application_id: initStatus.applicationID,
|
|
4308
|
+
recovery_attempts: initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
|
|
4309
|
+
workflow_timeout_ms: initStatus.timeoutMS ?? null,
|
|
4310
|
+
workflow_deadline_epoch_ms: initStatus.deadlineEpochMS ?? null,
|
|
4311
|
+
// Legacy column: the payload lives in workflow_input.
|
|
4312
|
+
inputs: null,
|
|
4313
|
+
deduplication_id: initStatus.deduplicationID ?? null,
|
|
4314
|
+
priority: initStatus.priority,
|
|
4315
|
+
queue_partition_key: initStatus.queuePartitionKey ?? null,
|
|
4316
|
+
forked_from: initStatus.forkedFrom ?? null,
|
|
4317
|
+
parent_workflow_id: initStatus.parentWorkflowID ?? null,
|
|
4318
|
+
...this.parentValues(initStatus.parentWorkflowID),
|
|
4319
|
+
serialization: initStatus.serialization,
|
|
4320
|
+
creator_xid: creatorXid,
|
|
4321
|
+
delay_until_epoch_ms: initStatus.delayUntilEpochMS ?? null,
|
|
4322
|
+
attributes: initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
|
|
4323
|
+
schedule_name: initStatus.scheduleName ?? null,
|
|
4324
|
+
debounce_deadline_epoch_ms: initStatus.debounceDeadlineEpochMS ?? null,
|
|
4325
|
+
is_debounced: initStatus.isDebounced ?? false,
|
|
4326
|
+
application_name: initStatus.applicationName ?? null,
|
|
4327
|
+
// A direct start runs at once, so its creator token also owns the execution.
|
|
4328
|
+
owner_xid: initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
|
|
4329
|
+
};
|
|
4330
|
+
const columns = Object.keys(values);
|
|
4331
|
+
const inputValues = {
|
|
4332
|
+
...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
|
|
4333
|
+
workflow_uuid: initStatus.workflowUUID,
|
|
4334
|
+
inputs: initStatus.input ?? null,
|
|
4335
|
+
};
|
|
4336
|
+
const inputColumns = Object.keys(inputValues);
|
|
3672
4337
|
try {
|
|
3673
|
-
const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
name,
|
|
3677
|
-
class_name,
|
|
3678
|
-
config_name,
|
|
3679
|
-
queue_name,
|
|
3680
|
-
authenticated_user,
|
|
3681
|
-
assumed_role,
|
|
3682
|
-
authenticated_roles,
|
|
3683
|
-
executor_id,
|
|
3684
|
-
application_version,
|
|
3685
|
-
application_id,
|
|
3686
|
-
recovery_attempts,
|
|
3687
|
-
workflow_timeout_ms,
|
|
3688
|
-
workflow_deadline_epoch_ms,
|
|
3689
|
-
inputs,
|
|
3690
|
-
deduplication_id,
|
|
3691
|
-
priority,
|
|
3692
|
-
queue_partition_key,
|
|
3693
|
-
forked_from,
|
|
3694
|
-
parent_workflow_id,
|
|
3695
|
-
serialization,
|
|
3696
|
-
creator_xid,
|
|
3697
|
-
delay_until_epoch_ms,
|
|
3698
|
-
attributes,
|
|
3699
|
-
schedule_name,
|
|
3700
|
-
debounce_deadline_epoch_ms,
|
|
3701
|
-
is_debounced,
|
|
3702
|
-
application_name,
|
|
3703
|
-
owner_xid
|
|
3704
|
-
) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)
|
|
3705
|
-
ON CONFLICT (workflow_uuid)
|
|
4338
|
+
const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
|
|
4339
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
4340
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')})
|
|
3706
4341
|
-- A no-op update, so an existing row comes back unchanged for the caller to inspect.
|
|
3707
4342
|
DO UPDATE SET creator_xid = workflow_status.creator_xid
|
|
3708
|
-
RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`,
|
|
3709
|
-
initStatus.workflowUUID,
|
|
3710
|
-
initStatus.status,
|
|
3711
|
-
initStatus.workflowName,
|
|
3712
|
-
// For cross-language compatibility, these variables MUST be NULL in the database when not set
|
|
3713
|
-
initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
|
|
3714
|
-
initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
|
|
3715
|
-
initStatus.queueName ?? null,
|
|
3716
|
-
initStatus.authenticatedUser,
|
|
3717
|
-
initStatus.assumedRole,
|
|
3718
|
-
JSON.stringify(initStatus.authenticatedRoles),
|
|
3719
|
-
initStatus.executorId,
|
|
3720
|
-
initStatus.applicationVersion ?? null,
|
|
3721
|
-
initStatus.applicationID,
|
|
3722
|
-
initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
|
|
3723
|
-
initStatus.timeoutMS ?? null,
|
|
3724
|
-
initStatus.deadlineEpochMS ?? null,
|
|
3725
|
-
// Legacy column: the payload lives in workflow_input.
|
|
3726
|
-
null,
|
|
3727
|
-
initStatus.deduplicationID ?? null,
|
|
3728
|
-
initStatus.priority,
|
|
3729
|
-
initStatus.queuePartitionKey ?? null,
|
|
3730
|
-
initStatus.forkedFrom ?? null,
|
|
3731
|
-
initStatus.parentWorkflowID ?? null,
|
|
3732
|
-
initStatus.serialization,
|
|
3733
|
-
creatorXid,
|
|
3734
|
-
initStatus.delayUntilEpochMS ?? null,
|
|
3735
|
-
initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
|
|
3736
|
-
initStatus.scheduleName ?? null,
|
|
3737
|
-
initStatus.debounceDeadlineEpochMS ?? null,
|
|
3738
|
-
initStatus.isDebounced ?? false,
|
|
3739
|
-
initStatus.applicationName ?? null,
|
|
3740
|
-
// A direct start runs at once, so its creator token also owns the execution.
|
|
3741
|
-
initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
|
|
3742
|
-
]);
|
|
4343
|
+
RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`, Object.values(values));
|
|
3743
4344
|
if (rows.length === 0) {
|
|
3744
4345
|
throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
|
|
3745
4346
|
}
|
|
3746
4347
|
// Two statements, not a data-modifying CTE: at scale the CTE costs more than the round trip it saves.
|
|
3747
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (
|
|
3748
|
-
VALUES ($1,
|
|
3749
|
-
ON CONFLICT (workflow_uuid) DO NOTHING`,
|
|
4348
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (${inputColumns.join(', ')})
|
|
4349
|
+
VALUES (${inputColumns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
4350
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`, Object.values(inputValues));
|
|
3750
4351
|
const ret = rows[0];
|
|
3751
4352
|
ret.class_name = ret.class_name ?? '';
|
|
3752
4353
|
ret.config_name = ret.config_name ?? '';
|
|
@@ -3762,16 +4363,17 @@ class SystemDatabase {
|
|
|
3762
4363
|
}
|
|
3763
4364
|
}
|
|
3764
4365
|
async getWorkflowStatusValue(client, workflowID) {
|
|
3765
|
-
const
|
|
4366
|
+
const params = [];
|
|
4367
|
+
const { rows } = await client.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
|
|
3766
4368
|
return rows.length === 0 ? undefined : rows[0].status;
|
|
3767
4369
|
}
|
|
3768
4370
|
async updateWorkflowStatus(client, workflowID, status, options = {}) {
|
|
3769
4371
|
// Use SQL now() so updated_at and completed_at (when set together) are
|
|
3770
4372
|
// computed in the same statement against the same clock.
|
|
3771
4373
|
const nowMsExpr = `(EXTRACT(EPOCH FROM now()) * 1000)::bigint`;
|
|
3772
|
-
let setClause = `SET status=$
|
|
3773
|
-
|
|
3774
|
-
|
|
4374
|
+
let setClause = `SET status=$1, updated_at=${nowMsExpr}`;
|
|
4375
|
+
const args = [status];
|
|
4376
|
+
let whereClause = `WHERE ${this.wf(undefined, workflowID, args)}`;
|
|
3775
4377
|
const update = options.update ?? {};
|
|
3776
4378
|
if (update.resetRecoveryAttempts) {
|
|
3777
4379
|
setClause += `, recovery_attempts = 0`;
|
|
@@ -3830,27 +4432,31 @@ class SystemDatabase {
|
|
|
3830
4432
|
async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
3831
4433
|
const ownerXid = options.ownerChecked ? undefined : (0, context_1.currentOwnerXid)(workflowID);
|
|
3832
4434
|
if (ownerXid !== undefined) {
|
|
3833
|
-
await this
|
|
3834
|
-
}
|
|
4435
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
4436
|
+
}
|
|
4437
|
+
const values = {
|
|
4438
|
+
...this.wfValues(workflowID),
|
|
4439
|
+
workflow_uuid: workflowID,
|
|
4440
|
+
function_id: functionID,
|
|
4441
|
+
output: options.output ?? null,
|
|
4442
|
+
error: options.error ?? null,
|
|
4443
|
+
function_name: functionName,
|
|
4444
|
+
child_workflow_id: options.childWorkflowID ?? null,
|
|
4445
|
+
...(options.childValues ?? this.childValues(options.childWorkflowID)),
|
|
4446
|
+
started_at_epoch_ms: startTimeEpochMs,
|
|
4447
|
+
completed_at_epoch_ms: endTimeEpochMs,
|
|
4448
|
+
serialization: options.serialization ?? null,
|
|
4449
|
+
// Mirrors the parent: only the running application records its steps.
|
|
4450
|
+
application_name: this.appName ?? null,
|
|
4451
|
+
};
|
|
4452
|
+
const columns = Object.keys(values);
|
|
3835
4453
|
try {
|
|
3836
4454
|
const out = await client.query(`INSERT INTO "${this.schemaName}".operation_outputs
|
|
3837
|
-
(
|
|
3838
|
-
VALUES ($
|
|
3839
|
-
ON CONFLICT (workflow_uuid, function_id) DO UPDATE
|
|
4455
|
+
(${columns.join(', ')}, retention_timestamp)
|
|
4456
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')}, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
|
|
4457
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id')}) DO UPDATE
|
|
3840
4458
|
SET completed_at_epoch_ms = operation_outputs.completed_at_epoch_ms
|
|
3841
|
-
RETURNING completed_at_epoch_ms;`,
|
|
3842
|
-
workflowID,
|
|
3843
|
-
functionID,
|
|
3844
|
-
options.output ?? null,
|
|
3845
|
-
options.error ?? null,
|
|
3846
|
-
functionName,
|
|
3847
|
-
options.childWorkflowID ?? null,
|
|
3848
|
-
startTimeEpochMs,
|
|
3849
|
-
endTimeEpochMs,
|
|
3850
|
-
options.serialization ?? null,
|
|
3851
|
-
// Mirrors the parent: only the running application records its steps.
|
|
3852
|
-
this.appName ?? null,
|
|
3853
|
-
]);
|
|
4459
|
+
RETURNING completed_at_epoch_ms;`, Object.values(values));
|
|
3854
4460
|
// A row already there with another completion time is this execution's own doing
|
|
3855
4461
|
// (a retry carries the same time), so it is nondeterminism, not a duplicate.
|
|
3856
4462
|
if (checkConflict &&
|
|
@@ -3867,7 +4473,7 @@ class SystemDatabase {
|
|
|
3867
4473
|
}
|
|
3868
4474
|
}
|
|
3869
4475
|
/** Run `fn` in a READ COMMITTED transaction on `client`, so an ownership check holds its row lock until the write commits. */
|
|
3870
|
-
async
|
|
4476
|
+
async inTransaction(client, fn) {
|
|
3871
4477
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
3872
4478
|
let result;
|
|
3873
4479
|
try {
|
|
@@ -3882,22 +4488,26 @@ class SystemDatabase {
|
|
|
3882
4488
|
return result;
|
|
3883
4489
|
}
|
|
3884
4490
|
/** Throw DBOSWorkflowConflictError unless ownerXid still owns the workflow; the row lock holds until commit, so no hand-off lands in between. */
|
|
3885
|
-
async
|
|
3886
|
-
const
|
|
4491
|
+
async checkOwner(client, workflowID, ownerXid) {
|
|
4492
|
+
const params = [];
|
|
4493
|
+
const { rows } = await client.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)} FOR NO KEY UPDATE`, params);
|
|
3887
4494
|
if (rows[0]?.owner_xid !== ownerXid) {
|
|
3888
4495
|
throw new error_1.DBOSWorkflowConflictError(workflowID);
|
|
3889
4496
|
}
|
|
3890
4497
|
}
|
|
3891
4498
|
/** The workflow's current ownership token; null if unowned or missing. */
|
|
3892
4499
|
async getWorkflowOwner(workflowID) {
|
|
3893
|
-
const
|
|
4500
|
+
const params = [];
|
|
4501
|
+
const { rows } = await this.pool.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
|
|
3894
4502
|
return rows[0]?.owner_xid ?? null;
|
|
3895
4503
|
}
|
|
3896
4504
|
async #getOperationResultAndThrowIfCancelled(client, workflowID, functionID) {
|
|
3897
4505
|
await this.#checkIfCanceled(client, workflowID);
|
|
3898
|
-
const
|
|
4506
|
+
const childKey = this.keyColumn(undefined, 'child_shard_key');
|
|
4507
|
+
const params = [functionID];
|
|
4508
|
+
const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name, serialization${childKey !== undefined ? `, ${childKey}` : ''}
|
|
3899
4509
|
FROM "${this.schemaName}".operation_outputs
|
|
3900
|
-
WHERE
|
|
4510
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
|
|
3901
4511
|
if (rows.length === 0) {
|
|
3902
4512
|
return undefined;
|
|
3903
4513
|
}
|
|
@@ -3906,6 +4516,7 @@ class SystemDatabase {
|
|
|
3906
4516
|
output: rows[0].output,
|
|
3907
4517
|
error: rows[0].error,
|
|
3908
4518
|
childWorkflowID: rows[0].child_workflow_id,
|
|
4519
|
+
childShardKey: rows[0].child_shard_key ?? undefined,
|
|
3909
4520
|
functionName: rows[0].function_name,
|
|
3910
4521
|
// Return serialization so recv/getEvent replay deserializes with the stored format, not the default.
|
|
3911
4522
|
serialization: rows[0].serialization,
|
|
@@ -3943,7 +4554,11 @@ class SystemDatabase {
|
|
|
3943
4554
|
// been cancelled.
|
|
3944
4555
|
// For an actual sleep, completed_at is the wake deadline so the step's duration reflects the
|
|
3945
4556
|
// sleep; a timeout marker records zero duration since its deadline may never be reached.
|
|
3946
|
-
async
|
|
4557
|
+
async durableSleepInternal(workflowID, functionID, durationMS, recordCompletionAtDeadline = false) {
|
|
4558
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
4559
|
+
return await this.ownKey(workflowID, undefined, () => this.#durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline));
|
|
4560
|
+
}
|
|
4561
|
+
async #durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline) {
|
|
3947
4562
|
const startTimeMs = Date.now();
|
|
3948
4563
|
// Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
|
|
3949
4564
|
const endTimeMs = startTimeMs + Math.ceil(durationMS);
|
|
@@ -3956,7 +4571,7 @@ class SystemDatabase {
|
|
|
3956
4571
|
}
|
|
3957
4572
|
return JSON.parse(res.output);
|
|
3958
4573
|
}
|
|
3959
|
-
await this
|
|
4574
|
+
await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
|
|
3960
4575
|
output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
|
|
3961
4576
|
serialization: serialization_1.DBOSPortableJSON.name(),
|
|
3962
4577
|
}));
|
|
@@ -4136,7 +4751,7 @@ __decorate([
|
|
|
4136
4751
|
__decorate([
|
|
4137
4752
|
dbRetry(),
|
|
4138
4753
|
__metadata("design:type", Function),
|
|
4139
|
-
__metadata("design:paramtypes", [String, Number]),
|
|
4754
|
+
__metadata("design:paramtypes", [String, Number, String]),
|
|
4140
4755
|
__metadata("design:returntype", Promise)
|
|
4141
4756
|
], SystemDatabase.prototype, "getOperationResultAndThrowIfCancelled", null);
|
|
4142
4757
|
__decorate([
|
|
@@ -4196,9 +4811,9 @@ __decorate([
|
|
|
4196
4811
|
__decorate([
|
|
4197
4812
|
dbRetry(),
|
|
4198
4813
|
__metadata("design:type", Function),
|
|
4199
|
-
__metadata("design:paramtypes", [Array
|
|
4814
|
+
__metadata("design:paramtypes", [Array]),
|
|
4200
4815
|
__metadata("design:returntype", Promise)
|
|
4201
|
-
], SystemDatabase.prototype, "
|
|
4816
|
+
], SystemDatabase.prototype, "deleteWorkflowsInternal", null);
|
|
4202
4817
|
__decorate([
|
|
4203
4818
|
dbRetry()
|
|
4204
4819
|
// A missing row normally means the workflow has not been inserted yet, so
|
|
@@ -4232,19 +4847,19 @@ __decorate([
|
|
|
4232
4847
|
__decorate([
|
|
4233
4848
|
dbRetry(),
|
|
4234
4849
|
__metadata("design:type", Function),
|
|
4235
|
-
__metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String]),
|
|
4850
|
+
__metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String, String]),
|
|
4236
4851
|
__metadata("design:returntype", Promise)
|
|
4237
4852
|
], SystemDatabase.prototype, "send", null);
|
|
4238
4853
|
__decorate([
|
|
4239
4854
|
dbRetry(),
|
|
4240
4855
|
__metadata("design:type", Function),
|
|
4241
|
-
__metadata("design:paramtypes", [String, String, Object, Object, Object, String]),
|
|
4856
|
+
__metadata("design:paramtypes", [String, String, Object, Object, Object, String, String]),
|
|
4242
4857
|
__metadata("design:returntype", Promise)
|
|
4243
4858
|
], SystemDatabase.prototype, "sendFromStep", null);
|
|
4244
4859
|
__decorate([
|
|
4245
4860
|
dbRetry(),
|
|
4246
4861
|
__metadata("design:type", Function),
|
|
4247
|
-
__metadata("design:paramtypes", [String, Object, Object, Object, String]),
|
|
4862
|
+
__metadata("design:paramtypes", [String, Object, Object, Object, String, String]),
|
|
4248
4863
|
__metadata("design:returntype", Promise)
|
|
4249
4864
|
], SystemDatabase.prototype, "sendDirectStandalone", null);
|
|
4250
4865
|
__decorate([
|
|
@@ -4298,7 +4913,7 @@ __decorate([
|
|
|
4298
4913
|
__decorate([
|
|
4299
4914
|
dbRetry(),
|
|
4300
4915
|
__metadata("design:type", Function),
|
|
4301
|
-
__metadata("design:paramtypes", [Object]),
|
|
4916
|
+
__metadata("design:paramtypes", [Object, Object]),
|
|
4302
4917
|
__metadata("design:returntype", Promise)
|
|
4303
4918
|
], SystemDatabase.prototype, "listWorkflows", null);
|
|
4304
4919
|
__decorate([
|