@dbos-inc/dbos-sdk 5.4.10-preview → 5.5.1-preview
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli/cli.js +56 -13
- package/dist/src/cli/cli.js.map +1 -1
- package/dist/src/client.d.ts +20 -10
- package/dist/src/client.d.ts.map +1 -1
- package/dist/src/client.js +76 -44
- package/dist/src/client.js.map +1 -1
- package/dist/src/config.d.ts +1 -0
- package/dist/src/config.d.ts.map +1 -1
- package/dist/src/config.js +1 -0
- package/dist/src/config.js.map +1 -1
- package/dist/src/context.d.ts +6 -0
- package/dist/src/context.d.ts.map +1 -1
- package/dist/src/context.js +15 -1
- package/dist/src/context.js.map +1 -1
- package/dist/src/datasource.d.ts +2 -0
- package/dist/src/datasource.d.ts.map +1 -1
- package/dist/src/datasource.js +7 -1
- package/dist/src/datasource.js.map +1 -1
- package/dist/src/dbos-executor.d.ts +7 -4
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js +56 -30
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts +30 -10
- package/dist/src/dbos.d.ts.map +1 -1
- package/dist/src/dbos.js +136 -64
- package/dist/src/dbos.js.map +1 -1
- package/dist/src/debugpoint.d.ts +1 -0
- package/dist/src/debugpoint.d.ts.map +1 -1
- package/dist/src/debugpoint.js +4 -2
- package/dist/src/debugpoint.js.map +1 -1
- package/dist/src/enqueue_options.d.ts +2 -0
- package/dist/src/enqueue_options.d.ts.map +1 -1
- package/dist/src/enqueue_options.js +3 -0
- package/dist/src/enqueue_options.js.map +1 -1
- package/dist/src/enqueue_workflow.d.ts.map +1 -1
- package/dist/src/enqueue_workflow.js +11 -5
- package/dist/src/enqueue_workflow.js.map +1 -1
- package/dist/src/enterprise.d.ts +23 -3
- package/dist/src/enterprise.d.ts.map +1 -1
- package/dist/src/enterprise.js +81 -44
- package/dist/src/enterprise.js.map +1 -1
- package/dist/src/eventreceiver.d.ts +2 -0
- package/dist/src/eventreceiver.d.ts.map +1 -1
- package/dist/src/eventreceiver.js +7 -1
- package/dist/src/eventreceiver.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/scheduler/scheduler.d.ts +5 -0
- package/dist/src/scheduler/scheduler.d.ts.map +1 -1
- package/dist/src/scheduler/scheduler.js +14 -4
- package/dist/src/scheduler/scheduler.js.map +1 -1
- package/dist/src/shard_keys.d.ts +13 -0
- package/dist/src/shard_keys.d.ts.map +1 -0
- package/dist/src/shard_keys.js +55 -0
- package/dist/src/shard_keys.js.map +1 -0
- package/dist/src/streams.d.ts +2 -0
- package/dist/src/streams.d.ts.map +1 -1
- package/dist/src/streams.js +5 -2
- package/dist/src/streams.js.map +1 -1
- package/dist/src/system_database.d.ts +150 -10
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +1250 -589
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/wfqueue.d.ts +1 -2
- package/dist/src/wfqueue.d.ts.map +1 -1
- package/dist/src/wfqueue.js +83 -63
- package/dist/src/wfqueue.js.map +1 -1
- package/dist/src/workflow.d.ts +12 -2
- package/dist/src/workflow.d.ts.map +1 -1
- package/dist/src/workflow.js +14 -9
- package/dist/src/workflow.js.map +1 -1
- package/dist/src/workflow_management.d.ts +5 -3
- package/dist/src/workflow_management.d.ts.map +1 -1
- package/dist/src/workflow_management.js +45 -14
- package/dist/src/workflow_management.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
|
@@ -1,15 +1,38 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
2
18
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
19
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
20
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
21
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
22
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
23
|
};
|
|
24
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
25
|
+
if (mod && mod.__esModule) return mod;
|
|
26
|
+
var result = {};
|
|
27
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
28
|
+
__setModuleDefault(result, mod);
|
|
29
|
+
return result;
|
|
30
|
+
};
|
|
8
31
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
32
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
33
|
};
|
|
11
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.SystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
35
|
+
exports.SystemDatabase = exports.createSystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.qualify = exports.holdsShardedSchema = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
13
36
|
const dbos_executor_1 = require("./dbos-executor");
|
|
14
37
|
const pg_1 = require("pg");
|
|
15
38
|
const error_1 = require("./error");
|
|
@@ -18,6 +41,8 @@ const utils_1 = require("./utils");
|
|
|
18
41
|
const context_1 = require("./context");
|
|
19
42
|
const async_hooks_1 = require("async_hooks");
|
|
20
43
|
const crypto_1 = require("crypto");
|
|
44
|
+
const shard_keys_1 = require("./shard_keys");
|
|
45
|
+
const enterprise = __importStar(require("./enterprise"));
|
|
21
46
|
const utils_2 = require("./utils");
|
|
22
47
|
const database_utils_1 = require("./database_utils");
|
|
23
48
|
const migration_runner_1 = require("./sysdb_migrations/migration_runner");
|
|
@@ -70,6 +95,8 @@ function validateObservabilityQueryTimeoutMs(value) {
|
|
|
70
95
|
exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutMs;
|
|
71
96
|
// Bounds how long a frozen or unreachable client can hold system database locks.
|
|
72
97
|
exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = 60_000;
|
|
98
|
+
// How long a background job's lock may sit idle before Postgres ends its holder's session.
|
|
99
|
+
exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = 5000;
|
|
73
100
|
/** The timeout is rendered into a `SET idle_in_transaction_session_timeout`, which takes only an in-range integer. */
|
|
74
101
|
function validateIdleTransactionTimeoutMs(value) {
|
|
75
102
|
if (value !== undefined && (!Number.isFinite(value) || value > MAX_STATEMENT_TIMEOUT_MS)) {
|
|
@@ -157,6 +184,7 @@ async function grantDbosSchemaPermissions(databaseUrl, roleName, logger, schemaN
|
|
|
157
184
|
await client.end();
|
|
158
185
|
}
|
|
159
186
|
}
|
|
187
|
+
exports.grantDbosSchemaPermissions = grantDbosSchemaPermissions;
|
|
160
188
|
/**
|
|
161
189
|
* Check out a connection carrying our own error handler, so a socket death while we hold it is
|
|
162
190
|
* logged instead of thrown at an emitter with no listener. `release()` takes the handler back off,
|
|
@@ -219,6 +247,7 @@ async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = '
|
|
|
219
247
|
}
|
|
220
248
|
const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
|
|
221
249
|
try {
|
|
250
|
+
await refuseShardedSchema(client, schemaName);
|
|
222
251
|
const isCockroach = await isCockroachDB(client);
|
|
223
252
|
await (0, migration_runner_1.runSysMigrationsPg)(client, (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }), schemaName, {
|
|
224
253
|
onWarn: (e) => logger.info(e),
|
|
@@ -242,6 +271,7 @@ exports.migrateSystemDatabase = migrateSystemDatabase;
|
|
|
242
271
|
async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
|
|
243
272
|
const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
|
|
244
273
|
try {
|
|
274
|
+
await refuseShardedSchema(client, schemaName);
|
|
245
275
|
const isCockroach = await isCockroachDB(client);
|
|
246
276
|
const requiredVersion = (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }).length;
|
|
247
277
|
const currentVersion = await (0, migration_runner_1.getCurrentSysDBVersion)(client, schemaName);
|
|
@@ -259,6 +289,41 @@ async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = '
|
|
|
259
289
|
}
|
|
260
290
|
}
|
|
261
291
|
exports.verifySystemDatabase = verifySystemDatabase;
|
|
292
|
+
/** Throw if this schema is sharded: an unsharded handle's statements would span every tenant's rows. */
|
|
293
|
+
async function refuseShardedSchema(client, schemaName) {
|
|
294
|
+
const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
|
|
295
|
+
WHERE table_schema = $1 AND table_name = 'workflow_status' AND column_name = 'shard_key'`, [schemaName]);
|
|
296
|
+
if (rows.length > 0) {
|
|
297
|
+
throw new error_1.DBOSError(`The DBOS system schema ${schemaName} is sharded. Enable sharding to use it: ` +
|
|
298
|
+
'set sharding in your DBOS configuration, or pass sharding: true to DBOSClient.');
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
/** Whether this database holds a sharded DBOS system schema (this one, if named); false if it can't be reached. */
|
|
302
|
+
async function holdsShardedSchema(databaseUrl, schemaName) {
|
|
303
|
+
let client;
|
|
304
|
+
try {
|
|
305
|
+
// Bounded, so an unreachable host fails as fast as the caller's own connection would.
|
|
306
|
+
client = new pg_1.Client({ ...(0, utils_2.getClientConfig)(databaseUrl), connectionTimeoutMillis: 10_000 });
|
|
307
|
+
}
|
|
308
|
+
catch {
|
|
309
|
+
// A malformed URL, for the caller's own connection to report.
|
|
310
|
+
return false;
|
|
311
|
+
}
|
|
312
|
+
client.on('error', () => { });
|
|
313
|
+
try {
|
|
314
|
+
await client.connect();
|
|
315
|
+
const { rows } = await client.query(`SELECT 1 FROM information_schema.columns
|
|
316
|
+
WHERE table_name = 'workflow_status' AND column_name = 'shard_key'${schemaName !== undefined ? ' AND table_schema = $1' : ''} LIMIT 1`, schemaName !== undefined ? [schemaName] : []);
|
|
317
|
+
return rows.length > 0;
|
|
318
|
+
}
|
|
319
|
+
catch {
|
|
320
|
+
return false;
|
|
321
|
+
}
|
|
322
|
+
finally {
|
|
323
|
+
await client.end().catch(() => { });
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
exports.holdsShardedSchema = holdsShardedSchema;
|
|
262
327
|
class NotificationMap {
|
|
263
328
|
map = new Map();
|
|
264
329
|
curCK = 0;
|
|
@@ -300,6 +365,23 @@ function mapVersionInfo(row) {
|
|
|
300
365
|
};
|
|
301
366
|
}
|
|
302
367
|
const SCHEDULE_COLUMNS = 'schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name';
|
|
368
|
+
/** A schedule's own columns, as a new row stores them. */
|
|
369
|
+
function scheduleRow(schedule, owner) {
|
|
370
|
+
return {
|
|
371
|
+
schedule_id: schedule.scheduleId,
|
|
372
|
+
schedule_name: schedule.scheduleName,
|
|
373
|
+
workflow_name: schedule.workflowName,
|
|
374
|
+
workflow_class_name: schedule.workflowClassName,
|
|
375
|
+
schedule: schedule.schedule,
|
|
376
|
+
status: schedule.status,
|
|
377
|
+
context: schedule.context,
|
|
378
|
+
last_fired_at: schedule.lastFiredAt,
|
|
379
|
+
automatic_backfill: schedule.automaticBackfill,
|
|
380
|
+
cron_timezone: schedule.cronTimezone,
|
|
381
|
+
queue_name: schedule.queueName,
|
|
382
|
+
application_name: owner ?? null,
|
|
383
|
+
};
|
|
384
|
+
}
|
|
303
385
|
function mapWorkflowSchedule(row) {
|
|
304
386
|
return {
|
|
305
387
|
scheduleId: row.schedule_id,
|
|
@@ -314,6 +396,7 @@ function mapWorkflowSchedule(row) {
|
|
|
314
396
|
cronTimezone: row.cron_timezone ?? null,
|
|
315
397
|
queueName: row.queue_name ?? null,
|
|
316
398
|
applicationName: row.application_name ?? undefined,
|
|
399
|
+
shardKey: row.workflow_shard_key ?? undefined,
|
|
317
400
|
};
|
|
318
401
|
}
|
|
319
402
|
function mapWorkflowStatus(row) {
|
|
@@ -356,8 +439,16 @@ function mapWorkflowStatus(row) {
|
|
|
356
439
|
debounceDeadlineEpochMS: row.debounce_deadline_epoch_ms ? Number(row.debounce_deadline_epoch_ms) : undefined,
|
|
357
440
|
isDebounced: row.is_debounced ?? false,
|
|
358
441
|
applicationName: row.application_name ?? undefined,
|
|
442
|
+
// Read back only from a sharded system database, whose rows carry them.
|
|
443
|
+
shardKey: row.shard_key ?? undefined,
|
|
444
|
+
parentShardKey: row.parent_shard_key ?? undefined,
|
|
359
445
|
};
|
|
360
446
|
}
|
|
447
|
+
/** `column`, qualified by `alias` if one is given. */
|
|
448
|
+
function qualify(alias, column) {
|
|
449
|
+
return alias ? `${alias}.${column}` : column;
|
|
450
|
+
}
|
|
451
|
+
exports.qualify = qualify;
|
|
361
452
|
// SQLSTATE classes/codes that are generally safe to retry
|
|
362
453
|
// https://www.postgresql.org/docs/current/errcodes-appendix.html
|
|
363
454
|
const RETRY_SQLSTATE_PREFIXES = new Set([
|
|
@@ -527,6 +618,26 @@ function dbRetry(options = {}) {
|
|
|
527
618
|
return descriptor;
|
|
528
619
|
};
|
|
529
620
|
}
|
|
621
|
+
/** Build the system database these options describe. */
|
|
622
|
+
function createSystemDatabase(options) {
|
|
623
|
+
if (options.sharding) {
|
|
624
|
+
const { ShardedSystemDatabase } = enterprise.load(enterprise.SHARDING.description);
|
|
625
|
+
return new ShardedSystemDatabase({
|
|
626
|
+
systemDatabaseUrl: options.systemDatabaseUrl,
|
|
627
|
+
logger: options.logger,
|
|
628
|
+
serializer: options.serializer,
|
|
629
|
+
sysDbPoolSize: options.sysDbPoolSize,
|
|
630
|
+
systemDatabasePool: options.systemDatabasePool,
|
|
631
|
+
schemaName: options.schemaName,
|
|
632
|
+
pollingConcurrency: options.pollingConcurrency,
|
|
633
|
+
appName: options.appName,
|
|
634
|
+
observabilityQueryTimeoutMs: options.observabilityQueryTimeoutMs,
|
|
635
|
+
idleTransactionTimeoutMs: options.idleTransactionTimeoutMs,
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
return new SystemDatabase(options.systemDatabaseUrl, options.logger, options.serializer, options.sysDbPoolSize, options.systemDatabasePool, options.schemaName, options.useListenNotify, options.pollingConcurrency, options.notificationCoalesceMs, options.appName, options.observabilityQueryTimeoutMs, options.idleTransactionTimeoutMs);
|
|
639
|
+
}
|
|
640
|
+
exports.createSystemDatabase = createSystemDatabase;
|
|
530
641
|
/**
|
|
531
642
|
* General notes:
|
|
532
643
|
* The responsibilities of the `SystemDatabase` are to store data for workflows, and
|
|
@@ -595,6 +706,7 @@ class SystemDatabase {
|
|
|
595
706
|
pollLimiter;
|
|
596
707
|
/** Executions running in this process, by workflow ID; a re-dispatch runs alongside its stale execution, which parks at its next write. */
|
|
597
708
|
runningWorkflowMap = new Map();
|
|
709
|
+
// Keyed by waiter key, as one ID can live under several shard keys.
|
|
598
710
|
#cancelWatchers = new Map();
|
|
599
711
|
#cancelPollerLoop = undefined;
|
|
600
712
|
#cancelPollerWake = null;
|
|
@@ -727,6 +839,209 @@ class SystemDatabase {
|
|
|
727
839
|
observabilityFilter(column, value, params) {
|
|
728
840
|
return this.appNameFilter(column, value ?? this.appName, params);
|
|
729
841
|
}
|
|
842
|
+
// ==================== Shard Keys ====================
|
|
843
|
+
// Statements locate rows through these, so a sharded subclass adds its key; each appends to `params` and returns SQL.
|
|
844
|
+
/** Whether this handle routes every statement by shard key. */
|
|
845
|
+
get sharded() {
|
|
846
|
+
return false;
|
|
847
|
+
}
|
|
848
|
+
#rejectShardKey(workflowID, shardKey) {
|
|
849
|
+
if ((shardKey !== undefined && shardKey !== null) || (0, shard_keys_1.boundShardKey)(workflowID) !== undefined) {
|
|
850
|
+
throw new error_1.DBOSError('A shard key was given, but this system database is not sharded');
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
/** Predicate selecting one workflow's rows: those whose `column` is `workflowID`. */
|
|
854
|
+
wf(alias, workflowID, params, column = 'workflow_uuid', shardKey) {
|
|
855
|
+
this.#rejectShardKey(workflowID, shardKey);
|
|
856
|
+
params.push(workflowID);
|
|
857
|
+
return `${qualify(alias, column)} = $${params.length}`;
|
|
858
|
+
}
|
|
859
|
+
/** Predicate selecting several workflows' rows. */
|
|
860
|
+
wfIn(alias, workflowIDs, params) {
|
|
861
|
+
for (const workflowID of workflowIDs) {
|
|
862
|
+
this.#rejectShardKey(workflowID, undefined);
|
|
863
|
+
}
|
|
864
|
+
params.push(workflowIDs);
|
|
865
|
+
return `${qualify(alias, 'workflow_uuid')} = ANY($${params.length}::text[])`;
|
|
866
|
+
}
|
|
867
|
+
/** Condition joining two tables' rows of the same workflow. */
|
|
868
|
+
wfJoin(left, right) {
|
|
869
|
+
return `${left}.workflow_uuid = ${right}.workflow_uuid`;
|
|
870
|
+
}
|
|
871
|
+
/** Extra column values for a new row of this workflow. */
|
|
872
|
+
wfValues(workflowID, shardKey) {
|
|
873
|
+
this.#rejectShardKey(workflowID, shardKey);
|
|
874
|
+
return {};
|
|
875
|
+
}
|
|
876
|
+
/** Extra column values storing the shard key of a new workflow's parent. */
|
|
877
|
+
parentValues(_parentWorkflowID) {
|
|
878
|
+
return {};
|
|
879
|
+
}
|
|
880
|
+
/** Extra column values storing the shard key of the workflow a step refers to. */
|
|
881
|
+
childValues(_childWorkflowID) {
|
|
882
|
+
return {};
|
|
883
|
+
}
|
|
884
|
+
/** Extra (column, expression) pairs for an INSERT ... SELECT of this workflow's rows. */
|
|
885
|
+
wfColumns(_workflowID, _params) {
|
|
886
|
+
return [];
|
|
887
|
+
}
|
|
888
|
+
/** Conflict target for a unique key of a workflow table. */
|
|
889
|
+
keyConflict(...columns) {
|
|
890
|
+
return columns.join(', ');
|
|
891
|
+
}
|
|
892
|
+
/** Insert rows into `table` in one statement, returning the RETURNING rows; a sharded subclass inserts row by row. */
|
|
893
|
+
async insertRows(client, table, columns, rows, suffix = '') {
|
|
894
|
+
if (rows.length === 0)
|
|
895
|
+
return [];
|
|
896
|
+
const params = [];
|
|
897
|
+
const tuples = rows.map((row) => `(${row.map((value) => `$${params.push(value)}`).join(', ')})`);
|
|
898
|
+
const { rows: returned } = await client.query(`INSERT INTO "${this.schemaName}".${table} (${columns.join(', ')}) VALUES ${tuples.join(', ')} ${suffix}`, params);
|
|
899
|
+
return returned;
|
|
900
|
+
}
|
|
901
|
+
/** Predicate a queue claim must also meet, evaluated once before it scans; sharded, it caps claimers per key. */
|
|
902
|
+
claimGate(_queueName, _partitionKey, _params) {
|
|
903
|
+
return 'TRUE';
|
|
904
|
+
}
|
|
905
|
+
/** Statement whose `acquired` says whether a claim took its budget scope's transaction lock; no row means nothing to claim. */
|
|
906
|
+
claimLockQuery(queueName, partitionKey, params) {
|
|
907
|
+
const name = JSON.stringify(['dbos.queue_claim', this.schemaName, queueName, partitionKey ?? null]);
|
|
908
|
+
params.push(advisoryLockKey(name).toString());
|
|
909
|
+
return `SELECT pg_try_advisory_xact_lock($${params.length}) AS acquired`;
|
|
910
|
+
}
|
|
911
|
+
/** Statement whose `held` lists which of `partitionKeys` a sweep's transaction locked; no row means nothing to claim. */
|
|
912
|
+
partitionLockQuery(queueName, partitionKeys, params) {
|
|
913
|
+
const name = JSON.stringify(['dbos.partition_claim', this.schemaName, queueName]);
|
|
914
|
+
// Two-int keys: the queue, then the partition, hashed so every worker computes the same lock.
|
|
915
|
+
const lockIDs = partitionKeys.map((pk) => (0, crypto_1.createHash)('sha256').update(pk).digest().readInt32BE(0));
|
|
916
|
+
params.push(partitionKeys, lockIDs, Number(BigInt.asIntN(32, advisoryLockKey(name))));
|
|
917
|
+
const n = params.length;
|
|
918
|
+
return `SELECT coalesce(array_agg(p.pk), '{}') AS held
|
|
919
|
+
FROM unnest($${n - 2}::text[], $${n - 1}::int4[]) AS p(pk, lock_id)
|
|
920
|
+
WHERE pg_try_advisory_xact_lock($${n}::int4, p.lock_id)`;
|
|
921
|
+
}
|
|
922
|
+
/** Predicate confining a statement that names no single workflow to the current shard. */
|
|
923
|
+
scoped(_alias, _params) {
|
|
924
|
+
return 'TRUE';
|
|
925
|
+
}
|
|
926
|
+
/** Extra (column, expression) pairs for an INSERT ... SELECT confined to the current shard. */
|
|
927
|
+
scopedColumns(_params) {
|
|
928
|
+
return [];
|
|
929
|
+
}
|
|
930
|
+
/** Predicate selecting a global table's rows (queues, schedules, versions). */
|
|
931
|
+
global(_alias, _params) {
|
|
932
|
+
return 'TRUE';
|
|
933
|
+
}
|
|
934
|
+
/** Extra column values for a new row of a global table. */
|
|
935
|
+
globalValues() {
|
|
936
|
+
return {};
|
|
937
|
+
}
|
|
938
|
+
/** Predicate selecting rows whose `column` holds one of these shard keys. */
|
|
939
|
+
keyIn(_alias, _shardKeys, _params, _column = 'shard_key') {
|
|
940
|
+
throw new error_1.DBOSError('Filtering by shard key requires a sharded system database');
|
|
941
|
+
}
|
|
942
|
+
/** A shard key column to read back with a row, if this handle has one. */
|
|
943
|
+
keyColumn(_alias, _column = 'shard_key') {
|
|
944
|
+
return undefined;
|
|
945
|
+
}
|
|
946
|
+
/** Extra column values storing the shard key a schedule's workflows run under. */
|
|
947
|
+
scheduleValues(schedule) {
|
|
948
|
+
if (schedule.shardKey !== undefined && schedule.shardKey !== null) {
|
|
949
|
+
throw new error_1.DBOSError('Scheduling under a shard key requires a sharded system database');
|
|
950
|
+
}
|
|
951
|
+
return {};
|
|
952
|
+
}
|
|
953
|
+
/** The column holding the shard key a schedule's workflows run under, if any. */
|
|
954
|
+
scheduleKeyColumn() {
|
|
955
|
+
return undefined;
|
|
956
|
+
}
|
|
957
|
+
/** This workflow's shard key, resolved now for a statement or waiter that needs it later. */
|
|
958
|
+
keyHint(workflowID) {
|
|
959
|
+
this.#rejectShardKey(workflowID, undefined);
|
|
960
|
+
return undefined;
|
|
961
|
+
}
|
|
962
|
+
/** The notification-map key for a workflow's topic or key, distinct per shard key as IDs repeat across keys. */
|
|
963
|
+
waiter(workflowID, name) {
|
|
964
|
+
const key = this.keyHint(workflowID);
|
|
965
|
+
const payload = `${workflowID}::${name}`;
|
|
966
|
+
return key === undefined ? payload : `${[...key].length}:${key}:${payload}`;
|
|
967
|
+
}
|
|
968
|
+
/** Bind a workflow to `key`, else to the running workflow's own key even where a same-ID target is bound. */
|
|
969
|
+
ownKey(workflowID, key, callback) {
|
|
970
|
+
if (key === undefined) {
|
|
971
|
+
const ctx = (0, context_1.getCurrentContextStore)();
|
|
972
|
+
if (ctx !== undefined && ctx.workflowId === workflowID) {
|
|
973
|
+
key = ctx.shardKey;
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
return (0, shard_keys_1.bindShardKeys)([[workflowID, key]], callback);
|
|
977
|
+
}
|
|
978
|
+
/** Run `callback` under this application's lock for a background job, passing whether it was taken. */
|
|
979
|
+
async backgroundJobLock(jobName, callback) {
|
|
980
|
+
// CockroachDB grants every advisory lock, so each process runs every job.
|
|
981
|
+
if (await this.#isCockroach()) {
|
|
982
|
+
return await callback(true);
|
|
983
|
+
}
|
|
984
|
+
// Leading 8 bytes of SHA-256, big-endian signed, as for the retention lock.
|
|
985
|
+
const key = advisoryLockKey(`dbos.background_job.${this.schemaName}.${this.appName ?? ''}.${jobName}`);
|
|
986
|
+
return await this.lockedBackgroundJob(jobName, `SELECT pg_try_advisory_xact_lock($1) AS acquired FROM set_config('idle_in_transaction_session_timeout', $2, true)`, [key.toString(), String(exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS)], callback);
|
|
987
|
+
}
|
|
988
|
+
/** Take a transaction-scoped background job lock with `lockQuery`, which returns `acquired`, and run `callback`. */
|
|
989
|
+
async lockedBackgroundJob(jobName, lockQuery, lockParams, callback) {
|
|
990
|
+
const client = await this.connect();
|
|
991
|
+
try {
|
|
992
|
+
await client.query('BEGIN');
|
|
993
|
+
// The local idle timeout ends a frozen holder's session, releasing its lock.
|
|
994
|
+
const { rows } = await client.query(lockQuery, lockParams);
|
|
995
|
+
return await callback(rows[0]?.acquired === true);
|
|
996
|
+
}
|
|
997
|
+
finally {
|
|
998
|
+
try {
|
|
999
|
+
await client.query('ROLLBACK');
|
|
1000
|
+
client.release();
|
|
1001
|
+
}
|
|
1002
|
+
catch (e) {
|
|
1003
|
+
// A session the idle timeout ended has released the lock already.
|
|
1004
|
+
this.logger.debug(`Could not release the ${jobName} lock: ${String(e)}`);
|
|
1005
|
+
client.release(e instanceof Error ? e : true);
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
#cockroach = undefined;
|
|
1010
|
+
/** Whether this handle's database is CockroachDB, read on first use. */
|
|
1011
|
+
#isCockroach() {
|
|
1012
|
+
this.#cockroach ??= (async () => {
|
|
1013
|
+
const client = await this.connect();
|
|
1014
|
+
try {
|
|
1015
|
+
return await isCockroachDB(client);
|
|
1016
|
+
}
|
|
1017
|
+
finally {
|
|
1018
|
+
client.release();
|
|
1019
|
+
}
|
|
1020
|
+
})();
|
|
1021
|
+
// A failed read is retried on next use, rather than cached.
|
|
1022
|
+
this.#cockroach.catch(() => {
|
|
1023
|
+
this.#cockroach = undefined;
|
|
1024
|
+
});
|
|
1025
|
+
return this.#cockroach;
|
|
1026
|
+
}
|
|
1027
|
+
/** Scopes a queue's workers dequeue from in turn; one unkeyed scope unless sharded. */
|
|
1028
|
+
dequeueScopes(_queueName) {
|
|
1029
|
+
return Promise.resolve([undefined]);
|
|
1030
|
+
}
|
|
1031
|
+
/** Whether a dequeue failed because another worker holds the rows it needs. */
|
|
1032
|
+
isLockConflict(e) {
|
|
1033
|
+
return isPgDatabaseError(e) && e.code === '55P03';
|
|
1034
|
+
}
|
|
1035
|
+
/** Whether a dequeue failed on a serialization conflict with a concurrent one. */
|
|
1036
|
+
isSerializationConflict(e) {
|
|
1037
|
+
return isPgDatabaseError(e) && e.code === '40001';
|
|
1038
|
+
}
|
|
1039
|
+
/** Whether `appVersion` is this application's latest registered version, or none is registered. */
|
|
1040
|
+
async isLatestVersion(appVersion, client) {
|
|
1041
|
+
// Own plus unclaimed: a named peer's deploy must not demote this one.
|
|
1042
|
+
const latest = await this.#latestApplicationVersionName(client ?? this.pool);
|
|
1043
|
+
return latest === undefined || latest === appVersion;
|
|
1044
|
+
}
|
|
730
1045
|
/**
|
|
731
1046
|
* The version name a dequeue treats as latest: this application's own plus unclaimed
|
|
732
1047
|
* ones, so a named peer's deploy does not demote this one.
|
|
@@ -737,15 +1052,17 @@ class SystemDatabase {
|
|
|
737
1052
|
async #latestApplicationVersion(client, applicationName) {
|
|
738
1053
|
const columns = 'version_id, version_name, version_timestamp, created_at, application_name';
|
|
739
1054
|
const table = `"${this.schemaName}".application_versions`;
|
|
1055
|
+
const params = applicationName ? [applicationName] : [];
|
|
1056
|
+
const scope = this.global(undefined, params);
|
|
740
1057
|
// Probe each eligible owner separately; an OR filter can scan an unbounded history of newer peer versions.
|
|
741
1058
|
// Keep the constant owner in the ordering so the IS NULL probe uses the composite index too.
|
|
742
1059
|
const query = applicationName
|
|
743
|
-
? `(SELECT ${columns} FROM ${table} WHERE application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
1060
|
+
? `(SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name = $1 ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
744
1061
|
UNION ALL
|
|
745
|
-
(SELECT ${columns} FROM ${table} WHERE application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
1062
|
+
(SELECT ${columns} FROM ${table} WHERE ${scope} AND application_name IS NULL ORDER BY application_name, version_timestamp DESC LIMIT 1)
|
|
746
1063
|
ORDER BY version_timestamp DESC LIMIT 1`
|
|
747
|
-
: `SELECT ${columns} FROM ${table} ORDER BY version_timestamp DESC LIMIT 1`;
|
|
748
|
-
const { rows } = await client.query(query,
|
|
1064
|
+
: `SELECT ${columns} FROM ${table} WHERE ${scope} ORDER BY version_timestamp DESC LIMIT 1`;
|
|
1065
|
+
const { rows } = await client.query(query, params);
|
|
749
1066
|
return rows[0];
|
|
750
1067
|
}
|
|
751
1068
|
/**
|
|
@@ -753,7 +1070,8 @@ class SystemDatabase {
|
|
|
753
1070
|
* leaves the owner intact; a named one collides only with a different name.
|
|
754
1071
|
*/
|
|
755
1072
|
async #resolveRowOwner(client, table, keyColumn, name, owner, kind) {
|
|
756
|
-
const
|
|
1073
|
+
const params = [name];
|
|
1074
|
+
const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${this.global(undefined, params)} AND ${keyColumn} = $1`, params);
|
|
757
1075
|
const current = rows[0]?.application_name ?? null;
|
|
758
1076
|
if (current === null)
|
|
759
1077
|
return owner;
|
|
@@ -770,8 +1088,12 @@ class SystemDatabase {
|
|
|
770
1088
|
}
|
|
771
1089
|
/** Migrates the system database, or, when `runMigrations` is false, verifies it is already migrated. */
|
|
772
1090
|
async init(runMigrations = true) {
|
|
773
|
-
|
|
774
|
-
|
|
1091
|
+
if (runMigrations) {
|
|
1092
|
+
await this.runMigrations();
|
|
1093
|
+
}
|
|
1094
|
+
else {
|
|
1095
|
+
await this.verifyMigrations();
|
|
1096
|
+
}
|
|
775
1097
|
if (this.shouldUseDBNotifications) {
|
|
776
1098
|
await this.#listenForNotifications();
|
|
777
1099
|
// Push coalesced stream and event notifications off the write path.
|
|
@@ -779,6 +1101,32 @@ class SystemDatabase {
|
|
|
779
1101
|
this.#notifierLoop = this.#runNotifier();
|
|
780
1102
|
}
|
|
781
1103
|
}
|
|
1104
|
+
/** Create or migrate the system database. */
|
|
1105
|
+
async runMigrations() {
|
|
1106
|
+
await ensureSystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
|
|
1107
|
+
}
|
|
1108
|
+
/** Throw if this unsharded handle's schema is sharded; an unreachable database is left for its first statement. */
|
|
1109
|
+
async checkShardedSchema() {
|
|
1110
|
+
if (this.sharded)
|
|
1111
|
+
return;
|
|
1112
|
+
let client;
|
|
1113
|
+
try {
|
|
1114
|
+
client = await this.connect();
|
|
1115
|
+
}
|
|
1116
|
+
catch {
|
|
1117
|
+
return;
|
|
1118
|
+
}
|
|
1119
|
+
try {
|
|
1120
|
+
await refuseShardedSchema(client, this.schemaName);
|
|
1121
|
+
}
|
|
1122
|
+
finally {
|
|
1123
|
+
client.release();
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
/** Check the system database is migrated, creating and changing nothing. */
|
|
1127
|
+
async verifyMigrations() {
|
|
1128
|
+
await verifySystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
|
|
1129
|
+
}
|
|
782
1130
|
async destroy() {
|
|
783
1131
|
// Set synchronously, before any await, so no reconnect is scheduled or published after this point.
|
|
784
1132
|
this.#notificationsStopped = true;
|
|
@@ -824,7 +1172,7 @@ class SystemDatabase {
|
|
|
824
1172
|
/** Runs on `client` if given, joining its transaction; otherwise in its own retried transaction. */
|
|
825
1173
|
async initWorkflowStatus(initStatus, creatorXid, client, reusePolicy = 'return-existing') {
|
|
826
1174
|
if (client !== undefined) {
|
|
827
|
-
return await this
|
|
1175
|
+
return await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
828
1176
|
}
|
|
829
1177
|
return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
|
|
830
1178
|
}
|
|
@@ -833,7 +1181,7 @@ class SystemDatabase {
|
|
|
833
1181
|
let shouldCommit = false;
|
|
834
1182
|
try {
|
|
835
1183
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
836
|
-
const result = await this
|
|
1184
|
+
const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
837
1185
|
// If there is an existing DB record and we aren't here to recover it, leave it be.
|
|
838
1186
|
shouldCommit = result.shouldExecuteOnThisExecutor;
|
|
839
1187
|
return result;
|
|
@@ -861,10 +1209,12 @@ class SystemDatabase {
|
|
|
861
1209
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
862
1210
|
const parentOwnerXid = (0, context_1.currentOwnerXid)(parentWorkflowID);
|
|
863
1211
|
if (parentOwnerXid !== undefined) {
|
|
864
|
-
await this
|
|
1212
|
+
await this.checkOwner(client, parentWorkflowID, parentOwnerXid);
|
|
865
1213
|
}
|
|
866
|
-
const result = await this
|
|
867
|
-
await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs,
|
|
1214
|
+
const result = await this.initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy);
|
|
1215
|
+
await this.recordOperationResultInternal(client, parentWorkflowID, parentFunctionID, initStatus.workflowName, true, startTimeEpochMs, endTimeEpochMs,
|
|
1216
|
+
// Unkeyed, as only a sharded database records the child's key, through its own override.
|
|
1217
|
+
{ childWorkflowID: initStatus.workflowUUID, ownerChecked: true, childValues: {} });
|
|
868
1218
|
shouldCommit = true;
|
|
869
1219
|
return result;
|
|
870
1220
|
}
|
|
@@ -883,7 +1233,7 @@ class SystemDatabase {
|
|
|
883
1233
|
}
|
|
884
1234
|
}
|
|
885
1235
|
}
|
|
886
|
-
async
|
|
1236
|
+
async initWorkflowStatusInternal(client, initStatus, creatorXid, reusePolicy) {
|
|
887
1237
|
const resRow = await this.insertWorkflowStatus(client, initStatus, creatorXid);
|
|
888
1238
|
if (reusePolicy === 'reject' && (creatorXid === null || resRow.creator_xid !== creatorXid)) {
|
|
889
1239
|
throw new error_1.DBOSWorkflowIDInUseError(initStatus.workflowUUID, resRow.status, resRow.name);
|
|
@@ -915,6 +1265,7 @@ class SystemDatabase {
|
|
|
915
1265
|
async deadLetterWorkflows(workflowIDs, minRecoveryAttempts) {
|
|
916
1266
|
if (workflowIDs.length === 0)
|
|
917
1267
|
return;
|
|
1268
|
+
const params = [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflow_1.StatusString.PENDING, minRecoveryAttempts];
|
|
918
1269
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
919
1270
|
SET status = $1,
|
|
920
1271
|
owner_xid = NULL,
|
|
@@ -923,17 +1274,18 @@ class SystemDatabase {
|
|
|
923
1274
|
queue_name = NULL,
|
|
924
1275
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
925
1276
|
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
926
|
-
WHERE
|
|
1277
|
+
WHERE ${this.wfIn(undefined, workflowIDs, params)} AND status = $2 AND recovery_attempts >= $3`, params);
|
|
927
1278
|
}
|
|
928
1279
|
/** Highest created_at among still-active rows per partition key, used to seed the in-memory cursor. */
|
|
929
1280
|
async #maxPartitionKeyCreatedAt(keys) {
|
|
930
1281
|
const maxima = new Map();
|
|
931
1282
|
if (keys.length === 0)
|
|
932
1283
|
return maxima;
|
|
1284
|
+
const params = [keys, [workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING]];
|
|
933
1285
|
const { rows } = await this.pool.query(`SELECT queue_partition_key, MAX(created_at) AS max_created_at
|
|
934
1286
|
FROM "${this.schemaName}".workflow_status
|
|
935
|
-
WHERE queue_partition_key = ANY($1) AND status = ANY($2)
|
|
936
|
-
GROUP BY queue_partition_key`,
|
|
1287
|
+
WHERE ${this.scoped(undefined, params)} AND queue_partition_key = ANY($1) AND status = ANY($2)
|
|
1288
|
+
GROUP BY queue_partition_key`, params);
|
|
937
1289
|
for (const row of rows) {
|
|
938
1290
|
if (row.max_created_at !== null) {
|
|
939
1291
|
maxima.set(row.queue_partition_key, Number(row.max_created_at));
|
|
@@ -1039,39 +1391,60 @@ class SystemDatabase {
|
|
|
1039
1391
|
const chunkSize = 500;
|
|
1040
1392
|
for (let start = 0; start < statuses.length; start += chunkSize) {
|
|
1041
1393
|
const chunk = statuses.slice(start, start + chunkSize);
|
|
1042
|
-
const
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1394
|
+
const keyed = chunk.map((status) => ({
|
|
1395
|
+
...this.wfValues(status.workflowUUID, status.shardKey),
|
|
1396
|
+
...this.parentValues(status.parentWorkflowID),
|
|
1397
|
+
}));
|
|
1398
|
+
const rows = chunk.map((status, i) => {
|
|
1047
1399
|
const createdAt = createdAts[start + i];
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1400
|
+
return [
|
|
1401
|
+
...Object.values(keyed[i]),
|
|
1402
|
+
status.workflowUUID,
|
|
1403
|
+
status.status,
|
|
1404
|
+
status.workflowName,
|
|
1405
|
+
// For cross-language compatibility, these MUST be NULL in the database when not set
|
|
1406
|
+
status.workflowClassName === '' ? null : status.workflowClassName,
|
|
1407
|
+
status.workflowConfigName === '' ? null : status.workflowConfigName,
|
|
1408
|
+
status.queueName ?? null,
|
|
1409
|
+
status.authenticatedUser,
|
|
1410
|
+
status.assumedRole,
|
|
1411
|
+
JSON.stringify(status.authenticatedRoles),
|
|
1412
|
+
status.executorId,
|
|
1413
|
+
status.applicationVersion ?? null,
|
|
1414
|
+
status.applicationID,
|
|
1415
|
+
createdAt,
|
|
1416
|
+
0,
|
|
1417
|
+
createdAt,
|
|
1418
|
+
status.timeoutMS ?? null,
|
|
1419
|
+
status.deadlineEpochMS ?? null,
|
|
1420
|
+
// Legacy column: the payload lives in workflow_input.
|
|
1421
|
+
null,
|
|
1422
|
+
null,
|
|
1423
|
+
status.priority,
|
|
1424
|
+
status.queuePartitionKey ?? null,
|
|
1425
|
+
status.parentWorkflowID ?? null,
|
|
1426
|
+
status.serialization,
|
|
1427
|
+
null,
|
|
1428
|
+
status.delayUntilEpochMS ?? null,
|
|
1429
|
+
status.attributes ? JSON.stringify(status.attributes) : null,
|
|
1430
|
+
status.scheduleName ?? null,
|
|
1431
|
+
status.applicationName ?? null,
|
|
1432
|
+
];
|
|
1433
|
+
});
|
|
1434
|
+
const returned = await this.insertRows(client, 'workflow_status', [...Object.keys(keyed[0]), ...columns], rows, `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING RETURNING workflow_uuid`);
|
|
1435
|
+
for (const row of returned) {
|
|
1060
1436
|
inserted.add(row.workflow_uuid);
|
|
1061
1437
|
}
|
|
1062
1438
|
}
|
|
1063
1439
|
for (let start = 0; start < statuses.length; start += chunkSize) {
|
|
1064
1440
|
const chunk = statuses.slice(start, start + chunkSize);
|
|
1065
|
-
const
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
}
|
|
1072
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
|
|
1073
|
-
VALUES ${tuples.join(', ')}
|
|
1074
|
-
ON CONFLICT (workflow_uuid) DO NOTHING`, params);
|
|
1441
|
+
const keyed = chunk.map((status) => this.wfValues(status.workflowUUID, status.shardKey));
|
|
1442
|
+
await this.insertRows(client, 'workflow_input', [...Object.keys(keyed[0]), 'workflow_uuid', 'inputs', 'retention_timestamp'], chunk.map((status, i) => [
|
|
1443
|
+
...Object.values(keyed[i]),
|
|
1444
|
+
status.workflowUUID,
|
|
1445
|
+
status.input,
|
|
1446
|
+
createdAts[start + i],
|
|
1447
|
+
]), `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`);
|
|
1075
1448
|
}
|
|
1076
1449
|
await client.query('COMMIT');
|
|
1077
1450
|
}
|
|
@@ -1085,22 +1458,10 @@ class SystemDatabase {
|
|
|
1085
1458
|
return inserted;
|
|
1086
1459
|
}
|
|
1087
1460
|
async recordWorkflowOutput(workflowID, status, ownerXid) {
|
|
1088
|
-
|
|
1089
|
-
try {
|
|
1090
|
-
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
|
|
1091
|
-
}
|
|
1092
|
-
finally {
|
|
1093
|
-
client.release();
|
|
1094
|
-
}
|
|
1461
|
+
return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
|
|
1095
1462
|
}
|
|
1096
1463
|
async recordWorkflowError(workflowID, status, ownerXid) {
|
|
1097
|
-
|
|
1098
|
-
try {
|
|
1099
|
-
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
|
|
1100
|
-
}
|
|
1101
|
-
finally {
|
|
1102
|
-
client.release();
|
|
1103
|
-
}
|
|
1464
|
+
return await this.#recordWorkflowOutcome(workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
|
|
1104
1465
|
}
|
|
1105
1466
|
// Record a workflow's terminal outcome (SUCCESS or ERROR), reporting whether
|
|
1106
1467
|
// the write landed. The write applies only to a PENDING row still owned by
|
|
@@ -1111,8 +1472,49 @@ class SystemDatabase {
|
|
|
1111
1472
|
// terminal, handed to another execution (e.g. by a concurrent resume or
|
|
1112
1473
|
// recovery), or deleted; the caller resolves which by awaiting the recorded
|
|
1113
1474
|
// outcome.
|
|
1114
|
-
async #recordWorkflowOutcome(
|
|
1475
|
+
async #recordWorkflowOutcome(workflowID, status, outcome, ownerXid) {
|
|
1115
1476
|
ownerXid ??= (0, context_1.currentOwnerXid)(workflowID);
|
|
1477
|
+
if (!(await this.outcomeInOneStatement())) {
|
|
1478
|
+
return await this.#recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid);
|
|
1479
|
+
}
|
|
1480
|
+
// One autocommit round trip; the upsert reads the UPDATE's rows, so the output lands only with the status.
|
|
1481
|
+
const params = [status, workflow_1.StatusString.PENDING, outcome.output ?? null, outcome.error ?? null];
|
|
1482
|
+
const target = this.wf(undefined, workflowID, params);
|
|
1483
|
+
const owned = ownerXid !== undefined ? `AND owner_xid = $${params.push(ownerXid)}` : '';
|
|
1484
|
+
const keys = this.wfValues(workflowID);
|
|
1485
|
+
const keyColumns = Object.keys(keys);
|
|
1486
|
+
const keyExprs = Object.values(keys).map((value) => `$${params.push(value)}::text`);
|
|
1487
|
+
const workflowParam = `$${params.push(workflowID)}::text`;
|
|
1488
|
+
const { rows } = await this.pool.query(`WITH updated AS (
|
|
1489
|
+
UPDATE "${this.schemaName}".workflow_status
|
|
1490
|
+
SET status = $1,
|
|
1491
|
+
-- As the workflow is complete, remove its deduplication ID; a finished workflow has no owner.
|
|
1492
|
+
deduplication_id = NULL,
|
|
1493
|
+
owner_xid = NULL,
|
|
1494
|
+
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1495
|
+
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1496
|
+
WHERE ${target} AND status = $2 ${owned}
|
|
1497
|
+
RETURNING 1
|
|
1498
|
+
), stored AS (
|
|
1499
|
+
INSERT INTO "${this.schemaName}".workflow_output (${[...keyColumns, 'workflow_uuid', 'output', 'error'].join(', ')})
|
|
1500
|
+
SELECT ${[...keyExprs, workflowParam, '$3::text', '$4::text'].join(', ')}
|
|
1501
|
+
WHERE EXISTS (SELECT 1 FROM updated)
|
|
1502
|
+
-- EXCLUDED is the row this INSERT proposed, so these are the values above.
|
|
1503
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error
|
|
1504
|
+
-- CockroachDB refuses a data-modifying CTE without RETURNING.
|
|
1505
|
+
RETURNING 1
|
|
1506
|
+
)
|
|
1507
|
+
-- Selecting from the CTEs rather than inserting at the top level lets a router see the shard key.
|
|
1508
|
+
SELECT count(*) AS count FROM updated`, params);
|
|
1509
|
+
return Number(rows[0]?.count ?? 0) > 0;
|
|
1510
|
+
}
|
|
1511
|
+
/** Whether a workflow's outcome is written in one statement, rather than a transaction of two. */
|
|
1512
|
+
outcomeInOneStatement() {
|
|
1513
|
+
return Promise.resolve(true);
|
|
1514
|
+
}
|
|
1515
|
+
/** The outcome write as a transaction of two statements, for a backend that cannot run it as one. */
|
|
1516
|
+
async #recordWorkflowOutcomeInTransaction(workflowID, status, outcome, ownerXid) {
|
|
1517
|
+
const client = await this.connect();
|
|
1116
1518
|
let committed = false;
|
|
1117
1519
|
try {
|
|
1118
1520
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
@@ -1126,19 +1528,31 @@ class SystemDatabase {
|
|
|
1126
1528
|
// The outcome was not ours to write, so leave no orphan payload.
|
|
1127
1529
|
return false;
|
|
1128
1530
|
}
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1531
|
+
const values = {
|
|
1532
|
+
...this.wfValues(workflowID),
|
|
1533
|
+
workflow_uuid: workflowID,
|
|
1534
|
+
output: outcome.output ?? null,
|
|
1535
|
+
error: outcome.error ?? null,
|
|
1536
|
+
};
|
|
1537
|
+
const columns = Object.keys(values);
|
|
1538
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_output (${columns.join(', ')})
|
|
1539
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
1540
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error`, Object.values(values));
|
|
1132
1541
|
committed = true;
|
|
1133
1542
|
return true;
|
|
1134
1543
|
}
|
|
1135
1544
|
finally {
|
|
1136
|
-
|
|
1137
|
-
|
|
1545
|
+
try {
|
|
1546
|
+
if (committed) {
|
|
1547
|
+
await client.query('COMMIT');
|
|
1548
|
+
}
|
|
1549
|
+
else {
|
|
1550
|
+
// Swallowed: a rollback failure must not mask why we are rolling back.
|
|
1551
|
+
await client.query('ROLLBACK').catch(() => undefined);
|
|
1552
|
+
}
|
|
1138
1553
|
}
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
await client.query('ROLLBACK').catch(() => undefined);
|
|
1554
|
+
finally {
|
|
1555
|
+
client.release();
|
|
1142
1556
|
}
|
|
1143
1557
|
}
|
|
1144
1558
|
}
|
|
@@ -1147,6 +1561,7 @@ class SystemDatabase {
|
|
|
1147
1561
|
const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
|
|
1148
1562
|
// executor_id defaults to "local", so it collides across applications.
|
|
1149
1563
|
const scope = this.appNameFilter('application_name', this.appName, params);
|
|
1564
|
+
const keyColumn = this.keyColumn(undefined);
|
|
1150
1565
|
const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1151
1566
|
SET started_at_epoch_ms = NULL,
|
|
1152
1567
|
status = $1,
|
|
@@ -1157,27 +1572,31 @@ class SystemDatabase {
|
|
|
1157
1572
|
AND executor_id = $4
|
|
1158
1573
|
AND application_version = $5
|
|
1159
1574
|
AND ${scope}
|
|
1160
|
-
RETURNING workflow_uuid`, params);
|
|
1161
|
-
return result.rows.map((row) => row.workflow_uuid);
|
|
1575
|
+
RETURNING workflow_uuid${keyColumn !== undefined ? `, ${keyColumn} AS shard_key` : ''}`, params);
|
|
1576
|
+
return result.rows.map((row) => ({ workflowID: row.workflow_uuid, shardKey: row.shard_key ?? undefined }));
|
|
1162
1577
|
}
|
|
1163
1578
|
async getWorkflowStatus(workflowID, callerID, callerFN) {
|
|
1579
|
+
// Resolved before binding the caller's own key, which a same-ID target would share.
|
|
1580
|
+
const shardKey = this.keyHint(workflowID);
|
|
1164
1581
|
// Inside a workflow this runs on the transaction's client: taking a second pool connection while
|
|
1165
1582
|
// holding the first deadlocks the pool once every connection is held by a waiting caller.
|
|
1166
1583
|
const funcGetStatus = async (client) => {
|
|
1167
|
-
const statuses = await this.#listWorkflows({ workflowIDs: [workflowID] }, client);
|
|
1584
|
+
const statuses = await this.#listWorkflows({ workflowIDs: [workflowID], shardKey }, {}, client);
|
|
1168
1585
|
const status = statuses.find((s) => s.workflowUUID === workflowID);
|
|
1169
1586
|
return status ? JSON.stringify(status) : null;
|
|
1170
1587
|
};
|
|
1171
1588
|
if (callerID && callerFN !== undefined) {
|
|
1172
|
-
const
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1589
|
+
const json = await this.ownKey(callerID, undefined, async () => {
|
|
1590
|
+
const client = await this.connect();
|
|
1591
|
+
try {
|
|
1592
|
+
// Check if the operation has been done before for OAOO (only do this inside a workflow).
|
|
1593
|
+
return await this.inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, () => funcGetStatus(client)));
|
|
1594
|
+
}
|
|
1595
|
+
finally {
|
|
1596
|
+
client.release();
|
|
1597
|
+
}
|
|
1598
|
+
});
|
|
1599
|
+
return parseStatus(json);
|
|
1181
1600
|
}
|
|
1182
1601
|
else {
|
|
1183
1602
|
const json = await funcGetStatus();
|
|
@@ -1191,7 +1610,7 @@ class SystemDatabase {
|
|
|
1191
1610
|
statusFetchChunkSize = 500;
|
|
1192
1611
|
// Retried per chunk so a reconnect refetches one chunk, not every chunk before it.
|
|
1193
1612
|
async fetchWorkflowStatusChunk(workflowIDs) {
|
|
1194
|
-
return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false });
|
|
1613
|
+
return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false }, { keyedIDs: true });
|
|
1195
1614
|
}
|
|
1196
1615
|
/** Fetch many statuses in as few round trips as possible. IDs with no row are omitted. */
|
|
1197
1616
|
async getWorkflowStatuses(workflowIDs) {
|
|
@@ -1222,23 +1641,26 @@ class SystemDatabase {
|
|
|
1222
1641
|
}
|
|
1223
1642
|
}
|
|
1224
1643
|
// ==================== Step Results ====================
|
|
1225
|
-
async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1644
|
+
async getOperationResultAndThrowIfCancelled(workflowID, functionID, ownerKey) {
|
|
1645
|
+
return await this.ownKey(workflowID, ownerKey, async () => {
|
|
1646
|
+
const client = await this.connect();
|
|
1647
|
+
try {
|
|
1648
|
+
return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
1649
|
+
}
|
|
1650
|
+
finally {
|
|
1651
|
+
client.release();
|
|
1652
|
+
}
|
|
1653
|
+
});
|
|
1233
1654
|
}
|
|
1234
1655
|
async getAllOperationResults(workflowID, limit, offset, loadOutput = true) {
|
|
1235
1656
|
// Skip the output/error payload columns when the caller does not need them,
|
|
1236
1657
|
// mirroring how listWorkflows skips its payload joins when loadOutput is false.
|
|
1658
|
+
const childKey = this.keyColumn(undefined, 'child_shard_key');
|
|
1237
1659
|
const columns = loadOutput
|
|
1238
1660
|
? '*'
|
|
1239
|
-
:
|
|
1240
|
-
|
|
1241
|
-
|
|
1661
|
+
: `workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? `, ${childKey}` : ''}`;
|
|
1662
|
+
const params = [];
|
|
1663
|
+
let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE ${this.wf(undefined, workflowID, params)} ORDER BY function_id`;
|
|
1242
1664
|
if (limit !== undefined) {
|
|
1243
1665
|
params.push(limit);
|
|
1244
1666
|
query += ` LIMIT $${params.length}`;
|
|
@@ -1251,14 +1673,18 @@ class SystemDatabase {
|
|
|
1251
1673
|
return rows;
|
|
1252
1674
|
}
|
|
1253
1675
|
async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1676
|
+
// Resolved before binding the caller's own key, which a same-ID child would share.
|
|
1677
|
+
const childValues = this.childValues(options.childWorkflowID);
|
|
1678
|
+
await this.ownKey(workflowID, options.ownerKey, async () => {
|
|
1679
|
+
const client = await this.connect();
|
|
1680
|
+
try {
|
|
1681
|
+
await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, { ...options, childValues }));
|
|
1682
|
+
}
|
|
1683
|
+
finally {
|
|
1684
|
+
client.release();
|
|
1685
|
+
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
|
|
1686
|
+
}
|
|
1687
|
+
});
|
|
1262
1688
|
}
|
|
1263
1689
|
/** Checkpoint an operation's failure so a replay re-throws it instead of re-running. */
|
|
1264
1690
|
async recordOperationError(workflowID, functionID, functionName, startTimeEpochMs, error, childWorkflowID) {
|
|
@@ -1332,9 +1758,10 @@ class SystemDatabase {
|
|
|
1332
1758
|
throw new TypeError('functionID must be defined');
|
|
1333
1759
|
patchName = `DBOS.patch-${patchName}`;
|
|
1334
1760
|
const readCheckpointName = async (queryable) => {
|
|
1761
|
+
const params = [functionID];
|
|
1335
1762
|
const { rows } = await queryable.query(`SELECT function_name
|
|
1336
1763
|
FROM "${this.schemaName}".operation_outputs
|
|
1337
|
-
WHERE
|
|
1764
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
|
|
1338
1765
|
return rows.length === 0 ? undefined : rows[0].function_name;
|
|
1339
1766
|
};
|
|
1340
1767
|
if (deprecated) {
|
|
@@ -1349,7 +1776,7 @@ class SystemDatabase {
|
|
|
1349
1776
|
// If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
|
|
1350
1777
|
const client = await this.connect();
|
|
1351
1778
|
try {
|
|
1352
|
-
return await this
|
|
1779
|
+
return await this.inTransaction(client, async () => {
|
|
1353
1780
|
const checkpointName = await readCheckpointName(client);
|
|
1354
1781
|
if (checkpointName !== undefined) {
|
|
1355
1782
|
const matches = checkpointName === patchName;
|
|
@@ -1375,7 +1802,7 @@ class SystemDatabase {
|
|
|
1375
1802
|
let frontier = workflowIDs;
|
|
1376
1803
|
while (frontier.length > 0) {
|
|
1377
1804
|
await this.#cancelWorkflows(frontier);
|
|
1378
|
-
const children = await this
|
|
1805
|
+
const children = await this.getDirectChildren(frontier);
|
|
1379
1806
|
frontier = children.filter((id) => !visited.has(id));
|
|
1380
1807
|
for (const id of frontier) {
|
|
1381
1808
|
visited.add(id);
|
|
@@ -1383,41 +1810,44 @@ class SystemDatabase {
|
|
|
1383
1810
|
}
|
|
1384
1811
|
}
|
|
1385
1812
|
async #cancelWorkflows(workflowIDs) {
|
|
1813
|
+
const params = [workflow_1.StatusString.CANCELLED, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR];
|
|
1386
1814
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1387
1815
|
SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
1388
1816
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1389
1817
|
completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1390
|
-
WHERE
|
|
1391
|
-
AND status NOT IN ($
|
|
1818
|
+
WHERE ${this.wfIn(undefined, workflowIDs, params)}
|
|
1819
|
+
AND status NOT IN ($2, $3)`, params);
|
|
1392
1820
|
}
|
|
1393
1821
|
async checkIfCanceled(workflowID) {
|
|
1394
|
-
|
|
1822
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
1823
|
+
await this.ownKey(workflowID, undefined, () => this.#checkIfCanceled(this.pool, workflowID));
|
|
1395
1824
|
}
|
|
1396
1825
|
async resumeWorkflows(workflowIDs, queueName) {
|
|
1397
1826
|
const client = await this.connect();
|
|
1398
1827
|
try {
|
|
1399
1828
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1400
1829
|
// Check existence separately: a zero-row update also means "already complete", a legal no-op.
|
|
1401
|
-
const
|
|
1830
|
+
const existingParams = [];
|
|
1831
|
+
const { rows } = await client.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${this.wfIn(undefined, workflowIDs, existingParams)}`, existingParams);
|
|
1402
1832
|
const existing = new Set(rows.map((r) => r.workflow_uuid));
|
|
1403
1833
|
const missing = workflowIDs.filter((id) => !existing.has(id));
|
|
1404
1834
|
if (missing.length > 0) {
|
|
1405
1835
|
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${missing.join(', ')} does not exist`);
|
|
1406
1836
|
}
|
|
1837
|
+
const updateParams = [
|
|
1838
|
+
workflow_1.StatusString.ENQUEUED,
|
|
1839
|
+
queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1840
|
+
workflow_1.StatusString.SUCCESS,
|
|
1841
|
+
workflow_1.StatusString.ERROR,
|
|
1842
|
+
];
|
|
1407
1843
|
await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1408
1844
|
SET status = $1, owner_xid = NULL, queue_name = $2, recovery_attempts = 0,
|
|
1409
1845
|
workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
|
|
1410
1846
|
started_at_epoch_ms = NULL,
|
|
1411
1847
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
|
|
1412
1848
|
completed_at = NULL
|
|
1413
|
-
WHERE
|
|
1414
|
-
AND status NOT IN ($
|
|
1415
|
-
workflow_1.StatusString.ENQUEUED,
|
|
1416
|
-
queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1417
|
-
workflowIDs,
|
|
1418
|
-
workflow_1.StatusString.SUCCESS,
|
|
1419
|
-
workflow_1.StatusString.ERROR,
|
|
1420
|
-
]);
|
|
1849
|
+
WHERE ${this.wfIn(undefined, workflowIDs, updateParams)}
|
|
1850
|
+
AND status NOT IN ($3, $4)`, updateParams);
|
|
1421
1851
|
await client.query('COMMIT');
|
|
1422
1852
|
}
|
|
1423
1853
|
catch (e) {
|
|
@@ -1430,16 +1860,18 @@ class SystemDatabase {
|
|
|
1430
1860
|
}
|
|
1431
1861
|
}
|
|
1432
1862
|
async setWorkflowPriority(workflowID, priority) {
|
|
1863
|
+
const params = [priority, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED];
|
|
1433
1864
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1434
1865
|
SET priority = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1435
|
-
WHERE
|
|
1436
|
-
AND status IN ($
|
|
1866
|
+
WHERE ${this.wf(undefined, workflowID, params)}
|
|
1867
|
+
AND status IN ($2, $3)`, params);
|
|
1437
1868
|
}
|
|
1438
1869
|
async setWorkflowDelay(workflowID, delayUntilEpochMS) {
|
|
1870
|
+
const params = [delayUntilEpochMS, workflow_1.StatusString.DELAYED];
|
|
1439
1871
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1440
1872
|
SET delay_until_epoch_ms = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
1441
|
-
WHERE
|
|
1442
|
-
AND status = $
|
|
1873
|
+
WHERE ${this.wf(undefined, workflowID, params)}
|
|
1874
|
+
AND status = $2`, params);
|
|
1443
1875
|
}
|
|
1444
1876
|
/**
|
|
1445
1877
|
* Extend an existing debounced DELAYED workflow's delay and update its inputs, atomically.
|
|
@@ -1505,7 +1937,7 @@ class SystemDatabase {
|
|
|
1505
1937
|
if (updated.rows.length > 0) {
|
|
1506
1938
|
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
|
|
1507
1939
|
VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
|
|
1508
|
-
ON CONFLICT (workflow_uuid) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
|
|
1940
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
|
|
1509
1941
|
return {
|
|
1510
1942
|
bouncedWorkflowID: updated.rows[0].workflow_uuid,
|
|
1511
1943
|
holderWorkflowID: null,
|
|
@@ -1515,10 +1947,11 @@ class SystemDatabase {
|
|
|
1515
1947
|
holderApplicationName: null,
|
|
1516
1948
|
};
|
|
1517
1949
|
}
|
|
1518
|
-
//
|
|
1950
|
+
// No application filter, so a holder that blocked the update above is reportable.
|
|
1951
|
+
const holderParams = [params.queueName, params.deduplicationID];
|
|
1519
1952
|
const holder = await client.query(`SELECT workflow_uuid, is_debounced, name, class_name, application_name
|
|
1520
1953
|
FROM "${this.schemaName}".workflow_status
|
|
1521
|
-
WHERE queue_name = $1 AND deduplication_id = $2`,
|
|
1954
|
+
WHERE ${this.scoped(undefined, holderParams)} AND queue_name = $1 AND deduplication_id = $2`, holderParams);
|
|
1522
1955
|
if (holder.rows.length === 0) {
|
|
1523
1956
|
return {
|
|
1524
1957
|
bouncedWorkflowID: null,
|
|
@@ -1539,7 +1972,7 @@ class SystemDatabase {
|
|
|
1539
1972
|
};
|
|
1540
1973
|
}
|
|
1541
1974
|
// Get the immediate (one-level) child workflow IDs for a set of workflows.
|
|
1542
|
-
async
|
|
1975
|
+
async getDirectChildren(workflowIDs) {
|
|
1543
1976
|
if (workflowIDs.length === 0) {
|
|
1544
1977
|
return [];
|
|
1545
1978
|
}
|
|
@@ -1553,7 +1986,7 @@ class SystemDatabase {
|
|
|
1553
1986
|
const descendants = new Set();
|
|
1554
1987
|
let frontier = [workflowID];
|
|
1555
1988
|
while (frontier.length > 0) {
|
|
1556
|
-
const children = await this
|
|
1989
|
+
const children = await this.getDirectChildren(frontier);
|
|
1557
1990
|
frontier = children.filter((id) => !descendants.has(id));
|
|
1558
1991
|
for (const id of frontier) {
|
|
1559
1992
|
descendants.add(id);
|
|
@@ -1561,6 +1994,7 @@ class SystemDatabase {
|
|
|
1561
1994
|
}
|
|
1562
1995
|
return [...descendants];
|
|
1563
1996
|
}
|
|
1997
|
+
/** Delete workflows and all associated data, and with deleteChildren every workflow they started, recursively. */
|
|
1564
1998
|
async deleteWorkflows(workflowIDs, deleteChildren = false) {
|
|
1565
1999
|
const allIds = [...workflowIDs];
|
|
1566
2000
|
if (deleteChildren) {
|
|
@@ -1568,15 +2002,18 @@ class SystemDatabase {
|
|
|
1568
2002
|
allIds.push(...(await this.getWorkflowChildren(wfid)));
|
|
1569
2003
|
}
|
|
1570
2004
|
}
|
|
2005
|
+
await this.deleteWorkflowsInternal(allIds);
|
|
2006
|
+
}
|
|
2007
|
+
async deleteWorkflowsInternal(allIds) {
|
|
1571
2008
|
const client = await this.connect();
|
|
1572
2009
|
try {
|
|
1573
2010
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1574
2011
|
// The payload tables carry no foreign key, so the status delete does not cascade
|
|
1575
2012
|
// into them; without this they survive as orphans.
|
|
1576
|
-
for (const table of ['workflow_input', 'workflow_output', 'operation_outputs']) {
|
|
1577
|
-
|
|
2013
|
+
for (const table of ['workflow_input', 'workflow_output', 'operation_outputs', 'workflow_status']) {
|
|
2014
|
+
const params = [];
|
|
2015
|
+
await client.query(`DELETE FROM "${this.schemaName}".${table} WHERE ${this.wfIn(undefined, allIds, params)}`, params);
|
|
1578
2016
|
}
|
|
1579
|
-
await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [allIds]);
|
|
1580
2017
|
await client.query('COMMIT');
|
|
1581
2018
|
}
|
|
1582
2019
|
catch (e) {
|
|
@@ -1617,7 +2054,8 @@ class SystemDatabase {
|
|
|
1617
2054
|
const client = await this.connect();
|
|
1618
2055
|
try {
|
|
1619
2056
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1620
|
-
const
|
|
2057
|
+
const statusParams = [];
|
|
2058
|
+
const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE ${this.wf(undefined, workflowID, statusParams)}`, statusParams);
|
|
1621
2059
|
if (statusRows.length === 0) {
|
|
1622
2060
|
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
|
|
1623
2061
|
}
|
|
@@ -1628,56 +2066,61 @@ class SystemDatabase {
|
|
|
1628
2066
|
// Roll workflow_events back to the last value published before the cut, using
|
|
1629
2067
|
// workflow_events_history as an undo log, then rewind that history too.
|
|
1630
2068
|
// `publishedPastCut` is the correlated EXISTS both halves key off.
|
|
1631
|
-
const publishedPastCut = `EXISTS (
|
|
2069
|
+
const publishedPastCut = (key, params) => `EXISTS (
|
|
1632
2070
|
SELECT 1 FROM "${schema}".workflow_events_history discarded
|
|
1633
|
-
WHERE discarded
|
|
2071
|
+
WHERE ${this.wf('discarded', workflowID, params)} AND discarded.key = ${key} AND discarded.function_id >= $1
|
|
1634
2072
|
)`;
|
|
1635
2073
|
// First unpublish everything the discarded run touched.
|
|
2074
|
+
const unpublishParams = [startStep];
|
|
1636
2075
|
await client.query(`DELETE FROM "${schema}".workflow_events
|
|
1637
|
-
WHERE
|
|
2076
|
+
WHERE ${this.wf(undefined, workflowID, unpublishParams)}
|
|
2077
|
+
AND ${publishedPastCut('"' + schema + '".workflow_events.key', unpublishParams)}`, unpublishParams);
|
|
1638
2078
|
// Then restore those keys, if any, to the value published below the cut.
|
|
1639
|
-
|
|
1640
|
-
|
|
2079
|
+
const restoreParams = [startStep];
|
|
2080
|
+
const keyColumns = this.wfColumns(workflowID, restoreParams);
|
|
2081
|
+
await client.query(`INSERT INTO "${schema}".workflow_events (${[...keyColumns.map(([c]) => c), 'workflow_uuid', 'key', 'value', 'serialization'].join(', ')})
|
|
2082
|
+
SELECT ${[...keyColumns.map(([, e]) => e), 'surviving.workflow_uuid', 'surviving.key', 'surviving.value', 'surviving.serialization'].join(', ')}
|
|
1641
2083
|
FROM (
|
|
1642
2084
|
SELECT weh.workflow_uuid, weh.key, weh.value, weh.serialization,
|
|
1643
2085
|
ROW_NUMBER() OVER (PARTITION BY weh.key ORDER BY weh.function_id DESC) AS rn
|
|
1644
2086
|
FROM "${schema}".workflow_events_history weh
|
|
1645
|
-
WHERE weh
|
|
1646
|
-
AND ${publishedPastCut
|
|
2087
|
+
WHERE ${this.wf('weh', workflowID, restoreParams)} AND weh.function_id < $1
|
|
2088
|
+
AND ${publishedPastCut('weh.key', restoreParams)}
|
|
1647
2089
|
) surviving
|
|
1648
|
-
WHERE surviving.rn = 1`,
|
|
2090
|
+
WHERE surviving.rn = 1`, restoreParams);
|
|
1649
2091
|
// Clear the streams' close sentinels so the replay can append again.
|
|
2092
|
+
const sentinelParams = [startStep, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name()];
|
|
1650
2093
|
await client.query(`DELETE FROM "${schema}".streams
|
|
1651
|
-
WHERE
|
|
2094
|
+
WHERE ${this.wf(undefined, workflowID, sentinelParams)} AND function_id >= $1 AND value = $2 AND serialization = $3`, sentinelParams);
|
|
1652
2095
|
// Discard the steps and the events history past the cut.
|
|
1653
2096
|
for (const table of ['operation_outputs', 'workflow_events_history']) {
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
startStep,
|
|
1657
|
-
]);
|
|
2097
|
+
const params = [startStep];
|
|
2098
|
+
await client.query(`DELETE FROM "${schema}".${table} WHERE ${this.wf(undefined, workflowID, params)} AND function_id >= $1`, params);
|
|
1658
2099
|
}
|
|
1659
2100
|
// Delete messages consumed or received after the rewind point
|
|
1660
|
-
|
|
1661
|
-
await client.query(`DELETE FROM "${schema}".
|
|
2101
|
+
const messageParams = [startStep];
|
|
2102
|
+
await client.query(`DELETE FROM "${schema}".notifications
|
|
2103
|
+
WHERE ${this.wf(undefined, workflowID, messageParams, 'destination_uuid')}
|
|
2104
|
+
AND (consumed_by_function_id >= $1 OR consumed = false)`, messageParams);
|
|
2105
|
+
const outputParams = [];
|
|
2106
|
+
await client.query(`DELETE FROM "${schema}".workflow_output WHERE ${this.wf(undefined, workflowID, outputParams)}`, outputParams);
|
|
1662
2107
|
// Re-enqueue the workflow. Re-asserting the status we read keeps a workflow that
|
|
1663
2108
|
// moved on underneath us from being resurrected.
|
|
1664
|
-
const setVersion = options.applicationVersion !== undefined ? ', application_version = $6' : '';
|
|
1665
2109
|
const params = [
|
|
1666
2110
|
workflow_1.StatusString.ENQUEUED,
|
|
1667
2111
|
options.queueName ?? utils_1.INTERNAL_QUEUE_NAME,
|
|
1668
2112
|
options.queuePartitionKey ?? null,
|
|
1669
|
-
workflowID,
|
|
1670
2113
|
status,
|
|
1671
2114
|
];
|
|
1672
|
-
|
|
1673
|
-
params.push(options.applicationVersion)
|
|
1674
|
-
|
|
2115
|
+
const setVersion = options.applicationVersion !== undefined
|
|
2116
|
+
? `, application_version = $${params.push(options.applicationVersion)}`
|
|
2117
|
+
: '';
|
|
1675
2118
|
const { rowCount } = await client.query(`UPDATE "${schema}".workflow_status
|
|
1676
2119
|
SET status = $1, owner_xid = NULL, queue_name = $2, queue_partition_key = $3, recovery_attempts = 0,
|
|
1677
2120
|
workflow_deadline_epoch_ms = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
|
|
1678
2121
|
completed_at = NULL, output = NULL, error = NULL,
|
|
1679
2122
|
updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint${setVersion}
|
|
1680
|
-
WHERE
|
|
2123
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND status = $4`, params);
|
|
1681
2124
|
if (rowCount !== 1) {
|
|
1682
2125
|
throw new error_1.DBOSError(`Workflow ${workflowID} changed status while being rewound; retry the rewind`);
|
|
1683
2126
|
}
|
|
@@ -1707,13 +2150,14 @@ class SystemDatabase {
|
|
|
1707
2150
|
try {
|
|
1708
2151
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1709
2152
|
// Fetch the status of all original workflows inside the transaction.
|
|
2153
|
+
const statusParams = [];
|
|
1710
2154
|
const { rows: statusRows } = await client.query(`SELECT ws.workflow_uuid, ws.name, ws.class_name, ws.config_name, ws.application_id,
|
|
1711
2155
|
ws.authenticated_user, ws.authenticated_roles, ws.assumed_role,
|
|
1712
2156
|
COALESCE(wi.inputs, ws.inputs) AS inputs, ws.serialization,
|
|
1713
2157
|
ws.application_version, ws.attributes, ws.application_name
|
|
1714
2158
|
FROM "${this.schemaName}".workflow_status ws
|
|
1715
|
-
LEFT JOIN "${this.schemaName}".workflow_input wi ON
|
|
1716
|
-
WHERE ws
|
|
2159
|
+
LEFT JOIN "${this.schemaName}".workflow_input wi ON ${this.wfJoin('ws', 'wi')}
|
|
2160
|
+
WHERE ${this.wfIn('ws', originalWorkflowIDs, statusParams)}`, statusParams);
|
|
1717
2161
|
const statusByID = new Map(statusRows.map((r) => [r.workflow_uuid, r]));
|
|
1718
2162
|
for (const wid of originalWorkflowIDs) {
|
|
1719
2163
|
if (!statusByID.has(wid)) {
|
|
@@ -1749,35 +2193,42 @@ class SystemDatabase {
|
|
|
1749
2193
|
forkedWorkflowIDs[i],
|
|
1750
2194
|
statusByID.get(origID).application_name ?? this.appName ?? null,
|
|
1751
2195
|
]));
|
|
1752
|
-
const
|
|
1753
|
-
|
|
1754
|
-
let paramIdx = 1;
|
|
1755
|
-
for (let i = 0; i < originalWorkflowIDs.length; i++) {
|
|
1756
|
-
const origID = originalWorkflowIDs[i];
|
|
2196
|
+
const forkKeys = forkedWorkflowIDs.map((forkID) => this.wfValues(forkID));
|
|
2197
|
+
await this.insertRows(client, 'workflow_status', [...Object.keys(forkKeys[0]), ...insertCols], originalWorkflowIDs.map((origID, i) => {
|
|
1757
2198
|
const forkID = forkedWorkflowIDs[i];
|
|
1758
2199
|
const ws = statusByID.get(origID);
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
2200
|
+
return [
|
|
2201
|
+
...Object.values(forkKeys[i]),
|
|
2202
|
+
forkID,
|
|
2203
|
+
workflow_1.StatusString.ENQUEUED,
|
|
2204
|
+
ws.name,
|
|
2205
|
+
ws.class_name ?? null,
|
|
2206
|
+
ws.config_name ?? null,
|
|
2207
|
+
queueName,
|
|
2208
|
+
ws.authenticated_user,
|
|
2209
|
+
ws.assumed_role,
|
|
2210
|
+
ws.authenticated_roles,
|
|
2211
|
+
options.applicationVersion ?? ws.application_version ?? null,
|
|
2212
|
+
ws.application_id,
|
|
2213
|
+
// Legacy column: the payload lives in workflow_input.
|
|
2214
|
+
null,
|
|
2215
|
+
options.queuePartitionKey ?? null,
|
|
2216
|
+
origID,
|
|
2217
|
+
ws.serialization,
|
|
2218
|
+
ws.attributes ? JSON.stringify(ws.attributes) : null,
|
|
2219
|
+
forkOwners.get(forkID) ?? null,
|
|
2220
|
+
...(options.timeoutMS !== undefined ? [options.timeoutMS] : []),
|
|
2221
|
+
];
|
|
2222
|
+
}));
|
|
2223
|
+
await this.insertRows(client, 'workflow_input', [...Object.keys(forkKeys[0]), 'workflow_uuid', 'inputs'], originalWorkflowIDs.map((origID, i) => [
|
|
2224
|
+
...Object.values(forkKeys[i]),
|
|
2225
|
+
forkedWorkflowIDs[i],
|
|
2226
|
+
statusByID.get(origID).inputs,
|
|
2227
|
+
]));
|
|
1779
2228
|
// Mark all original workflows as having been forked from.
|
|
1780
|
-
|
|
2229
|
+
const forkedFromParams = [];
|
|
2230
|
+
await client.query(`UPDATE "${this.schemaName}".workflow_status SET was_forked_from = TRUE
|
|
2231
|
+
WHERE ${this.wfIn(undefined, originalWorkflowIDs, forkedFromParams)}`, forkedFromParams);
|
|
1781
2232
|
// For workflows with start_step > 0, copy checkpoints/events/streams.
|
|
1782
2233
|
// Build a mapping CTE so each copy is a single SQL statement.
|
|
1783
2234
|
const forkMappings = originalWorkflowIDs
|
|
@@ -1794,55 +2245,85 @@ class SystemDatabase {
|
|
|
1794
2245
|
mIdx += 4;
|
|
1795
2246
|
}
|
|
1796
2247
|
const mappingCTE = `WITH mapping(orig_id, fork_id, start_step, owner) AS (VALUES ${mappingValues.join(', ')})`;
|
|
2248
|
+
// The forks live under their originals' shard key, read from the scope; a copy's params extend the mapping's.
|
|
2249
|
+
const copyParams = () => [...mappingParams];
|
|
2250
|
+
const keyed = (params) => {
|
|
2251
|
+
const columns = this.scopedColumns(params);
|
|
2252
|
+
return { names: columns.map(([c]) => `${c}, `).join(''), exprs: columns.map(([, e]) => `${e}, `).join('') };
|
|
2253
|
+
};
|
|
2254
|
+
// A replacement child is a fork, so it shares the replaced child's key.
|
|
2255
|
+
const childKey = this.keyColumn('oo', 'child_shard_key');
|
|
2256
|
+
if (options.replacementChildren && childKey !== undefined) {
|
|
2257
|
+
// A replacement names a child by ID, which can't pick one of several keys.
|
|
2258
|
+
const ambiguityParams = copyParams();
|
|
2259
|
+
const replaced = `$${ambiguityParams.push(Object.keys(options.replacementChildren))}::text[]`;
|
|
2260
|
+
const { rows: ambiguous } = await client.query(`${mappingCTE}
|
|
2261
|
+
SELECT oo.child_workflow_id
|
|
2262
|
+
FROM mapping m
|
|
2263
|
+
JOIN "${this.schemaName}".operation_outputs oo
|
|
2264
|
+
ON ${this.scoped('oo', ambiguityParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step
|
|
2265
|
+
WHERE oo.child_workflow_id = ANY(${replaced})
|
|
2266
|
+
GROUP BY oo.child_workflow_id
|
|
2267
|
+
HAVING COUNT(DISTINCT ${childKey}) > 1
|
|
2268
|
+
LIMIT 1`, ambiguityParams);
|
|
2269
|
+
if (ambiguous.length > 0) {
|
|
2270
|
+
throw new error_1.DBOSError(`Can't replace child workflow ${ambiguous[0].child_workflow_id}: the forked steps refer to children with that ID under more than one shard key`);
|
|
2271
|
+
}
|
|
2272
|
+
}
|
|
1797
2273
|
// Build the child_workflow_id expression, applying replacements if provided.
|
|
2274
|
+
const ooParams = copyParams();
|
|
1798
2275
|
let childWfExpr = 'oo.child_workflow_id';
|
|
1799
|
-
const ooParams = [...mappingParams];
|
|
1800
2276
|
if (options.replacementChildren && Object.keys(options.replacementChildren).length > 0) {
|
|
1801
2277
|
const whenClauses = [];
|
|
1802
2278
|
for (const [oldId, newId] of Object.entries(options.replacementChildren)) {
|
|
1803
|
-
whenClauses.push(`WHEN oo.child_workflow_id = $${
|
|
1804
|
-
ooParams.push(oldId, newId);
|
|
1805
|
-
mIdx += 2;
|
|
2279
|
+
whenClauses.push(`WHEN oo.child_workflow_id = $${ooParams.push(oldId)} THEN $${ooParams.push(newId)}::text`);
|
|
1806
2280
|
}
|
|
1807
2281
|
childWfExpr = `CASE ${whenClauses.join(' ')} ELSE oo.child_workflow_id END`;
|
|
1808
2282
|
}
|
|
2283
|
+
const ooKeys = keyed(ooParams);
|
|
1809
2284
|
// Copy operation outputs
|
|
1810
2285
|
await client.query(`${mappingCTE}
|
|
1811
2286
|
INSERT INTO "${this.schemaName}".operation_outputs
|
|
1812
|
-
(workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp)
|
|
1813
|
-
SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint
|
|
2287
|
+
(${ooKeys.names}workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp${childKey !== undefined ? ', child_shard_key' : ''})
|
|
2288
|
+
SELECT ${ooKeys.exprs}m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint${childKey !== undefined ? `, ${childKey}` : ''}
|
|
1814
2289
|
FROM mapping m
|
|
1815
2290
|
JOIN "${this.schemaName}".operation_outputs oo
|
|
1816
|
-
ON oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
|
|
2291
|
+
ON ${this.scoped('oo', ooParams)} AND oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
|
|
1817
2292
|
// Copy streams
|
|
2293
|
+
const streamParams = copyParams();
|
|
2294
|
+
const streamKeys = keyed(streamParams);
|
|
1818
2295
|
await client.query(`${mappingCTE}
|
|
1819
2296
|
INSERT INTO "${this.schemaName}".streams
|
|
1820
|
-
(workflow_uuid, key, value, serialization, "offset", function_id)
|
|
1821
|
-
SELECT m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
|
|
2297
|
+
(${streamKeys.names}workflow_uuid, key, value, serialization, "offset", function_id)
|
|
2298
|
+
SELECT ${streamKeys.exprs}m.fork_id, s.key, s.value, s.serialization, s."offset", s.function_id
|
|
1822
2299
|
FROM mapping m
|
|
1823
2300
|
JOIN "${this.schemaName}".streams s
|
|
1824
|
-
ON s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`,
|
|
2301
|
+
ON ${this.scoped('s', streamParams)} AND s.workflow_uuid = m.orig_id AND s.function_id < m.start_step`, streamParams);
|
|
1825
2302
|
// Copy events history
|
|
2303
|
+
const historyParams = copyParams();
|
|
2304
|
+
const historyKeys = keyed(historyParams);
|
|
1826
2305
|
await client.query(`${mappingCTE}
|
|
1827
2306
|
INSERT INTO "${this.schemaName}".workflow_events_history
|
|
1828
|
-
(workflow_uuid, function_id, key, value, serialization)
|
|
1829
|
-
SELECT m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
|
|
2307
|
+
(${historyKeys.names}workflow_uuid, function_id, key, value, serialization)
|
|
2308
|
+
SELECT ${historyKeys.exprs}m.fork_id, weh.function_id, weh.key, weh.value, weh.serialization
|
|
1830
2309
|
FROM mapping m
|
|
1831
2310
|
JOIN "${this.schemaName}".workflow_events_history weh
|
|
1832
|
-
ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`,
|
|
2311
|
+
ON ${this.scoped('weh', historyParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step`, historyParams);
|
|
1833
2312
|
// Copy only the latest version of each event using a window function
|
|
2313
|
+
const eventParams = copyParams();
|
|
2314
|
+
const eventKeys = keyed(eventParams);
|
|
1834
2315
|
await client.query(`${mappingCTE}
|
|
1835
2316
|
INSERT INTO "${this.schemaName}".workflow_events
|
|
1836
|
-
(workflow_uuid, key, value, serialization)
|
|
1837
|
-
SELECT ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
|
|
2317
|
+
(${eventKeys.names}workflow_uuid, key, value, serialization)
|
|
2318
|
+
SELECT ${eventKeys.exprs}ranked.workflow_uuid, ranked.key, ranked.value, ranked.serialization
|
|
1838
2319
|
FROM (
|
|
1839
2320
|
SELECT m.fork_id AS workflow_uuid, weh.key, weh.value, weh.serialization,
|
|
1840
|
-
ROW_NUMBER() OVER (PARTITION BY
|
|
2321
|
+
ROW_NUMBER() OVER (PARTITION BY m.fork_id, weh.key ORDER BY weh.function_id DESC) AS rn
|
|
1841
2322
|
FROM mapping m
|
|
1842
2323
|
JOIN "${this.schemaName}".workflow_events_history weh
|
|
1843
|
-
ON weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
|
|
2324
|
+
ON ${this.scoped('weh', eventParams)} AND weh.workflow_uuid = m.orig_id AND weh.function_id < m.start_step
|
|
1844
2325
|
) ranked
|
|
1845
|
-
WHERE ranked.rn = 1`,
|
|
2326
|
+
WHERE ranked.rn = 1`, eventParams);
|
|
1846
2327
|
}
|
|
1847
2328
|
await client.query('COMMIT');
|
|
1848
2329
|
return forkedWorkflowIDs;
|
|
@@ -1965,27 +2446,36 @@ class SystemDatabase {
|
|
|
1965
2446
|
}
|
|
1966
2447
|
/** Cancellation check for polling waits: goes through the limiter so readers cannot starve the pool. */
|
|
1967
2448
|
async checkIfCanceledLimited(workflowID) {
|
|
1968
|
-
|
|
2449
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
2450
|
+
const params = [];
|
|
2451
|
+
const target = this.ownKey(workflowID, undefined, () => this.wf(undefined, workflowID, params));
|
|
2452
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${target}`, params));
|
|
2453
|
+
if (rows[0]?.status === workflow_1.StatusString.CANCELLED) {
|
|
2454
|
+
throw new error_1.DBOSWorkflowCancelledError(workflowID);
|
|
2455
|
+
}
|
|
1969
2456
|
}
|
|
1970
2457
|
watchForCancellation(workflowID) {
|
|
1971
2458
|
const controller = new AbortController();
|
|
1972
2459
|
let state = 'idle';
|
|
2460
|
+
// The running workflow's own key, resolved now for the poller that reads it later outside this context.
|
|
2461
|
+
const shardKey = this.ownKey(workflowID, undefined, () => this.keyHint(workflowID));
|
|
2462
|
+
const waiterKey = this.ownKey(workflowID, undefined, () => this.waiter(workflowID, ''));
|
|
1973
2463
|
const start = () => {
|
|
1974
2464
|
state = 'watching';
|
|
1975
2465
|
if (this.#destroyed)
|
|
1976
2466
|
return;
|
|
1977
|
-
let watchers = this.#cancelWatchers.get(
|
|
2467
|
+
let watchers = this.#cancelWatchers.get(waiterKey);
|
|
1978
2468
|
if (watchers === undefined) {
|
|
1979
|
-
watchers = new Set();
|
|
1980
|
-
this.#cancelWatchers.set(
|
|
2469
|
+
watchers = { workflowID, shardKey, controllers: new Set() };
|
|
2470
|
+
this.#cancelWatchers.set(waiterKey, watchers);
|
|
1981
2471
|
}
|
|
1982
|
-
watchers.add(controller);
|
|
1983
|
-
this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => this.#runCancelPoller());
|
|
2472
|
+
watchers.controllers.add(controller);
|
|
2473
|
+
this.#cancelPollerLoop ??= this.#cancelPollerScope.runInAsyncScope(() => (0, shard_keys_1.withClearedShardKeyState)(() => this.#runCancelPoller()));
|
|
1984
2474
|
};
|
|
1985
2475
|
const stop = () => {
|
|
1986
|
-
const watchers = this.#cancelWatchers.get(
|
|
1987
|
-
if (watchers?.delete(controller) && watchers.size === 0) {
|
|
1988
|
-
this.#cancelWatchers.delete(
|
|
2476
|
+
const watchers = this.#cancelWatchers.get(waiterKey);
|
|
2477
|
+
if (watchers?.controllers.delete(controller) && watchers.controllers.size === 0) {
|
|
2478
|
+
this.#cancelWatchers.delete(waiterKey);
|
|
1989
2479
|
if (this.#cancelWatchers.size === 0)
|
|
1990
2480
|
this.#cancelPollerWake?.();
|
|
1991
2481
|
}
|
|
@@ -2024,15 +2514,36 @@ class SystemDatabase {
|
|
|
2024
2514
|
this.#cancelPollerLoop = undefined;
|
|
2025
2515
|
}
|
|
2026
2516
|
async #abortCancelledWatchers() {
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2517
|
+
// One read per shard key, each bound to its own workflows.
|
|
2518
|
+
const byKey = new Map();
|
|
2519
|
+
for (const { workflowID, shardKey } of this.#cancelWatchers.values()) {
|
|
2520
|
+
const ids = byKey.get(shardKey) ?? [];
|
|
2521
|
+
ids.push(workflowID);
|
|
2522
|
+
byKey.set(shardKey, ids);
|
|
2523
|
+
}
|
|
2524
|
+
for (const [shardKey, workflowIDs] of byKey) {
|
|
2525
|
+
// Each key's read on its own, so one key's failure doesn't hide another key's cancellations.
|
|
2526
|
+
let rows;
|
|
2527
|
+
try {
|
|
2528
|
+
({ rows } = await (0, shard_keys_1.bindShardKeys)(workflowIDs.map((id) => [id, shardKey]), () => {
|
|
2529
|
+
const params = [workflow_1.StatusString.CANCELLED];
|
|
2530
|
+
const ids = this.wfIn(undefined, workflowIDs, params);
|
|
2531
|
+
return this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status WHERE ${ids} AND status = $1`, params));
|
|
2532
|
+
}));
|
|
2533
|
+
}
|
|
2534
|
+
catch (e) {
|
|
2535
|
+
this.logger.warn(`Error polling status for cancellable steps: ${String(e)}`);
|
|
2031
2536
|
continue;
|
|
2032
|
-
|
|
2033
|
-
const
|
|
2034
|
-
for (const
|
|
2035
|
-
|
|
2537
|
+
}
|
|
2538
|
+
const cancelled = new Set(rows.map((row) => row.workflow_uuid));
|
|
2539
|
+
for (const [waiterKey, watchers] of this.#cancelWatchers) {
|
|
2540
|
+
if (watchers.shardKey !== shardKey || !cancelled.has(watchers.workflowID))
|
|
2541
|
+
continue;
|
|
2542
|
+
this.#cancelWatchers.delete(waiterKey);
|
|
2543
|
+
const reason = new error_1.DBOSWorkflowCancelledError(watchers.workflowID);
|
|
2544
|
+
for (const controller of watchers.controllers)
|
|
2545
|
+
controller.abort(reason);
|
|
2546
|
+
}
|
|
2036
2547
|
}
|
|
2037
2548
|
}
|
|
2038
2549
|
// A missing row normally means the workflow has not been inserted yet, so
|
|
@@ -2044,21 +2555,23 @@ class SystemDatabase {
|
|
|
2044
2555
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2045
2556
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2046
2557
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
2047
|
-
// Record the durable timeout deadline once before polling.
|
|
2558
|
+
// Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
|
|
2048
2559
|
// first call and reads back the same value on recovery, so it never changes across iterations.
|
|
2049
2560
|
if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
|
|
2050
|
-
finishTime = await this
|
|
2561
|
+
finishTime = await this.durableSleepInternal(callerID, timerFuncID, timeoutms);
|
|
2051
2562
|
}
|
|
2052
2563
|
while (true) {
|
|
2053
2564
|
if (callerID)
|
|
2054
2565
|
await this.checkIfCanceledLimited(callerID);
|
|
2055
2566
|
let rows;
|
|
2056
2567
|
try {
|
|
2568
|
+
const params = [];
|
|
2569
|
+
const target = this.wf('ws', workflowID, params);
|
|
2057
2570
|
({ rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT ws.status, COALESCE(wo.output, ws.output) AS output,
|
|
2058
2571
|
COALESCE(wo.error, ws.error) AS error, ws.serialization
|
|
2059
2572
|
FROM "${this.schemaName}".workflow_status ws
|
|
2060
|
-
LEFT JOIN "${this.schemaName}".workflow_output wo ON
|
|
2061
|
-
WHERE
|
|
2573
|
+
LEFT JOIN "${this.schemaName}".workflow_output wo ON ${this.wfJoin('ws', 'wo')}
|
|
2574
|
+
WHERE ${target}`, params)));
|
|
2062
2575
|
}
|
|
2063
2576
|
catch (e) {
|
|
2064
2577
|
const err = e;
|
|
@@ -2095,21 +2608,27 @@ class SystemDatabase {
|
|
|
2095
2608
|
}
|
|
2096
2609
|
}
|
|
2097
2610
|
async awaitFirstWorkflowId(workflowIds, callerID, pollingIntervalMs) {
|
|
2098
|
-
const placeholders = workflowIds.map((_, i) => `$${i + 1}`).join(', ');
|
|
2099
2611
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
2100
2612
|
while (true) {
|
|
2101
2613
|
if (callerID)
|
|
2102
2614
|
await this.checkIfCanceledLimited(callerID);
|
|
2103
|
-
const
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
LIMIT 1`, workflowIds));
|
|
2107
|
-
if (rows.length > 0) {
|
|
2108
|
-
return rows[0].workflow_uuid;
|
|
2615
|
+
const first = await this.checkFirstWorkflowId(workflowIds);
|
|
2616
|
+
if (first !== undefined) {
|
|
2617
|
+
return first;
|
|
2109
2618
|
}
|
|
2110
2619
|
await (0, utils_1.sleepms)(pollIntervalMs);
|
|
2111
2620
|
}
|
|
2112
2621
|
}
|
|
2622
|
+
/** The first of these workflows found complete, if any. */
|
|
2623
|
+
async checkFirstWorkflowId(workflowIds) {
|
|
2624
|
+
const params = [];
|
|
2625
|
+
const ids = this.wfIn(undefined, workflowIds, params);
|
|
2626
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
2627
|
+
WHERE ${ids}
|
|
2628
|
+
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
|
|
2629
|
+
LIMIT 1`, params));
|
|
2630
|
+
return rows[0]?.workflow_uuid;
|
|
2631
|
+
}
|
|
2113
2632
|
async awaitWorkflowIds(workflowIds, callerID, pollingIntervalMs) {
|
|
2114
2633
|
const remainingWorkflowIds = new Set(workflowIds);
|
|
2115
2634
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
@@ -2117,11 +2636,8 @@ class SystemDatabase {
|
|
|
2117
2636
|
const currentWorkflowIds = [...remainingWorkflowIds];
|
|
2118
2637
|
if (callerID)
|
|
2119
2638
|
await this.checkIfCanceledLimited(callerID);
|
|
2120
|
-
const
|
|
2121
|
-
|
|
2122
|
-
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, [currentWorkflowIds]));
|
|
2123
|
-
for (const row of rows) {
|
|
2124
|
-
remainingWorkflowIds.delete(row.workflow_uuid);
|
|
2639
|
+
for (const workflowID of await this.checkCompletedWorkflowIds(currentWorkflowIds)) {
|
|
2640
|
+
remainingWorkflowIds.delete(workflowID);
|
|
2125
2641
|
}
|
|
2126
2642
|
if (remainingWorkflowIds.size === 0) {
|
|
2127
2643
|
return;
|
|
@@ -2129,9 +2645,18 @@ class SystemDatabase {
|
|
|
2129
2645
|
await (0, utils_1.sleepms)(pollIntervalMs);
|
|
2130
2646
|
}
|
|
2131
2647
|
}
|
|
2648
|
+
/** Those of these workflows found complete. */
|
|
2649
|
+
async checkCompletedWorkflowIds(workflowIds) {
|
|
2650
|
+
const params = [];
|
|
2651
|
+
const ids = this.wfIn(undefined, workflowIds, params);
|
|
2652
|
+
const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
2653
|
+
WHERE ${ids}
|
|
2654
|
+
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, params));
|
|
2655
|
+
return rows.map((row) => row.workflow_uuid);
|
|
2656
|
+
}
|
|
2132
2657
|
// ==================== Sleep ====================
|
|
2133
2658
|
async durableSleepms(workflowID, functionID, durationMS) {
|
|
2134
|
-
const endTime = await this
|
|
2659
|
+
const endTime = await this.durableSleepInternal(workflowID, functionID, durationMS, true);
|
|
2135
2660
|
while (Date.now() < endTime) {
|
|
2136
2661
|
await (0, utils_1.sleepms)(Math.min(endTime - Date.now(), utils_1.sleepConfig.maxTimeoutMS));
|
|
2137
2662
|
}
|
|
@@ -2139,31 +2664,28 @@ class SystemDatabase {
|
|
|
2139
2664
|
}
|
|
2140
2665
|
// ==================== Messaging ====================
|
|
2141
2666
|
nullTopic = '__null__topic__';
|
|
2142
|
-
async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey
|
|
2143
|
-
|
|
2144
|
-
|
|
2667
|
+
async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey,
|
|
2668
|
+
// The destination's shard key; required on a sharded system database.
|
|
2669
|
+
shardKey) {
|
|
2145
2670
|
const startTime = Date.now();
|
|
2146
2671
|
let missing;
|
|
2147
2672
|
const client = await this.connect();
|
|
2148
2673
|
try {
|
|
2149
2674
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
2150
2675
|
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
|
|
2151
|
-
await
|
|
2152
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2153
|
-
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
2676
|
+
await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2154
2677
|
return undefined;
|
|
2155
2678
|
});
|
|
2156
2679
|
await client.query('COMMIT');
|
|
2157
2680
|
}
|
|
2158
2681
|
catch (error) {
|
|
2159
2682
|
await client.query('ROLLBACK');
|
|
2160
|
-
|
|
2161
|
-
if (err.code === '23503') {
|
|
2683
|
+
if (error instanceof error_1.DBOSNonExistentWorkflowError) {
|
|
2162
2684
|
// Foreign key constraint violation (only expected for the INSERT query)
|
|
2163
|
-
missing =
|
|
2685
|
+
missing = error;
|
|
2164
2686
|
}
|
|
2165
2687
|
else {
|
|
2166
|
-
throw
|
|
2688
|
+
throw error;
|
|
2167
2689
|
}
|
|
2168
2690
|
}
|
|
2169
2691
|
finally {
|
|
@@ -2176,22 +2698,22 @@ class SystemDatabase {
|
|
|
2176
2698
|
}
|
|
2177
2699
|
}
|
|
2178
2700
|
/** Runs on `client` if given, joining its transaction; otherwise on the pool with retries. */
|
|
2179
|
-
async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client) {
|
|
2701
|
+
async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client, shardKey) {
|
|
2180
2702
|
if (client !== undefined) {
|
|
2181
|
-
return await this
|
|
2703
|
+
return await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2182
2704
|
}
|
|
2183
|
-
return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey);
|
|
2705
|
+
return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2184
2706
|
}
|
|
2185
2707
|
/** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
|
|
2186
|
-
async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
|
|
2708
|
+
async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2187
2709
|
const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
|
|
2188
2710
|
const client = await this.connect();
|
|
2189
2711
|
try {
|
|
2190
|
-
await this
|
|
2191
|
-
await this
|
|
2712
|
+
await this.inTransaction(client, async () => {
|
|
2713
|
+
await this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2192
2714
|
// After the insert, in the order the recorded send locks, so they cannot deadlock.
|
|
2193
2715
|
if (ownerXid !== undefined) {
|
|
2194
|
-
await this
|
|
2716
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
2195
2717
|
}
|
|
2196
2718
|
});
|
|
2197
2719
|
}
|
|
@@ -2199,17 +2721,23 @@ class SystemDatabase {
|
|
|
2199
2721
|
client.release();
|
|
2200
2722
|
}
|
|
2201
2723
|
}
|
|
2202
|
-
async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey) {
|
|
2203
|
-
return await this
|
|
2724
|
+
async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2725
|
+
return await this.sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey, shardKey);
|
|
2204
2726
|
}
|
|
2205
|
-
async
|
|
2727
|
+
async sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
|
|
2206
2728
|
topic = topic ?? this.nullTopic;
|
|
2207
|
-
//
|
|
2729
|
+
// An idempotency key is scoped per destination: suffix it with the recipient's workflow ID.
|
|
2208
2730
|
const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
|
|
2731
|
+
const values = {
|
|
2732
|
+
...this.wfValues(destinationID, shardKey),
|
|
2733
|
+
destination_uuid: destinationID,
|
|
2734
|
+
topic,
|
|
2735
|
+
message,
|
|
2736
|
+
serialization,
|
|
2737
|
+
message_uuid: messageUUID,
|
|
2738
|
+
};
|
|
2209
2739
|
try {
|
|
2210
|
-
await
|
|
2211
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2212
|
-
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
2740
|
+
await this.insertRows(db, 'notifications', Object.keys(values), [Object.values(values)], `ON CONFLICT (${this.keyConflict('message_uuid')}) DO NOTHING`);
|
|
2213
2741
|
}
|
|
2214
2742
|
catch (error) {
|
|
2215
2743
|
const err = error;
|
|
@@ -2233,10 +2761,10 @@ class SystemDatabase {
|
|
|
2233
2761
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2234
2762
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2235
2763
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
|
|
2236
|
-
// Record the durable timeout deadline once before polling.
|
|
2764
|
+
// Record the durable timeout deadline once before polling. durableSleepInternal persists it on the
|
|
2237
2765
|
// first call and reads back the same value on recovery, so it never changes across iterations.
|
|
2238
2766
|
if (timeoutms) {
|
|
2239
|
-
finishTime = await this
|
|
2767
|
+
finishTime = await this.durableSleepInternal(workflowID, timeoutFunctionID, timeoutms);
|
|
2240
2768
|
}
|
|
2241
2769
|
while (true) {
|
|
2242
2770
|
// register the key with the global notifications listener.
|
|
@@ -2244,12 +2772,14 @@ class SystemDatabase {
|
|
|
2244
2772
|
const messagePromise = new Promise((resolve) => {
|
|
2245
2773
|
resolveNotification = resolve;
|
|
2246
2774
|
});
|
|
2247
|
-
const payload =
|
|
2775
|
+
const payload = this.waiter(workflowID, topic);
|
|
2248
2776
|
const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
|
|
2249
2777
|
try {
|
|
2250
2778
|
await this.checkIfCanceledLimited(workflowID);
|
|
2251
2779
|
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
2252
|
-
const
|
|
2780
|
+
const params = [topic];
|
|
2781
|
+
const target = this.wf(undefined, workflowID, params, 'destination_uuid');
|
|
2782
|
+
const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE ${target} AND topic=$1 AND consumed = false;`, params))).rows;
|
|
2253
2783
|
if (initRecvRows.length !== 0)
|
|
2254
2784
|
break;
|
|
2255
2785
|
const ct = Date.now();
|
|
@@ -2276,21 +2806,22 @@ class SystemDatabase {
|
|
|
2276
2806
|
const client = await this.connect();
|
|
2277
2807
|
try {
|
|
2278
2808
|
await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
|
|
2809
|
+
const consumeParams = [topic, functionID];
|
|
2279
2810
|
const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
|
|
2280
|
-
SET consumed = true, consumed_by_function_id = $
|
|
2281
|
-
WHERE
|
|
2282
|
-
AND topic = $
|
|
2811
|
+
SET consumed = true, consumed_by_function_id = $2
|
|
2812
|
+
WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
|
|
2813
|
+
AND topic = $1
|
|
2283
2814
|
AND consumed = false
|
|
2284
2815
|
AND message_uuid = (
|
|
2285
2816
|
SELECT message_uuid
|
|
2286
2817
|
FROM "${this.schemaName}".notifications
|
|
2287
|
-
WHERE
|
|
2288
|
-
AND topic = $
|
|
2818
|
+
WHERE ${this.wf(undefined, workflowID, consumeParams, 'destination_uuid')}
|
|
2819
|
+
AND topic = $1
|
|
2289
2820
|
AND consumed = false
|
|
2290
2821
|
ORDER BY created_at_epoch_ms ASC
|
|
2291
2822
|
LIMIT 1
|
|
2292
2823
|
)
|
|
2293
|
-
RETURNING notifications.message, notifications.serialization;`,
|
|
2824
|
+
RETURNING notifications.message, notifications.serialization;`, consumeParams)).rows;
|
|
2294
2825
|
if (finalRecvRows.length > 0) {
|
|
2295
2826
|
message = finalRecvRows[0].message;
|
|
2296
2827
|
serialization = finalRecvRows[0].serialization;
|
|
@@ -2319,16 +2850,19 @@ class SystemDatabase {
|
|
|
2319
2850
|
// Only a real write (not a replay) should wake readers.
|
|
2320
2851
|
let didWrite = false;
|
|
2321
2852
|
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SETEVENT, workflowID, functionID, async () => {
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2853
|
+
const keys = this.wfValues(workflowID);
|
|
2854
|
+
const keyColumns = Object.keys(keys).map((c) => `${c}, `);
|
|
2855
|
+
const keyPlaceholders = (from) => Object.keys(keys).map((_, i) => `$${from + i}, `);
|
|
2856
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_events (${keyColumns.join('')}workflow_uuid, key, value, serialization)
|
|
2857
|
+
VALUES (${keyPlaceholders(5).join('')}$1, $2, $3, $4)
|
|
2858
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'key')})
|
|
2325
2859
|
DO UPDATE SET value = $3, serialization = $4
|
|
2326
|
-
RETURNING workflow_uuid;`, [workflowID, key, message, serialization]);
|
|
2860
|
+
RETURNING workflow_uuid;`, [workflowID, key, message, serialization, ...Object.values(keys)]);
|
|
2327
2861
|
// Also write to the immutable history table for fork support
|
|
2328
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (workflow_uuid, function_id, key, value, serialization)
|
|
2329
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
2330
|
-
ON CONFLICT (workflow_uuid, function_id, key)
|
|
2331
|
-
DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization]);
|
|
2862
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (${keyColumns.join('')}workflow_uuid, function_id, key, value, serialization)
|
|
2863
|
+
VALUES (${keyPlaceholders(6).join('')}$1, $2, $3, $4, $5)
|
|
2864
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id', 'key')})
|
|
2865
|
+
DO UPDATE SET value = $4, serialization = $5;`, [workflowID, functionID, key, message, serialization, ...Object.values(keys)]);
|
|
2332
2866
|
didWrite = true;
|
|
2333
2867
|
return undefined;
|
|
2334
2868
|
});
|
|
@@ -2362,15 +2896,15 @@ class SystemDatabase {
|
|
|
2362
2896
|
// Get the return the value. if it's in the DB, otherwise return null.
|
|
2363
2897
|
let value = null;
|
|
2364
2898
|
let valueSer = null;
|
|
2365
|
-
const payloadKey =
|
|
2899
|
+
const payloadKey = this.waiter(workflowID, key);
|
|
2366
2900
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
2367
2901
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
2368
2902
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
|
|
2369
2903
|
// If we have a callerWorkflow, we want a durable sleep, otherwise, not. Record the durable
|
|
2370
|
-
// timeout deadline once before polling.
|
|
2904
|
+
// timeout deadline once before polling. durableSleepInternal persists it on the first call and reads
|
|
2371
2905
|
// back the same value on recovery, so it never changes across iterations.
|
|
2372
2906
|
if (callerWorkflow && timeoutms) {
|
|
2373
|
-
finishTime = await this
|
|
2907
|
+
finishTime = await this.durableSleepInternal(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms);
|
|
2374
2908
|
}
|
|
2375
2909
|
// Register the key with the global notifications listener first... we do not want to look in the DB first
|
|
2376
2910
|
// or that would cause a timing hole.
|
|
@@ -2384,9 +2918,11 @@ class SystemDatabase {
|
|
|
2384
2918
|
if (callerWorkflow?.workflowID)
|
|
2385
2919
|
await this.checkIfCanceledLimited(callerWorkflow?.workflowID);
|
|
2386
2920
|
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
2921
|
+
const params = [key];
|
|
2922
|
+
const target = this.wf(undefined, workflowID, params);
|
|
2387
2923
|
const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT key, value, serialization
|
|
2388
2924
|
FROM "${this.schemaName}".workflow_events
|
|
2389
|
-
WHERE
|
|
2925
|
+
WHERE ${target} AND key=$1;`, params))).rows;
|
|
2390
2926
|
if (initRecvRows.length > 0) {
|
|
2391
2927
|
value = initRecvRows[0].value;
|
|
2392
2928
|
valueSer = initRecvRows[0].serialization;
|
|
@@ -2425,15 +2961,12 @@ class SystemDatabase {
|
|
|
2425
2961
|
try {
|
|
2426
2962
|
while (true) {
|
|
2427
2963
|
try {
|
|
2428
|
-
await this
|
|
2964
|
+
await this.inTransaction(client, async () => {
|
|
2429
2965
|
// Derives the first unused offset inside the insert; two writers can still pick the same one.
|
|
2430
|
-
await
|
|
2431
|
-
SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
|
|
2432
|
-
FROM "${this.schemaName}".streams s
|
|
2433
|
-
WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
|
|
2966
|
+
await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
|
|
2434
2967
|
// After the insert, in the order the workflow-level writes lock, so they cannot deadlock.
|
|
2435
2968
|
if (ownerXid !== undefined) {
|
|
2436
|
-
await this
|
|
2969
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
2437
2970
|
}
|
|
2438
2971
|
});
|
|
2439
2972
|
}
|
|
@@ -2467,10 +3000,7 @@ class SystemDatabase {
|
|
|
2467
3000
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
2468
3001
|
await this.#runAndRecordResult(client, functionName, workflowID, functionID, async () => {
|
|
2469
3002
|
// Derives the first unused offset inside the insert; two writers can still pick the same one.
|
|
2470
|
-
await
|
|
2471
|
-
SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
|
|
2472
|
-
FROM "${this.schemaName}".streams s
|
|
2473
|
-
WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
|
|
3003
|
+
await this.#insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization);
|
|
2474
3004
|
didWrite = true;
|
|
2475
3005
|
return undefined;
|
|
2476
3006
|
});
|
|
@@ -2502,6 +3032,15 @@ class SystemDatabase {
|
|
|
2502
3032
|
client.release();
|
|
2503
3033
|
}
|
|
2504
3034
|
}
|
|
3035
|
+
/** Append a value to a stream, at the first unused offset, derived inside the insert. */
|
|
3036
|
+
async #insertStreamValue(client, workflowID, functionID, key, serializedValue, serialization) {
|
|
3037
|
+
const params = [key, serializedValue, functionID, serialization, workflowID];
|
|
3038
|
+
const keyColumns = this.wfColumns(workflowID, params);
|
|
3039
|
+
await client.query(`INSERT INTO "${this.schemaName}".streams (${keyColumns.map(([c]) => `${c}, `).join('')}workflow_uuid, key, value, "offset", function_id, serialization)
|
|
3040
|
+
SELECT ${keyColumns.map(([, e]) => `${e}, `).join('')}$5::text, $1::text, $2::text, COALESCE(MAX(s."offset"), -1) + 1, $3::int, $4::text
|
|
3041
|
+
FROM "${this.schemaName}".streams s
|
|
3042
|
+
WHERE ${this.wf('s', workflowID, params)} AND s.key = $1`, params);
|
|
3043
|
+
}
|
|
2505
3044
|
async closeStreamFromWorkflow(workflowID, functionID, key) {
|
|
2506
3045
|
await this.writeStreamFromWorkflow(workflowID, functionID, key, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name(), exports.DBOS_FUNCNAME_CLOSESTREAM);
|
|
2507
3046
|
}
|
|
@@ -2511,13 +3050,15 @@ class SystemDatabase {
|
|
|
2511
3050
|
// Read the value at `offset` and the workflow's status in one query: status null = no such workflow, value undefined = nothing at that offset.
|
|
2512
3051
|
async readStreamValue(workflowID, key, offset) {
|
|
2513
3052
|
// LEFT JOIN so a workflow with nothing at offset still reports its status (single PK lookup); under the poll limiter, inside @dbRetry so the permit frees across backoff.
|
|
3053
|
+
const params = [key, offset];
|
|
3054
|
+
const target = this.wf('ws', workflowID, params);
|
|
2514
3055
|
const result = await this.#pollWithLimiter(() => this.pool.query(
|
|
2515
3056
|
// "offset" is a reserved word, so alias it (stream_offset) to read it back plainly.
|
|
2516
3057
|
`SELECT ws.status AS status, s.value AS value, s.serialization AS serialization, s."offset" AS stream_offset
|
|
2517
3058
|
FROM "${this.schemaName}".workflow_status ws
|
|
2518
3059
|
LEFT OUTER JOIN "${this.schemaName}".streams s
|
|
2519
|
-
ON s
|
|
2520
|
-
WHERE
|
|
3060
|
+
ON ${this.wfJoin('s', 'ws')} AND s.key = $1 AND s."offset" = $2
|
|
3061
|
+
WHERE ${target}`, params));
|
|
2521
3062
|
if (result.rows.length === 0) {
|
|
2522
3063
|
return { status: null, value: undefined };
|
|
2523
3064
|
}
|
|
@@ -2654,7 +3195,7 @@ class SystemDatabase {
|
|
|
2654
3195
|
// Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v3, so cost scales with the number of partitions rather than the backlog depth.
|
|
2655
3196
|
const params = [queueName, workflow_1.StatusString.ENQUEUED];
|
|
2656
3197
|
// Only partitions this application can actually dequeue from.
|
|
2657
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3198
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2658
3199
|
const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
|
|
2659
3200
|
(SELECT MIN(queue_partition_key) AS pk
|
|
2660
3201
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -2671,7 +3212,14 @@ class SystemDatabase {
|
|
|
2671
3212
|
}
|
|
2672
3213
|
async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0,
|
|
2673
3214
|
// Written to every claimed row: the dispatches of this claim own their executions.
|
|
2674
|
-
ownerXid = null) {
|
|
3215
|
+
ownerXid = null, isLatestVersion) {
|
|
3216
|
+
const { claimedIDs } = await this.claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount, partitionLocalRunningCount, ownerXid, isLatestVersion);
|
|
3217
|
+
return claimedIDs;
|
|
3218
|
+
}
|
|
3219
|
+
/** Claim this queue's next workflows, plus the limit that left no room to claim any (undefined if every limit had room). */
|
|
3220
|
+
async claimStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0, ownerXid = null, isLatestVersion) {
|
|
3221
|
+
// Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
|
|
3222
|
+
const startTimeMs = Date.now();
|
|
2675
3223
|
const claimedIDs = [];
|
|
2676
3224
|
const partitionParams = queuePartitionKey !== undefined ? [queuePartitionKey] : [];
|
|
2677
3225
|
// Shares a concurrency or rate limit budget with other executors.
|
|
@@ -2679,13 +3227,49 @@ class SystemDatabase {
|
|
|
2679
3227
|
queue.partitionConcurrency !== undefined ||
|
|
2680
3228
|
queue.rateLimit !== undefined ||
|
|
2681
3229
|
queue.partitionRateLimit !== undefined;
|
|
2682
|
-
// Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
|
|
3230
|
+
// Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it: lock the whole queue.
|
|
2683
3231
|
const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
|
|
3232
|
+
// Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
|
|
3233
|
+
let maxTasks = Infinity;
|
|
3234
|
+
// The first limit found with no room, which the caller reads as why nothing was claimed.
|
|
3235
|
+
let stopReason = undefined;
|
|
3236
|
+
const limit = (room, reason) => {
|
|
3237
|
+
maxTasks = Math.min(maxTasks, room);
|
|
3238
|
+
if (maxTasks <= 0)
|
|
3239
|
+
stopReason ??= reason;
|
|
3240
|
+
};
|
|
3241
|
+
// This worker's own limits need no database, so a full worker returns before taking the claim lock a peer could use.
|
|
3242
|
+
if (queue.workerConcurrency !== undefined) {
|
|
3243
|
+
// Use the in-memory registry for this worker's running count — avoids a DB round trip.
|
|
3244
|
+
limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
|
|
3245
|
+
}
|
|
3246
|
+
if (queue.partitionWorkerConcurrency !== undefined) {
|
|
3247
|
+
limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
|
|
3248
|
+
}
|
|
3249
|
+
if (maxTasks <= 0) {
|
|
3250
|
+
return { claimedIDs, stopReason };
|
|
3251
|
+
}
|
|
3252
|
+
// A sharded database keeps the version row under another key, which the claim's transaction cannot span.
|
|
3253
|
+
if (isLatestVersion === undefined && this.sharded) {
|
|
3254
|
+
isLatestVersion = await this.isLatestVersion(appVersion);
|
|
3255
|
+
}
|
|
3256
|
+
// Row locks only collide when claimers see the same queue head, so serialize claims on the budget's scope; CockroachDB has no advisory locks before 26.3.
|
|
3257
|
+
const useClaimLock = hasSharedBudget && (this.sharded || !(await this.#isCockroach()));
|
|
2684
3258
|
const client = await this.connect();
|
|
2685
3259
|
try {
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
3260
|
+
if (useClaimLock) {
|
|
3261
|
+
await client.query('BEGIN');
|
|
3262
|
+
const lockParams = [];
|
|
3263
|
+
const { rows } = await client.query(this.claimLockQuery(queue.name, hasWriteSkew ? undefined : queuePartitionKey, lockParams), lockParams);
|
|
3264
|
+
if (rows[0]?.acquired !== true) {
|
|
3265
|
+
await client.query('COMMIT');
|
|
3266
|
+
// Another worker is claiming from this scope, so it still has work; the next poll retries.
|
|
3267
|
+
return rows.length > 0 ? { claimedIDs, stopReason: 'claim_lock' } : { claimedIDs };
|
|
3268
|
+
}
|
|
3269
|
+
}
|
|
3270
|
+
else if (hasSharedBudget) {
|
|
3271
|
+
// CockroachDB, without advisory locks before 26.3: SERIALIZABLE aborts claimers that spend the same budget.
|
|
3272
|
+
await client.query('BEGIN ISOLATION LEVEL SERIALIZABLE');
|
|
2689
3273
|
}
|
|
2690
3274
|
else {
|
|
2691
3275
|
await client.query('BEGIN');
|
|
@@ -2694,7 +3278,7 @@ class SystemDatabase {
|
|
|
2694
3278
|
const rateLimitRemaining = async (rateLimit, partitionScoped) => {
|
|
2695
3279
|
const params = [queue.name, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED, rateLimit.periodSec * 1000];
|
|
2696
3280
|
// Count only what this application would dequeue, matching the select below.
|
|
2697
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3281
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2698
3282
|
const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
|
|
2699
3283
|
const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
2700
3284
|
WHERE queue_name = $1
|
|
@@ -2713,35 +3297,26 @@ class SystemDatabase {
|
|
|
2713
3297
|
*/
|
|
2714
3298
|
const pendingCount = async (partitionScoped) => {
|
|
2715
3299
|
const params = [queue.name, workflow_1.StatusString.PENDING];
|
|
2716
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
3300
|
+
const scope = `${this.scoped(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
2717
3301
|
const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
|
|
2718
3302
|
const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
2719
3303
|
WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
|
|
2720
3304
|
return Number(rows[0]?.count ?? 0);
|
|
2721
3305
|
};
|
|
2722
|
-
// Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
|
|
2723
|
-
let maxTasks = Infinity;
|
|
2724
|
-
if (queue.workerConcurrency !== undefined) {
|
|
2725
|
-
// Use the in-memory registry for this worker's running count — avoids a DB round trip.
|
|
2726
|
-
maxTasks = Math.min(maxTasks, Math.max(0, queue.workerConcurrency - localRunningCount));
|
|
2727
|
-
}
|
|
2728
|
-
if (queue.partitionWorkerConcurrency !== undefined) {
|
|
2729
|
-
maxTasks = Math.min(maxTasks, Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount));
|
|
2730
|
-
}
|
|
2731
|
-
if (maxTasks <= 0) {
|
|
2732
|
-
await client.query('COMMIT');
|
|
2733
|
-
return claimedIDs;
|
|
2734
|
-
}
|
|
2735
3306
|
if (queue.rateLimit !== undefined) {
|
|
2736
3307
|
// Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
|
|
2737
|
-
|
|
3308
|
+
limit(await rateLimitRemaining(queue.rateLimit, false), 'limiter');
|
|
3309
|
+
if (maxTasks <= 0) {
|
|
3310
|
+
await client.query('COMMIT');
|
|
3311
|
+
return { claimedIDs, stopReason };
|
|
3312
|
+
}
|
|
2738
3313
|
}
|
|
2739
3314
|
if (queue.partitionRateLimit !== undefined) {
|
|
2740
|
-
|
|
3315
|
+
limit(await rateLimitRemaining(queue.partitionRateLimit, true), 'partition_limiter');
|
|
2741
3316
|
}
|
|
2742
3317
|
if (maxTasks <= 0) {
|
|
2743
3318
|
await client.query('COMMIT');
|
|
2744
|
-
return claimedIDs;
|
|
3319
|
+
return { claimedIDs, stopReason };
|
|
2745
3320
|
}
|
|
2746
3321
|
if (queue.concurrency !== undefined) {
|
|
2747
3322
|
// Global concurrency still requires a DB query since other workers may be running workflows too.
|
|
@@ -2749,23 +3324,26 @@ class SystemDatabase {
|
|
|
2749
3324
|
if (totalRunningTasks > queue.concurrency) {
|
|
2750
3325
|
this.logger.warn(`Total running tasks (${totalRunningTasks}) exceeds the global concurrency limit (${queue.concurrency})`);
|
|
2751
3326
|
}
|
|
2752
|
-
|
|
3327
|
+
limit(Math.max(0, queue.concurrency - totalRunningTasks), 'concurrency');
|
|
3328
|
+
if (maxTasks <= 0) {
|
|
3329
|
+
await client.query('COMMIT');
|
|
3330
|
+
return { claimedIDs, stopReason };
|
|
3331
|
+
}
|
|
2753
3332
|
}
|
|
2754
3333
|
if (queue.partitionConcurrency !== undefined) {
|
|
2755
3334
|
const partitionRunningTasks = await pendingCount(true);
|
|
2756
3335
|
if (partitionRunningTasks > queue.partitionConcurrency) {
|
|
2757
3336
|
this.logger.warn(`Total running tasks (${partitionRunningTasks}) on partition ${queuePartitionKey} of queue ${queue.name} exceeds the partition concurrency limit (${queue.partitionConcurrency})`);
|
|
2758
3337
|
}
|
|
2759
|
-
|
|
3338
|
+
limit(Math.max(0, queue.partitionConcurrency - partitionRunningTasks), 'partition_concurrency');
|
|
2760
3339
|
}
|
|
2761
3340
|
// Return immediately if there are no available tasks due to flow control limits
|
|
2762
3341
|
if (maxTasks <= 0) {
|
|
2763
3342
|
await client.query('COMMIT');
|
|
2764
|
-
return claimedIDs;
|
|
3343
|
+
return { claimedIDs, stopReason };
|
|
2765
3344
|
}
|
|
2766
3345
|
// Retrieve the first max_tasks workflows in the queue.
|
|
2767
|
-
|
|
2768
|
-
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
3346
|
+
isLatestVersion ??= await this.isLatestVersion(appVersion, client);
|
|
2769
3347
|
const versionClause = isLatestVersion
|
|
2770
3348
|
? '(application_version = $3 OR application_version IS NULL)'
|
|
2771
3349
|
: 'application_version = $3';
|
|
@@ -2775,13 +3353,17 @@ class SystemDatabase {
|
|
|
2775
3353
|
const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
|
|
2776
3354
|
const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
|
|
2777
3355
|
const selectScope = this.appNameFilter('application_name', this.appName, selectParams);
|
|
3356
|
+
const keyScope = this.scoped(undefined, selectParams);
|
|
3357
|
+
const claimGate = this.claimGate(queue.name, queuePartitionKey, selectParams);
|
|
2778
3358
|
const selectQuery = `
|
|
2779
3359
|
SELECT workflow_uuid
|
|
2780
3360
|
FROM "${this.schemaName}".workflow_status
|
|
2781
|
-
WHERE
|
|
3361
|
+
WHERE ${keyScope}
|
|
3362
|
+
AND status = $1
|
|
2782
3363
|
AND queue_name = $2
|
|
2783
3364
|
AND ${versionClause}
|
|
2784
3365
|
AND ${selectScope}
|
|
3366
|
+
AND ${claimGate}
|
|
2785
3367
|
${queuePartitionKey !== undefined ? 'AND queue_partition_key = $4' : ''}
|
|
2786
3368
|
ORDER BY priority ASC, created_at ASC
|
|
2787
3369
|
${limitClause}
|
|
@@ -2800,35 +3382,36 @@ class SystemDatabase {
|
|
|
2800
3382
|
executorID,
|
|
2801
3383
|
appVersion,
|
|
2802
3384
|
queue.rateLimit !== undefined || queue.partitionRateLimit !== undefined,
|
|
2803
|
-
workflowIDs,
|
|
2804
3385
|
workflow_1.StatusString.ENQUEUED,
|
|
2805
3386
|
// Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
|
|
2806
3387
|
this.appName ?? null,
|
|
2807
3388
|
ownerXid,
|
|
3389
|
+
startTimeMs,
|
|
2808
3390
|
];
|
|
3391
|
+
const claimedIDsFilter = this.wfIn(undefined, workflowIDs, updateParams);
|
|
2809
3392
|
// Re-check ownership alongside status, as the partitioned claim guard does.
|
|
2810
3393
|
const claimScope = this.appNameFilter('application_name', this.appName, updateParams);
|
|
2811
3394
|
// RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
|
|
2812
3395
|
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2813
3396
|
SET status = $1,
|
|
2814
|
-
owner_xid = $
|
|
3397
|
+
owner_xid = $7,
|
|
2815
3398
|
executor_id = $2,
|
|
2816
3399
|
application_version = $3,
|
|
2817
3400
|
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2818
3401
|
rate_limited = $4,
|
|
2819
|
-
application_name = COALESCE(application_name, $
|
|
3402
|
+
application_name = COALESCE(application_name, $6),
|
|
2820
3403
|
recovery_attempts = recovery_attempts + 1,
|
|
2821
3404
|
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2822
3405
|
workflow_deadline_epoch_ms = CASE
|
|
2823
3406
|
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
2824
3407
|
CASE
|
|
2825
|
-
WHEN workflow_timeout_ms > 9223372036854775807 -
|
|
3408
|
+
WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
|
|
2826
3409
|
THEN 9223372036854775807
|
|
2827
|
-
ELSE
|
|
3410
|
+
ELSE $8::bigint + workflow_timeout_ms
|
|
2828
3411
|
END
|
|
2829
3412
|
ELSE workflow_deadline_epoch_ms
|
|
2830
3413
|
END
|
|
2831
|
-
WHERE
|
|
3414
|
+
WHERE ${claimedIDsFilter} AND status = $5 AND ${claimScope}
|
|
2832
3415
|
RETURNING workflow_uuid`, updateParams);
|
|
2833
3416
|
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
2834
3417
|
for (const id of workflowIDs) {
|
|
@@ -2846,14 +3429,18 @@ class SystemDatabase {
|
|
|
2846
3429
|
client.release();
|
|
2847
3430
|
}
|
|
2848
3431
|
// Return the IDs of all functions we marked started
|
|
2849
|
-
return claimedIDs;
|
|
3432
|
+
return { claimedIDs };
|
|
2850
3433
|
}
|
|
2851
3434
|
/** Max heads admitted per sweep: bounds dispatch, not the partition walk; an unconstrained sweep takes the lowest keys, so higher keys can wait under sustained load. */
|
|
2852
3435
|
partitionedDequeueSweepCap = 8192;
|
|
2853
|
-
/**
|
|
3436
|
+
/** Max partitions a sweep claims per transaction, which bounds the locks it holds at once and for how long. */
|
|
3437
|
+
partitionedDequeueBatchSize = 64;
|
|
3438
|
+
/** Dequeue each partition's head-of-line workflow, at most {@link partitionedDequeueSweepCap} per sweep in transactions of {@link partitionedDequeueBatchSize}; only valid for partition-concurrency-1 queues with no queue-wide limit. */
|
|
2854
3439
|
async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity,
|
|
2855
3440
|
// Written to every claimed row: the dispatches of this claim own their executions.
|
|
2856
|
-
ownerXid = null) {
|
|
3441
|
+
ownerXid = null, isLatestVersion) {
|
|
3442
|
+
// Bound rather than now() in the deadline CASE, which Citus refuses for a non-immutable function.
|
|
3443
|
+
const startTimeMs = Date.now();
|
|
2857
3444
|
if (queue.partitionConcurrency !== 1 ||
|
|
2858
3445
|
queue.concurrency !== undefined ||
|
|
2859
3446
|
queue.rateLimit !== undefined ||
|
|
@@ -2861,14 +3448,35 @@ class SystemDatabase {
|
|
|
2861
3448
|
throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
|
|
2862
3449
|
}
|
|
2863
3450
|
// partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
|
|
3451
|
+
// A sharded database keeps the version row under another key, which the claim's transaction cannot span.
|
|
3452
|
+
if (isLatestVersion === undefined && this.sharded) {
|
|
3453
|
+
isLatestVersion = await this.isLatestVersion(appVersion);
|
|
3454
|
+
}
|
|
3455
|
+
// Sweeps whose heads differ lock disjoint rows, so a claim also takes its partition's lock; CockroachDB, without advisory locks before 26.3, relies on its SERIALIZABLE default.
|
|
3456
|
+
const usePartitionLocks = this.sharded || !(await this.#isCockroach());
|
|
2864
3457
|
const client = await this.connect();
|
|
2865
3458
|
try {
|
|
2866
|
-
await
|
|
2867
|
-
const latestVersion = await this.#latestApplicationVersionName(client);
|
|
2868
|
-
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
3459
|
+
isLatestVersion ??= await this.isLatestVersion(appVersion, client);
|
|
2869
3460
|
const versionClause = (n) => isLatestVersion
|
|
2870
3461
|
? `(application_version = $${n} OR application_version IS NULL)`
|
|
2871
3462
|
: `application_version = $${n}`;
|
|
3463
|
+
// The partition's head: $1 queue, $2 ENQUEUED, `version` the app version, `pk` the partition key expression.
|
|
3464
|
+
const headOf = (pk, version, scope) => `SELECT workflow_uuid
|
|
3465
|
+
FROM "${this.schemaName}".workflow_status
|
|
3466
|
+
WHERE queue_name = $1 AND status = $2
|
|
3467
|
+
AND queue_partition_key = ${pk}
|
|
3468
|
+
AND ${versionClause(version)}
|
|
3469
|
+
AND ${scope}
|
|
3470
|
+
-- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
|
|
3471
|
+
ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
|
|
3472
|
+
LIMIT 1`;
|
|
3473
|
+
// No application filter by design: a mutual-exclusion probe must block on any owner's row.
|
|
3474
|
+
const noPending = (pk, pending, scope) => `NOT EXISTS (
|
|
3475
|
+
SELECT 1
|
|
3476
|
+
FROM "${this.schemaName}".workflow_status
|
|
3477
|
+
WHERE ${scope} AND queue_name = $1 AND status = $${pending}
|
|
3478
|
+
AND queue_partition_key IS NOT NULL AND queue_partition_key = ${pk}
|
|
3479
|
+
)`;
|
|
2872
3480
|
// This worker's own budget bounds the sweep alongside the cap.
|
|
2873
3481
|
const sweepLimit = Math.min(this.partitionedDequeueSweepCap, maxTasks);
|
|
2874
3482
|
// When the worker's budget is the binding constraint, probe partitions in random order to prevent starvation.
|
|
@@ -2880,8 +3488,10 @@ class SystemDatabase {
|
|
|
2880
3488
|
workflow_1.StatusString.PENDING,
|
|
2881
3489
|
sweepLimit,
|
|
2882
3490
|
];
|
|
2883
|
-
const candidateScope = this.appNameFilter('application_name', this.appName, candidateParams)
|
|
3491
|
+
const candidateScope = `${this.scoped(undefined, candidateParams)} AND ${this.appNameFilter('application_name', this.appName, candidateParams)}`;
|
|
3492
|
+
const probeScope = this.scoped(undefined, candidateParams);
|
|
2884
3493
|
// Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
|
|
3494
|
+
// The walk takes no lock: each batch below picks its heads again under its partitions' locks.
|
|
2885
3495
|
const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
|
|
2886
3496
|
(SELECT MIN(queue_partition_key) AS pk
|
|
2887
3497
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -2897,118 +3507,140 @@ class SystemDatabase {
|
|
|
2897
3507
|
SELECT partitions.pk
|
|
2898
3508
|
FROM partitions
|
|
2899
3509
|
WHERE partitions.pk IS NOT NULL
|
|
2900
|
-
|
|
2901
|
-
AND NOT EXISTS (
|
|
2902
|
-
SELECT 1
|
|
2903
|
-
FROM "${this.schemaName}".workflow_status
|
|
2904
|
-
WHERE queue_name = $1 AND status = $4
|
|
2905
|
-
AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
|
|
2906
|
-
)
|
|
3510
|
+
AND ${noPending('partitions.pk', 4, probeScope)}
|
|
2907
3511
|
ORDER BY ${sweepOrder}
|
|
2908
3512
|
LIMIT $5
|
|
2909
3513
|
)
|
|
2910
|
-
SELECT
|
|
3514
|
+
SELECT chosen.pk
|
|
2911
3515
|
FROM chosen
|
|
2912
|
-
--
|
|
2913
|
-
JOIN LATERAL (
|
|
2914
|
-
|
|
2915
|
-
FROM "${this.schemaName}".workflow_status
|
|
2916
|
-
WHERE queue_name = $1 AND status = $2
|
|
2917
|
-
AND queue_partition_key = chosen.pk
|
|
2918
|
-
AND ${versionClause(3)}
|
|
2919
|
-
AND ${candidateScope}
|
|
2920
|
-
-- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
|
|
2921
|
-
ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
|
|
2922
|
-
LIMIT 1
|
|
2923
|
-
) head ON TRUE
|
|
2924
|
-
-- Which partitions were chosen is settled above; order the result so the claim, and the dispatch it feeds, are deterministic.
|
|
3516
|
+
-- Only partitions with a head this worker may run; LATERAL plans as a tight nested loop.
|
|
3517
|
+
JOIN LATERAL (${headOf('chosen.pk', 3, candidateScope)}) head ON TRUE
|
|
3518
|
+
-- Which partitions were chosen is settled above; order them so the claim, and the dispatch it feeds, are deterministic.
|
|
2925
3519
|
ORDER BY chosen.pk ASC`, candidateParams);
|
|
2926
|
-
const
|
|
2927
|
-
if (candidateIDs.length === 0) {
|
|
2928
|
-
await client.query('COMMIT');
|
|
2929
|
-
return [];
|
|
2930
|
-
}
|
|
2931
|
-
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
|
|
3520
|
+
const partitionKeys = candidateResult.rows.map((row) => row.pk);
|
|
2932
3521
|
// Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
|
|
2933
|
-
const claimGuard = (ids, status, name, version, scope) =>
|
|
3522
|
+
const claimGuard = (ids, status, name, version, scope) => `${ids}
|
|
2934
3523
|
AND status = $${status}
|
|
2935
3524
|
AND queue_name = $${name}
|
|
2936
3525
|
AND queue_partition_key IS NOT NULL
|
|
2937
3526
|
AND ${versionClause(version)}
|
|
2938
3527
|
AND ${scope}`;
|
|
2939
|
-
|
|
2940
|
-
const
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
3528
|
+
/** Claim the heads of these partitions inside the open transaction, returning the IDs it marked PENDING. */
|
|
3529
|
+
const claimBatch = async (batch) => {
|
|
3530
|
+
let heldPks = batch;
|
|
3531
|
+
if (usePartitionLocks) {
|
|
3532
|
+
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_BEFORE_LOCKS);
|
|
3533
|
+
const lockParams = [];
|
|
3534
|
+
const { rows } = await client.query(this.partitionLockQuery(queue.name, batch, lockParams), lockParams);
|
|
3535
|
+
// Partitions another sweep holds are left to it, or to the next poll.
|
|
3536
|
+
heldPks = rows[0]?.held ?? [];
|
|
3537
|
+
if (heldPks.length === 0)
|
|
3538
|
+
return [];
|
|
3539
|
+
}
|
|
3540
|
+
// Pick the heads again under the locks, so a previous holder's claims and any newly visible head are seen.
|
|
3541
|
+
const headParams = [queue.name, workflow_1.StatusString.ENQUEUED, appVersion, workflow_1.StatusString.PENDING];
|
|
3542
|
+
const headScope = `${this.scoped(undefined, headParams)} AND ${this.appNameFilter('application_name', this.appName, headParams)}`;
|
|
3543
|
+
const headProbeScope = this.scoped(undefined, headParams);
|
|
3544
|
+
headParams.push(heldPks);
|
|
3545
|
+
const headResult = await client.query(`SELECT head.workflow_uuid
|
|
3546
|
+
FROM unnest($${headParams.length}::text[]) AS chosen(pk)
|
|
3547
|
+
JOIN LATERAL (${headOf('chosen.pk', 3, headScope)}) head ON TRUE
|
|
3548
|
+
WHERE ${noPending('chosen.pk', 4, headProbeScope)}
|
|
3549
|
+
ORDER BY chosen.pk ASC`, headParams);
|
|
3550
|
+
const candidateIDs = headResult.rows.map((row) => row.workflow_uuid);
|
|
3551
|
+
if (candidateIDs.length === 0)
|
|
3552
|
+
return [];
|
|
3553
|
+
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
|
|
3554
|
+
const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
|
|
3555
|
+
const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
|
|
3556
|
+
const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
|
|
3557
|
+
// Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
|
|
3558
|
+
const lockedResult = await client.query(`SELECT workflow_uuid
|
|
3559
|
+
FROM "${this.schemaName}".workflow_status
|
|
3560
|
+
WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
|
|
3561
|
+
FOR UPDATE SKIP LOCKED`, lockedParams);
|
|
3562
|
+
const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
|
|
3563
|
+
// Preserve partition order for submission.
|
|
3564
|
+
const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
|
|
3565
|
+
if (claimIDs.length === 0)
|
|
3566
|
+
return [];
|
|
3567
|
+
const flipParams = [
|
|
3568
|
+
workflow_1.StatusString.PENDING,
|
|
3569
|
+
executorID,
|
|
3570
|
+
workflow_1.StatusString.ENQUEUED,
|
|
3571
|
+
queue.name,
|
|
3572
|
+
appVersion,
|
|
3573
|
+
// Claim the row, as the unpartitioned dequeue does.
|
|
3574
|
+
this.appName ?? null,
|
|
3575
|
+
ownerXid,
|
|
3576
|
+
startTimeMs,
|
|
3577
|
+
];
|
|
3578
|
+
const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
|
|
3579
|
+
const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
|
|
3580
|
+
// Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
|
|
3581
|
+
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
3582
|
+
SET status = $1,
|
|
3583
|
+
owner_xid = $7,
|
|
3584
|
+
executor_id = $2,
|
|
3585
|
+
application_version = $5,
|
|
3586
|
+
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
3587
|
+
rate_limited = FALSE,
|
|
3588
|
+
application_name = COALESCE(application_name, $6),
|
|
3589
|
+
recovery_attempts = recovery_attempts + 1,
|
|
3590
|
+
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
3591
|
+
workflow_deadline_epoch_ms = CASE
|
|
3592
|
+
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
3593
|
+
CASE
|
|
3594
|
+
WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
|
|
3595
|
+
THEN 9223372036854775807
|
|
3596
|
+
ELSE $8::bigint + workflow_timeout_ms
|
|
3597
|
+
END
|
|
3598
|
+
ELSE workflow_deadline_epoch_ms
|
|
3599
|
+
END
|
|
3600
|
+
WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
|
|
3601
|
+
RETURNING workflow_uuid`, flipParams);
|
|
3602
|
+
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
3603
|
+
return claimIDs.filter((id) => flippedIDs.has(id));
|
|
3604
|
+
};
|
|
3605
|
+
// One short transaction per batch, so a sweep holds at most a batch's locks, and only while it claims that batch.
|
|
3606
|
+
const claimedIDs = [];
|
|
3607
|
+
for (let start = 0; start < partitionKeys.length; start += this.partitionedDequeueBatchSize) {
|
|
3608
|
+
const batch = partitionKeys.slice(start, start + this.partitionedDequeueBatchSize);
|
|
3609
|
+
try {
|
|
3610
|
+
await client.query('BEGIN');
|
|
3611
|
+
const batchIDs = await claimBatch(batch);
|
|
3612
|
+
await client.query('COMMIT');
|
|
3613
|
+
claimedIDs.push(...batchIDs);
|
|
3614
|
+
}
|
|
3615
|
+
catch (error) {
|
|
3616
|
+
// Swallowed: a rollback failure must not mask why we are rolling back.
|
|
3617
|
+
await client.query('ROLLBACK').catch(() => undefined);
|
|
3618
|
+
// Earlier batches committed their claims, which must still reach dispatch; the next poll retries the rest.
|
|
3619
|
+
if (claimedIDs.length === 0)
|
|
3620
|
+
throw error;
|
|
3621
|
+
this.logger.warn(`[${queue.name}] stopping a partitioned sweep after ${claimedIDs.length} claim(s): ${String(error)}`);
|
|
3622
|
+
break;
|
|
3623
|
+
}
|
|
2952
3624
|
}
|
|
2953
|
-
const flipParams = [
|
|
2954
|
-
workflow_1.StatusString.PENDING,
|
|
2955
|
-
executorID,
|
|
2956
|
-
claimIDs,
|
|
2957
|
-
workflow_1.StatusString.ENQUEUED,
|
|
2958
|
-
queue.name,
|
|
2959
|
-
appVersion,
|
|
2960
|
-
// Claim the row, as the unpartitioned dequeue does.
|
|
2961
|
-
this.appName ?? null,
|
|
2962
|
-
ownerXid,
|
|
2963
|
-
];
|
|
2964
|
-
const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
|
|
2965
|
-
// Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
|
|
2966
|
-
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2967
|
-
SET status = $1,
|
|
2968
|
-
owner_xid = $8,
|
|
2969
|
-
executor_id = $2,
|
|
2970
|
-
application_version = $6,
|
|
2971
|
-
started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2972
|
-
rate_limited = FALSE,
|
|
2973
|
-
application_name = COALESCE(application_name, $7),
|
|
2974
|
-
recovery_attempts = recovery_attempts + 1,
|
|
2975
|
-
updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
|
|
2976
|
-
workflow_deadline_epoch_ms = CASE
|
|
2977
|
-
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
|
|
2978
|
-
CASE
|
|
2979
|
-
WHEN workflow_timeout_ms > 9223372036854775807 - (EXTRACT(epoch FROM now()) * 1000)::bigint
|
|
2980
|
-
THEN 9223372036854775807
|
|
2981
|
-
ELSE (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
|
|
2982
|
-
END
|
|
2983
|
-
ELSE workflow_deadline_epoch_ms
|
|
2984
|
-
END
|
|
2985
|
-
WHERE ${claimGuard(3, 4, 5, 6, flipScope)}
|
|
2986
|
-
RETURNING workflow_uuid`, flipParams);
|
|
2987
|
-
await client.query('COMMIT');
|
|
2988
|
-
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
2989
|
-
const claimedIDs = claimIDs.filter((id) => flippedIDs.has(id));
|
|
2990
3625
|
if (claimedIDs.length > 0) {
|
|
2991
3626
|
this.logger.debug(`[${queue.name}] dequeueing ${claimedIDs.length} task(s)`);
|
|
2992
3627
|
}
|
|
2993
3628
|
return claimedIDs;
|
|
2994
3629
|
}
|
|
2995
|
-
catch (error) {
|
|
2996
|
-
await client.query('ROLLBACK');
|
|
2997
|
-
throw error;
|
|
2998
|
-
}
|
|
2999
3630
|
finally {
|
|
3000
3631
|
client.release();
|
|
3001
3632
|
}
|
|
3002
3633
|
}
|
|
3003
3634
|
// ==================== Queries & Maintenance ====================
|
|
3004
|
-
|
|
3005
|
-
|
|
3635
|
+
/** With `keyedIDs`, the listed workflow IDs are addressed by their shard keys, as a batch read of known workflows. */
|
|
3636
|
+
async listWorkflows(input, options = {}) {
|
|
3637
|
+
return await this.#listWorkflows(input, options);
|
|
3006
3638
|
}
|
|
3007
3639
|
/**
|
|
3008
3640
|
* Unretried body of {@link listWorkflows}. An ID-keyed read runs on `client` when given, so a caller
|
|
3009
3641
|
* holding a transaction never waits on a second pool connection; the caller owns the retry.
|
|
3010
3642
|
*/
|
|
3011
|
-
async #listWorkflows(input, client) {
|
|
3643
|
+
async #listWorkflows(input, options = {}, client) {
|
|
3012
3644
|
const schemaName = this.schemaName;
|
|
3013
3645
|
const selectColumns = [
|
|
3014
3646
|
'workflow_uuid',
|
|
@@ -3050,16 +3682,23 @@ class SystemDatabase {
|
|
|
3050
3682
|
const payloadJoins = [];
|
|
3051
3683
|
if (input.loadInput) {
|
|
3052
3684
|
payloadColumns.push('COALESCE(wi.inputs, workflow_status.inputs) AS inputs');
|
|
3053
|
-
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON wi
|
|
3685
|
+
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON ${this.wfJoin('wi', 'workflow_status')}`);
|
|
3054
3686
|
}
|
|
3055
3687
|
if (input.loadOutput) {
|
|
3056
3688
|
payloadColumns.push('COALESCE(wo.output, workflow_status.output) AS output');
|
|
3057
3689
|
payloadColumns.push('COALESCE(wo.error, workflow_status.error) AS error');
|
|
3058
|
-
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON wo
|
|
3690
|
+
payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON ${this.wfJoin('wo', 'workflow_status')}`);
|
|
3059
3691
|
}
|
|
3060
3692
|
if (input.loadInput || input.loadOutput) {
|
|
3061
3693
|
selectColumns.push('serialization');
|
|
3062
3694
|
}
|
|
3695
|
+
const keyColumns = [];
|
|
3696
|
+
const keyColumn = this.keyColumn('workflow_status');
|
|
3697
|
+
if (keyColumn !== undefined)
|
|
3698
|
+
keyColumns.push(`${keyColumn} AS shard_key`);
|
|
3699
|
+
const parentKeyColumn = this.keyColumn('workflow_status', 'parent_shard_key');
|
|
3700
|
+
if (parentKeyColumn !== undefined)
|
|
3701
|
+
keyColumns.push(`${parentKeyColumn} AS parent_shard_key`);
|
|
3063
3702
|
input.sortDesc = input.sortDesc ?? false; // By default, sort in ascending order
|
|
3064
3703
|
// Build WHERE clauses
|
|
3065
3704
|
const whereClauses = [];
|
|
@@ -3102,6 +3741,18 @@ class SystemDatabase {
|
|
|
3102
3741
|
whereClauses.push(idKeyed
|
|
3103
3742
|
? this.appNameFilter('application_name', input.applicationName, params)
|
|
3104
3743
|
: this.observabilityFilter('application_name', input.applicationName, params));
|
|
3744
|
+
// An empty key list matches nothing, rather than every key.
|
|
3745
|
+
if (input.shardKey !== undefined && input.shardKey !== null) {
|
|
3746
|
+
const keys = Array.isArray(input.shardKey) ? input.shardKey : [input.shardKey];
|
|
3747
|
+
whereClauses.push(this.keyIn('workflow_status', keys, params));
|
|
3748
|
+
}
|
|
3749
|
+
if (input.parentShardKey !== undefined && input.parentShardKey !== null) {
|
|
3750
|
+
const keys = Array.isArray(input.parentShardKey) ? input.parentShardKey : [input.parentShardKey];
|
|
3751
|
+
whereClauses.push(this.keyIn('workflow_status', keys, params, 'parent_shard_key'));
|
|
3752
|
+
}
|
|
3753
|
+
if (options.keyedIDs && input.workflowIDs) {
|
|
3754
|
+
whereClauses.push(this.wfIn('workflow_status', input.workflowIDs, params));
|
|
3755
|
+
}
|
|
3105
3756
|
paramCounter = params.length + 1;
|
|
3106
3757
|
if (input.workflow_id_prefix) {
|
|
3107
3758
|
if (Array.isArray(input.workflow_id_prefix)) {
|
|
@@ -3116,7 +3767,7 @@ class SystemDatabase {
|
|
|
3116
3767
|
paramCounter++;
|
|
3117
3768
|
}
|
|
3118
3769
|
}
|
|
3119
|
-
if (input.workflowIDs) {
|
|
3770
|
+
if (input.workflowIDs && !options.keyedIDs) {
|
|
3120
3771
|
const placeholders = input.workflowIDs.map((_, i) => `$${paramCounter + i}`).join(', ');
|
|
3121
3772
|
whereClauses.push(`workflow_status.workflow_uuid IN (${placeholders})`);
|
|
3122
3773
|
params.push(...input.workflowIDs);
|
|
@@ -3198,7 +3849,7 @@ class SystemDatabase {
|
|
|
3198
3849
|
params.push(input.offset);
|
|
3199
3850
|
paramCounter++;
|
|
3200
3851
|
}
|
|
3201
|
-
const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns].join(', ');
|
|
3852
|
+
const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns, ...keyColumns].join(', ');
|
|
3202
3853
|
const query = `
|
|
3203
3854
|
SELECT ${projection}
|
|
3204
3855
|
FROM "${schemaName}".workflow_status
|
|
@@ -3218,23 +3869,16 @@ class SystemDatabase {
|
|
|
3218
3869
|
async createSchedule(schedule, client) {
|
|
3219
3870
|
const q = client ?? this.pool;
|
|
3220
3871
|
const owner = await this.#resolveRowOwner(q, 'workflow_schedules', 'schedule_name', schedule.scheduleName, schedule.applicationName, 'Schedule');
|
|
3872
|
+
const values = {
|
|
3873
|
+
...this.globalValues(),
|
|
3874
|
+
...this.scheduleValues(schedule),
|
|
3875
|
+
...scheduleRow(schedule, owner),
|
|
3876
|
+
};
|
|
3221
3877
|
try {
|
|
3222
|
-
await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
3223
|
-
(
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
schedule.scheduleName,
|
|
3227
|
-
schedule.workflowName,
|
|
3228
|
-
schedule.workflowClassName,
|
|
3229
|
-
schedule.schedule,
|
|
3230
|
-
schedule.status,
|
|
3231
|
-
schedule.context,
|
|
3232
|
-
schedule.lastFiredAt,
|
|
3233
|
-
schedule.automaticBackfill,
|
|
3234
|
-
schedule.cronTimezone,
|
|
3235
|
-
schedule.queueName,
|
|
3236
|
-
owner ?? null,
|
|
3237
|
-
]);
|
|
3878
|
+
await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
|
|
3879
|
+
VALUES (${Object.keys(values)
|
|
3880
|
+
.map((_, i) => `$${i + 1}`)
|
|
3881
|
+
.join(', ')})`, Object.values(values));
|
|
3238
3882
|
}
|
|
3239
3883
|
catch (e) {
|
|
3240
3884
|
if (isPgDatabaseError(e) && e.code === '23505') {
|
|
@@ -3281,31 +3925,37 @@ class SystemDatabase {
|
|
|
3281
3925
|
// Unset scopes to this application, as on every other observability query.
|
|
3282
3926
|
conditions.push(this.observabilityFilter('application_name', filters?.applicationName, params));
|
|
3283
3927
|
paramIdx = params.length + 1;
|
|
3928
|
+
conditions.push(this.global(undefined, params));
|
|
3284
3929
|
const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
|
|
3285
|
-
const result = await q.query(`SELECT ${
|
|
3930
|
+
const result = await q.query(`SELECT ${this.#scheduleColumns()}
|
|
3286
3931
|
FROM "${this.schemaName}".workflow_schedules${where}
|
|
3287
3932
|
ORDER BY schedule_name`, params);
|
|
3288
3933
|
return result.rows.map((row) => mapWorkflowSchedule(row));
|
|
3289
3934
|
}
|
|
3290
3935
|
async getSchedule(name, client) {
|
|
3291
3936
|
const q = client ?? this.pool;
|
|
3292
|
-
const
|
|
3937
|
+
const params = [name];
|
|
3938
|
+
const result = await q.query(`SELECT ${this.#scheduleColumns()}
|
|
3293
3939
|
FROM "${this.schemaName}".workflow_schedules
|
|
3294
|
-
WHERE schedule_name = $1`,
|
|
3940
|
+
WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
|
|
3295
3941
|
if (result.rows.length === 0)
|
|
3296
3942
|
return null;
|
|
3297
3943
|
return mapWorkflowSchedule(result.rows[0]);
|
|
3298
3944
|
}
|
|
3945
|
+
/** The columns a schedule is read back with, its workflows' shard key among them on a sharded database. */
|
|
3946
|
+
#scheduleColumns() {
|
|
3947
|
+
const keyColumn = this.scheduleKeyColumn();
|
|
3948
|
+
return keyColumn !== undefined ? `${SCHEDULE_COLUMNS}, ${keyColumn}` : SCHEDULE_COLUMNS;
|
|
3949
|
+
}
|
|
3299
3950
|
async deleteSchedule(name, client) {
|
|
3300
3951
|
const q = client ?? this.pool;
|
|
3301
|
-
|
|
3952
|
+
const params = [name];
|
|
3953
|
+
await q.query(`DELETE FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, params)} AND schedule_name = $1`, params);
|
|
3302
3954
|
}
|
|
3303
3955
|
async setScheduleStatus(name, status, client) {
|
|
3304
3956
|
const q = client ?? this.pool;
|
|
3305
|
-
|
|
3306
|
-
|
|
3307
|
-
name,
|
|
3308
|
-
]);
|
|
3957
|
+
const params = [status, name];
|
|
3958
|
+
await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET status = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
|
|
3309
3959
|
}
|
|
3310
3960
|
async updateSchedule(name, updates, client) {
|
|
3311
3961
|
const q = client ?? this.pool;
|
|
@@ -3328,22 +3978,22 @@ class SystemDatabase {
|
|
|
3328
3978
|
}
|
|
3329
3979
|
if (setClauses.length === 0) {
|
|
3330
3980
|
// Nothing to change, but still surface a missing schedule as an error.
|
|
3331
|
-
const
|
|
3332
|
-
|
|
3333
|
-
]);
|
|
3981
|
+
const existingParams = [name];
|
|
3982
|
+
const existing = await q.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules WHERE ${this.global(undefined, existingParams)} AND schedule_name = $1`, existingParams);
|
|
3334
3983
|
if (existing.rows.length === 0) {
|
|
3335
3984
|
throw new error_1.DBOSError(`Schedule '${name}' not found`);
|
|
3336
3985
|
}
|
|
3337
3986
|
return;
|
|
3338
3987
|
}
|
|
3339
3988
|
params.push(name);
|
|
3340
|
-
const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx}`, params);
|
|
3989
|
+
const result = await q.query(`UPDATE "${this.schemaName}".workflow_schedules SET ${setClauses.join(', ')} WHERE schedule_name = $${paramIdx} AND ${this.global(undefined, params)}`, params);
|
|
3341
3990
|
if (result.rowCount === 0) {
|
|
3342
3991
|
throw new error_1.DBOSError(`Schedule '${name}' not found`);
|
|
3343
3992
|
}
|
|
3344
3993
|
}
|
|
3345
3994
|
async updateLastFiredAt(name, lastFiredAt) {
|
|
3346
|
-
|
|
3995
|
+
const params = [lastFiredAt, name];
|
|
3996
|
+
await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE ${this.global(undefined, params)} AND schedule_name = $2`, params);
|
|
3347
3997
|
}
|
|
3348
3998
|
async applySchedules(schedules) {
|
|
3349
3999
|
const client = await this.connect();
|
|
@@ -3351,12 +4001,21 @@ class SystemDatabase {
|
|
|
3351
4001
|
await client.query('BEGIN');
|
|
3352
4002
|
for (const sched of schedules) {
|
|
3353
4003
|
const owner = await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
4004
|
+
const scheduleKeys = this.scheduleValues(sched);
|
|
4005
|
+
const values = {
|
|
4006
|
+
...this.globalValues(),
|
|
4007
|
+
...scheduleKeys,
|
|
4008
|
+
...scheduleRow(sched, owner),
|
|
4009
|
+
};
|
|
3354
4010
|
// Upsert on schedule_name; on conflict, preserve schedule_id and runtime state (status, last_fired_at) and update only the declared definition fields, so an unchanged re-apply is a no-op.
|
|
3355
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
3356
|
-
(
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
4011
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules (${Object.keys(values).join(', ')})
|
|
4012
|
+
VALUES (${Object.keys(values)
|
|
4013
|
+
.map((_, i) => `$${i + 1}`)
|
|
4014
|
+
.join(', ')})
|
|
4015
|
+
ON CONFLICT (${this.keyConflict('schedule_name')}) DO UPDATE SET
|
|
4016
|
+
${Object.keys(scheduleKeys)
|
|
4017
|
+
.map((c) => `${c} = EXCLUDED.${c}, `)
|
|
4018
|
+
.join('')}workflow_name = EXCLUDED.workflow_name,
|
|
3360
4019
|
workflow_class_name = EXCLUDED.workflow_class_name,
|
|
3361
4020
|
schedule = EXCLUDED.schedule,
|
|
3362
4021
|
context = EXCLUDED.context,
|
|
@@ -3364,20 +4023,7 @@ class SystemDatabase {
|
|
|
3364
4023
|
cron_timezone = EXCLUDED.cron_timezone,
|
|
3365
4024
|
queue_name = EXCLUDED.queue_name,
|
|
3366
4025
|
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it took.
|
|
3367
|
-
application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`,
|
|
3368
|
-
sched.scheduleId,
|
|
3369
|
-
sched.scheduleName,
|
|
3370
|
-
sched.workflowName,
|
|
3371
|
-
sched.workflowClassName,
|
|
3372
|
-
sched.schedule,
|
|
3373
|
-
sched.status,
|
|
3374
|
-
sched.context,
|
|
3375
|
-
sched.lastFiredAt,
|
|
3376
|
-
sched.automaticBackfill,
|
|
3377
|
-
sched.cronTimezone,
|
|
3378
|
-
sched.queueName,
|
|
3379
|
-
owner ?? null,
|
|
3380
|
-
]);
|
|
4026
|
+
application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, Object.values(values));
|
|
3381
4027
|
// Read back, since the guard above is silent about why it declined to claim.
|
|
3382
4028
|
await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
3383
4029
|
}
|
|
@@ -3402,14 +4048,23 @@ class SystemDatabase {
|
|
|
3402
4048
|
try {
|
|
3403
4049
|
await client.query('BEGIN');
|
|
3404
4050
|
// Claim a pre-upgrade row in place, so the version is not recreated or retimed.
|
|
4051
|
+
const claimParams = [owner ?? null, versionName];
|
|
3405
4052
|
const claimed = await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3406
4053
|
SET application_name = $1
|
|
3407
|
-
WHERE version_name = $2 AND application_name IS NULL`,
|
|
4054
|
+
WHERE ${this.global(undefined, claimParams)} AND version_name = $2 AND application_name IS NULL`, claimParams);
|
|
3408
4055
|
if ((claimed.rowCount ?? 0) === 0) {
|
|
4056
|
+
const values = {
|
|
4057
|
+
...this.globalValues(),
|
|
4058
|
+
version_id: (0, crypto_1.randomUUID)(),
|
|
4059
|
+
version_name: versionName,
|
|
4060
|
+
application_name: owner ?? null,
|
|
4061
|
+
};
|
|
3409
4062
|
// Targetless DO NOTHING: names no arbiter, so it survives version_name's uniqueness being dropped while still absorbing a concurrent registrar.
|
|
3410
|
-
await client.query(`INSERT INTO "${this.schemaName}".application_versions (
|
|
3411
|
-
VALUES ($
|
|
3412
|
-
|
|
4063
|
+
await client.query(`INSERT INTO "${this.schemaName}".application_versions (${Object.keys(values).join(', ')})
|
|
4064
|
+
VALUES (${Object.keys(values)
|
|
4065
|
+
.map((_, i) => `$${i + 1}`)
|
|
4066
|
+
.join(', ')})
|
|
4067
|
+
ON CONFLICT DO NOTHING`, Object.values(values));
|
|
3413
4068
|
}
|
|
3414
4069
|
// Read back, since the writes above are silent about why they declined to claim.
|
|
3415
4070
|
await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
|
|
@@ -3440,7 +4095,7 @@ class SystemDatabase {
|
|
|
3440
4095
|
params.push(resolved);
|
|
3441
4096
|
await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3442
4097
|
SET version_timestamp = $1, application_name = ${resolved === undefined ? 'application_name' : '$3'}
|
|
3443
|
-
WHERE version_name = $2 AND ${scope}`, params);
|
|
4098
|
+
WHERE ${this.global(undefined, params)} AND version_name = $2 AND ${scope}`, params);
|
|
3444
4099
|
await client.query('COMMIT');
|
|
3445
4100
|
}
|
|
3446
4101
|
catch (e) {
|
|
@@ -3453,7 +4108,7 @@ class SystemDatabase {
|
|
|
3453
4108
|
}
|
|
3454
4109
|
async listApplicationVersions() {
|
|
3455
4110
|
const params = [];
|
|
3456
|
-
const scope = this.appNameFilter('application_name', this.appName, params)
|
|
4111
|
+
const scope = `${this.global(undefined, params)} AND ${this.appNameFilter('application_name', this.appName, params)}`;
|
|
3457
4112
|
const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
|
|
3458
4113
|
FROM "${this.schemaName}".application_versions
|
|
3459
4114
|
WHERE ${scope}
|
|
@@ -3473,9 +4128,10 @@ class SystemDatabase {
|
|
|
3473
4128
|
}
|
|
3474
4129
|
// ==================== Queues ====================
|
|
3475
4130
|
async getQueue(name) {
|
|
4131
|
+
const params = [name];
|
|
3476
4132
|
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3477
4133
|
FROM "${this.schemaName}".queues
|
|
3478
|
-
WHERE name = $1`,
|
|
4134
|
+
WHERE ${this.global(undefined, params)} AND name = $1`, params);
|
|
3479
4135
|
return rows.length === 0 ? null : queueRecordFromRow(rows[0]);
|
|
3480
4136
|
}
|
|
3481
4137
|
/**
|
|
@@ -3484,14 +4140,15 @@ class SystemDatabase {
|
|
|
3484
4140
|
*/
|
|
3485
4141
|
async listQueues(applicationName) {
|
|
3486
4142
|
const params = [];
|
|
3487
|
-
const scope = this.observabilityFilter('application_name', applicationName, params)
|
|
4143
|
+
const scope = `${this.global(undefined, params)} AND ${this.observabilityFilter('application_name', applicationName, params)}`;
|
|
3488
4144
|
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3489
4145
|
FROM "${this.schemaName}".queues
|
|
3490
4146
|
WHERE ${scope}`, params);
|
|
3491
4147
|
return rows.map(queueRecordFromRow);
|
|
3492
4148
|
}
|
|
3493
4149
|
async deleteQueue(name) {
|
|
3494
|
-
|
|
4150
|
+
const params = [name];
|
|
4151
|
+
await this.pool.query(`DELETE FROM "${this.schemaName}".queues WHERE ${this.global(undefined, params)} AND name = $1`, params);
|
|
3495
4152
|
}
|
|
3496
4153
|
async updateQueue(name, fields) {
|
|
3497
4154
|
const setClauses = [];
|
|
@@ -3507,7 +4164,7 @@ class SystemDatabase {
|
|
|
3507
4164
|
setClauses.push(`"updated_at" = $${idx++}`);
|
|
3508
4165
|
params.push(Date.now());
|
|
3509
4166
|
params.push(name);
|
|
3510
|
-
await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx}`, params);
|
|
4167
|
+
await this.pool.query(`UPDATE "${this.schemaName}".queues SET ${setClauses.join(', ')} WHERE name = $${idx} AND ${this.global(undefined, params)}`, params);
|
|
3511
4168
|
}
|
|
3512
4169
|
/** Returns true iff this call inserted a new row (i.e. the queue did not
|
|
3513
4170
|
* previously exist). False if the row already existed, regardless of
|
|
@@ -3515,7 +4172,7 @@ class SystemDatabase {
|
|
|
3515
4172
|
async upsertQueue(record, updateExisting) {
|
|
3516
4173
|
const now = Date.now();
|
|
3517
4174
|
const onConflict = updateExisting
|
|
3518
|
-
? `ON CONFLICT (name) DO UPDATE SET
|
|
4175
|
+
? `ON CONFLICT (${this.keyConflict('name')}) DO UPDATE SET
|
|
3519
4176
|
concurrency = EXCLUDED.concurrency,
|
|
3520
4177
|
worker_concurrency = EXCLUDED.worker_concurrency,
|
|
3521
4178
|
rate_limit_max = EXCLUDED.rate_limit_max,
|
|
@@ -3530,21 +4187,27 @@ class SystemDatabase {
|
|
|
3530
4187
|
updated_at = EXCLUDED.updated_at,
|
|
3531
4188
|
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it just took.
|
|
3532
4189
|
application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
|
|
3533
|
-
: `ON CONFLICT (name) DO NOTHING`;
|
|
4190
|
+
: `ON CONFLICT (${this.keyConflict('name')}) DO NOTHING`;
|
|
3534
4191
|
const owner = record.applicationName ?? this.appName;
|
|
3535
4192
|
const client = await this.connect();
|
|
3536
4193
|
try {
|
|
3537
4194
|
await client.query('BEGIN');
|
|
3538
|
-
const
|
|
4195
|
+
const existedParams = [record.name];
|
|
4196
|
+
const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE ${this.global(undefined, existedParams)} AND name = $1`, existedParams);
|
|
3539
4197
|
// A name collision is a conflict in every mode: the name is the queue's address.
|
|
3540
4198
|
const resolvedOwner = await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
4199
|
+
const keys = this.globalValues();
|
|
3541
4200
|
await client.query(`INSERT INTO "${this.schemaName}".queues
|
|
3542
|
-
(
|
|
4201
|
+
(${Object.keys(keys)
|
|
4202
|
+
.map((c) => `${c}, `)
|
|
4203
|
+
.join('')}name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
|
|
3543
4204
|
priority_enabled, partition_queue, partition_concurrency, partition_worker_concurrency,
|
|
3544
4205
|
partition_rate_limit_max, partition_rate_limit_period_sec,
|
|
3545
4206
|
polling_interval_sec, updated_at, application_name)
|
|
3546
4207
|
-- priority_enabled is vestigial: every queue dispatches in priority order.
|
|
3547
|
-
VALUES ($
|
|
4208
|
+
VALUES (${Object.keys(keys)
|
|
4209
|
+
.map((_, i) => `$${14 + i}, `)
|
|
4210
|
+
.join('')}$1, $2, $3, $4, $5, TRUE, $6, $7, $8, $9, $10, $11, $12, $13)
|
|
3548
4211
|
${onConflict}`, [
|
|
3549
4212
|
record.name,
|
|
3550
4213
|
record.concurrency,
|
|
@@ -3559,6 +4222,7 @@ class SystemDatabase {
|
|
|
3559
4222
|
record.pollingIntervalSec,
|
|
3560
4223
|
now,
|
|
3561
4224
|
resolvedOwner ?? null,
|
|
4225
|
+
...Object.values(keys),
|
|
3562
4226
|
]);
|
|
3563
4227
|
// Read back, since the guard above is silent about why it declined to claim.
|
|
3564
4228
|
await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
@@ -3668,6 +4332,9 @@ class SystemDatabase {
|
|
|
3668
4332
|
params.push(statuses);
|
|
3669
4333
|
where = `${where} AND status = ANY($${params.length})`;
|
|
3670
4334
|
}
|
|
4335
|
+
else {
|
|
4336
|
+
where = `${where} AND ${this.global(undefined, params)}`;
|
|
4337
|
+
}
|
|
3671
4338
|
const res = await client.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${where}`, params);
|
|
3672
4339
|
return res.rowCount ?? 0;
|
|
3673
4340
|
};
|
|
@@ -3692,84 +4359,64 @@ class SystemDatabase {
|
|
|
3692
4359
|
}
|
|
3693
4360
|
// ==================== Internal ====================
|
|
3694
4361
|
async insertWorkflowStatus(client, initStatus, creatorXid) {
|
|
4362
|
+
const values = {
|
|
4363
|
+
...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
|
|
4364
|
+
workflow_uuid: initStatus.workflowUUID,
|
|
4365
|
+
status: initStatus.status,
|
|
4366
|
+
name: initStatus.workflowName,
|
|
4367
|
+
// For cross-language compatibility, these variables MUST be NULL in the database when not set
|
|
4368
|
+
class_name: initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
|
|
4369
|
+
config_name: initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
|
|
4370
|
+
queue_name: initStatus.queueName ?? null,
|
|
4371
|
+
authenticated_user: initStatus.authenticatedUser,
|
|
4372
|
+
assumed_role: initStatus.assumedRole,
|
|
4373
|
+
authenticated_roles: JSON.stringify(initStatus.authenticatedRoles),
|
|
4374
|
+
executor_id: initStatus.executorId,
|
|
4375
|
+
application_version: initStatus.applicationVersion ?? null,
|
|
4376
|
+
application_id: initStatus.applicationID,
|
|
4377
|
+
recovery_attempts: initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
|
|
4378
|
+
workflow_timeout_ms: initStatus.timeoutMS ?? null,
|
|
4379
|
+
workflow_deadline_epoch_ms: initStatus.deadlineEpochMS ?? null,
|
|
4380
|
+
// Legacy column: the payload lives in workflow_input.
|
|
4381
|
+
inputs: null,
|
|
4382
|
+
deduplication_id: initStatus.deduplicationID ?? null,
|
|
4383
|
+
priority: initStatus.priority,
|
|
4384
|
+
queue_partition_key: initStatus.queuePartitionKey ?? null,
|
|
4385
|
+
forked_from: initStatus.forkedFrom ?? null,
|
|
4386
|
+
parent_workflow_id: initStatus.parentWorkflowID ?? null,
|
|
4387
|
+
...this.parentValues(initStatus.parentWorkflowID),
|
|
4388
|
+
serialization: initStatus.serialization,
|
|
4389
|
+
creator_xid: creatorXid,
|
|
4390
|
+
delay_until_epoch_ms: initStatus.delayUntilEpochMS ?? null,
|
|
4391
|
+
attributes: initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
|
|
4392
|
+
schedule_name: initStatus.scheduleName ?? null,
|
|
4393
|
+
debounce_deadline_epoch_ms: initStatus.debounceDeadlineEpochMS ?? null,
|
|
4394
|
+
is_debounced: initStatus.isDebounced ?? false,
|
|
4395
|
+
application_name: initStatus.applicationName ?? null,
|
|
4396
|
+
// A direct start runs at once, so its creator token also owns the execution.
|
|
4397
|
+
owner_xid: initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
|
|
4398
|
+
};
|
|
4399
|
+
const columns = Object.keys(values);
|
|
4400
|
+
const inputValues = {
|
|
4401
|
+
...this.wfValues(initStatus.workflowUUID, initStatus.shardKey),
|
|
4402
|
+
workflow_uuid: initStatus.workflowUUID,
|
|
4403
|
+
inputs: initStatus.input ?? null,
|
|
4404
|
+
};
|
|
4405
|
+
const inputColumns = Object.keys(inputValues);
|
|
3695
4406
|
try {
|
|
3696
|
-
const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
name,
|
|
3700
|
-
class_name,
|
|
3701
|
-
config_name,
|
|
3702
|
-
queue_name,
|
|
3703
|
-
authenticated_user,
|
|
3704
|
-
assumed_role,
|
|
3705
|
-
authenticated_roles,
|
|
3706
|
-
executor_id,
|
|
3707
|
-
application_version,
|
|
3708
|
-
application_id,
|
|
3709
|
-
recovery_attempts,
|
|
3710
|
-
workflow_timeout_ms,
|
|
3711
|
-
workflow_deadline_epoch_ms,
|
|
3712
|
-
inputs,
|
|
3713
|
-
deduplication_id,
|
|
3714
|
-
priority,
|
|
3715
|
-
queue_partition_key,
|
|
3716
|
-
forked_from,
|
|
3717
|
-
parent_workflow_id,
|
|
3718
|
-
serialization,
|
|
3719
|
-
creator_xid,
|
|
3720
|
-
delay_until_epoch_ms,
|
|
3721
|
-
attributes,
|
|
3722
|
-
schedule_name,
|
|
3723
|
-
debounce_deadline_epoch_ms,
|
|
3724
|
-
is_debounced,
|
|
3725
|
-
application_name,
|
|
3726
|
-
owner_xid
|
|
3727
|
-
) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)
|
|
3728
|
-
ON CONFLICT (workflow_uuid)
|
|
4407
|
+
const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
|
|
4408
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
4409
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')})
|
|
3729
4410
|
-- A no-op update, so an existing row comes back unchanged for the caller to inspect.
|
|
3730
4411
|
DO UPDATE SET creator_xid = workflow_status.creator_xid
|
|
3731
|
-
RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`,
|
|
3732
|
-
initStatus.workflowUUID,
|
|
3733
|
-
initStatus.status,
|
|
3734
|
-
initStatus.workflowName,
|
|
3735
|
-
// For cross-language compatibility, these variables MUST be NULL in the database when not set
|
|
3736
|
-
initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
|
|
3737
|
-
initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
|
|
3738
|
-
initStatus.queueName ?? null,
|
|
3739
|
-
initStatus.authenticatedUser,
|
|
3740
|
-
initStatus.assumedRole,
|
|
3741
|
-
JSON.stringify(initStatus.authenticatedRoles),
|
|
3742
|
-
initStatus.executorId,
|
|
3743
|
-
initStatus.applicationVersion ?? null,
|
|
3744
|
-
initStatus.applicationID,
|
|
3745
|
-
initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
|
|
3746
|
-
initStatus.timeoutMS ?? null,
|
|
3747
|
-
initStatus.deadlineEpochMS ?? null,
|
|
3748
|
-
// Legacy column: the payload lives in workflow_input.
|
|
3749
|
-
null,
|
|
3750
|
-
initStatus.deduplicationID ?? null,
|
|
3751
|
-
initStatus.priority,
|
|
3752
|
-
initStatus.queuePartitionKey ?? null,
|
|
3753
|
-
initStatus.forkedFrom ?? null,
|
|
3754
|
-
initStatus.parentWorkflowID ?? null,
|
|
3755
|
-
initStatus.serialization,
|
|
3756
|
-
creatorXid,
|
|
3757
|
-
initStatus.delayUntilEpochMS ?? null,
|
|
3758
|
-
initStatus.attributes ? JSON.stringify(initStatus.attributes) : null,
|
|
3759
|
-
initStatus.scheduleName ?? null,
|
|
3760
|
-
initStatus.debounceDeadlineEpochMS ?? null,
|
|
3761
|
-
initStatus.isDebounced ?? false,
|
|
3762
|
-
initStatus.applicationName ?? null,
|
|
3763
|
-
// A direct start runs at once, so its creator token also owns the execution.
|
|
3764
|
-
initStatus.status === workflow_1.StatusString.PENDING ? creatorXid : null,
|
|
3765
|
-
]);
|
|
4412
|
+
RETURNING status, name, class_name, config_name, queue_name, executor_id, creator_xid, serialization`, Object.values(values));
|
|
3766
4413
|
if (rows.length === 0) {
|
|
3767
4414
|
throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
|
|
3768
4415
|
}
|
|
3769
4416
|
// Two statements, not a data-modifying CTE: at scale the CTE costs more than the round trip it saves.
|
|
3770
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (
|
|
3771
|
-
VALUES ($1,
|
|
3772
|
-
ON CONFLICT (workflow_uuid) DO NOTHING`,
|
|
4417
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_input (${inputColumns.join(', ')})
|
|
4418
|
+
VALUES (${inputColumns.map((_, i) => `$${i + 1}`).join(', ')})
|
|
4419
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`, Object.values(inputValues));
|
|
3773
4420
|
const ret = rows[0];
|
|
3774
4421
|
ret.class_name = ret.class_name ?? '';
|
|
3775
4422
|
ret.config_name = ret.config_name ?? '';
|
|
@@ -3785,16 +4432,17 @@ class SystemDatabase {
|
|
|
3785
4432
|
}
|
|
3786
4433
|
}
|
|
3787
4434
|
async getWorkflowStatusValue(client, workflowID) {
|
|
3788
|
-
const
|
|
4435
|
+
const params = [];
|
|
4436
|
+
const { rows } = await client.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
|
|
3789
4437
|
return rows.length === 0 ? undefined : rows[0].status;
|
|
3790
4438
|
}
|
|
3791
4439
|
async updateWorkflowStatus(client, workflowID, status, options = {}) {
|
|
3792
4440
|
// Use SQL now() so updated_at and completed_at (when set together) are
|
|
3793
4441
|
// computed in the same statement against the same clock.
|
|
3794
4442
|
const nowMsExpr = `(EXTRACT(EPOCH FROM now()) * 1000)::bigint`;
|
|
3795
|
-
let setClause = `SET status=$
|
|
3796
|
-
|
|
3797
|
-
|
|
4443
|
+
let setClause = `SET status=$1, updated_at=${nowMsExpr}`;
|
|
4444
|
+
const args = [status];
|
|
4445
|
+
let whereClause = `WHERE ${this.wf(undefined, workflowID, args)}`;
|
|
3798
4446
|
const update = options.update ?? {};
|
|
3799
4447
|
if (update.resetRecoveryAttempts) {
|
|
3800
4448
|
setClause += `, recovery_attempts = 0`;
|
|
@@ -3853,27 +4501,31 @@ class SystemDatabase {
|
|
|
3853
4501
|
async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
3854
4502
|
const ownerXid = options.ownerChecked ? undefined : (0, context_1.currentOwnerXid)(workflowID);
|
|
3855
4503
|
if (ownerXid !== undefined) {
|
|
3856
|
-
await this
|
|
3857
|
-
}
|
|
4504
|
+
await this.checkOwner(client, workflowID, ownerXid);
|
|
4505
|
+
}
|
|
4506
|
+
const values = {
|
|
4507
|
+
...this.wfValues(workflowID),
|
|
4508
|
+
workflow_uuid: workflowID,
|
|
4509
|
+
function_id: functionID,
|
|
4510
|
+
output: options.output ?? null,
|
|
4511
|
+
error: options.error ?? null,
|
|
4512
|
+
function_name: functionName,
|
|
4513
|
+
child_workflow_id: options.childWorkflowID ?? null,
|
|
4514
|
+
...(options.childValues ?? this.childValues(options.childWorkflowID)),
|
|
4515
|
+
started_at_epoch_ms: startTimeEpochMs,
|
|
4516
|
+
completed_at_epoch_ms: endTimeEpochMs,
|
|
4517
|
+
serialization: options.serialization ?? null,
|
|
4518
|
+
// Mirrors the parent: only the running application records its steps.
|
|
4519
|
+
application_name: this.appName ?? null,
|
|
4520
|
+
};
|
|
4521
|
+
const columns = Object.keys(values);
|
|
3858
4522
|
try {
|
|
3859
4523
|
const out = await client.query(`INSERT INTO "${this.schemaName}".operation_outputs
|
|
3860
|
-
(
|
|
3861
|
-
VALUES ($
|
|
3862
|
-
ON CONFLICT (workflow_uuid, function_id) DO UPDATE
|
|
4524
|
+
(${columns.join(', ')}, retention_timestamp)
|
|
4525
|
+
VALUES (${columns.map((_, i) => `$${i + 1}`).join(', ')}, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
|
|
4526
|
+
ON CONFLICT (${this.keyConflict('workflow_uuid', 'function_id')}) DO UPDATE
|
|
3863
4527
|
SET completed_at_epoch_ms = operation_outputs.completed_at_epoch_ms
|
|
3864
|
-
RETURNING completed_at_epoch_ms;`,
|
|
3865
|
-
workflowID,
|
|
3866
|
-
functionID,
|
|
3867
|
-
options.output ?? null,
|
|
3868
|
-
options.error ?? null,
|
|
3869
|
-
functionName,
|
|
3870
|
-
options.childWorkflowID ?? null,
|
|
3871
|
-
startTimeEpochMs,
|
|
3872
|
-
endTimeEpochMs,
|
|
3873
|
-
options.serialization ?? null,
|
|
3874
|
-
// Mirrors the parent: only the running application records its steps.
|
|
3875
|
-
this.appName ?? null,
|
|
3876
|
-
]);
|
|
4528
|
+
RETURNING completed_at_epoch_ms;`, Object.values(values));
|
|
3877
4529
|
// A row already there with another completion time is this execution's own doing
|
|
3878
4530
|
// (a retry carries the same time), so it is nondeterminism, not a duplicate.
|
|
3879
4531
|
if (checkConflict &&
|
|
@@ -3890,7 +4542,7 @@ class SystemDatabase {
|
|
|
3890
4542
|
}
|
|
3891
4543
|
}
|
|
3892
4544
|
/** Run `fn` in a READ COMMITTED transaction on `client`, so an ownership check holds its row lock until the write commits. */
|
|
3893
|
-
async
|
|
4545
|
+
async inTransaction(client, fn) {
|
|
3894
4546
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
3895
4547
|
let result;
|
|
3896
4548
|
try {
|
|
@@ -3905,22 +4557,26 @@ class SystemDatabase {
|
|
|
3905
4557
|
return result;
|
|
3906
4558
|
}
|
|
3907
4559
|
/** Throw DBOSWorkflowConflictError unless ownerXid still owns the workflow; the row lock holds until commit, so no hand-off lands in between. */
|
|
3908
|
-
async
|
|
3909
|
-
const
|
|
4560
|
+
async checkOwner(client, workflowID, ownerXid) {
|
|
4561
|
+
const params = [];
|
|
4562
|
+
const { rows } = await client.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)} FOR NO KEY UPDATE`, params);
|
|
3910
4563
|
if (rows[0]?.owner_xid !== ownerXid) {
|
|
3911
4564
|
throw new error_1.DBOSWorkflowConflictError(workflowID);
|
|
3912
4565
|
}
|
|
3913
4566
|
}
|
|
3914
4567
|
/** The workflow's current ownership token; null if unowned or missing. */
|
|
3915
4568
|
async getWorkflowOwner(workflowID) {
|
|
3916
|
-
const
|
|
4569
|
+
const params = [];
|
|
4570
|
+
const { rows } = await this.pool.query(`SELECT owner_xid FROM "${this.schemaName}".workflow_status WHERE ${this.wf(undefined, workflowID, params)}`, params);
|
|
3917
4571
|
return rows[0]?.owner_xid ?? null;
|
|
3918
4572
|
}
|
|
3919
4573
|
async #getOperationResultAndThrowIfCancelled(client, workflowID, functionID) {
|
|
3920
4574
|
await this.#checkIfCanceled(client, workflowID);
|
|
3921
|
-
const
|
|
4575
|
+
const childKey = this.keyColumn(undefined, 'child_shard_key');
|
|
4576
|
+
const params = [functionID];
|
|
4577
|
+
const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name, serialization${childKey !== undefined ? `, ${childKey}` : ''}
|
|
3922
4578
|
FROM "${this.schemaName}".operation_outputs
|
|
3923
|
-
WHERE
|
|
4579
|
+
WHERE ${this.wf(undefined, workflowID, params)} AND function_id=$1`, params);
|
|
3924
4580
|
if (rows.length === 0) {
|
|
3925
4581
|
return undefined;
|
|
3926
4582
|
}
|
|
@@ -3929,6 +4585,7 @@ class SystemDatabase {
|
|
|
3929
4585
|
output: rows[0].output,
|
|
3930
4586
|
error: rows[0].error,
|
|
3931
4587
|
childWorkflowID: rows[0].child_workflow_id,
|
|
4588
|
+
childShardKey: rows[0].child_shard_key ?? undefined,
|
|
3932
4589
|
functionName: rows[0].function_name,
|
|
3933
4590
|
// Return serialization so recv/getEvent replay deserializes with the stored format, not the default.
|
|
3934
4591
|
serialization: rows[0].serialization,
|
|
@@ -3966,7 +4623,11 @@ class SystemDatabase {
|
|
|
3966
4623
|
// been cancelled.
|
|
3967
4624
|
// For an actual sleep, completed_at is the wake deadline so the step's duration reflects the
|
|
3968
4625
|
// sleep; a timeout marker records zero duration since its deadline may never be reached.
|
|
3969
|
-
async
|
|
4626
|
+
async durableSleepInternal(workflowID, functionID, durationMS, recordCompletionAtDeadline = false) {
|
|
4627
|
+
// The caller's own workflow, even where a same-ID workflow is bound as a target.
|
|
4628
|
+
return await this.ownKey(workflowID, undefined, () => this.#durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline));
|
|
4629
|
+
}
|
|
4630
|
+
async #durableSleep(workflowID, functionID, durationMS, recordCompletionAtDeadline) {
|
|
3970
4631
|
const startTimeMs = Date.now();
|
|
3971
4632
|
// Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
|
|
3972
4633
|
const endTimeMs = startTimeMs + Math.ceil(durationMS);
|
|
@@ -3979,7 +4640,7 @@ class SystemDatabase {
|
|
|
3979
4640
|
}
|
|
3980
4641
|
return JSON.parse(res.output);
|
|
3981
4642
|
}
|
|
3982
|
-
await this
|
|
4643
|
+
await this.inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, startTimeMs, recordCompletionAtDeadline ? endTimeMs : startTimeMs, {
|
|
3983
4644
|
output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
|
|
3984
4645
|
serialization: serialization_1.DBOSPortableJSON.name(),
|
|
3985
4646
|
}));
|
|
@@ -4159,7 +4820,7 @@ __decorate([
|
|
|
4159
4820
|
__decorate([
|
|
4160
4821
|
dbRetry(),
|
|
4161
4822
|
__metadata("design:type", Function),
|
|
4162
|
-
__metadata("design:paramtypes", [String, Number]),
|
|
4823
|
+
__metadata("design:paramtypes", [String, Number, String]),
|
|
4163
4824
|
__metadata("design:returntype", Promise)
|
|
4164
4825
|
], SystemDatabase.prototype, "getOperationResultAndThrowIfCancelled", null);
|
|
4165
4826
|
__decorate([
|
|
@@ -4219,9 +4880,9 @@ __decorate([
|
|
|
4219
4880
|
__decorate([
|
|
4220
4881
|
dbRetry(),
|
|
4221
4882
|
__metadata("design:type", Function),
|
|
4222
|
-
__metadata("design:paramtypes", [Array
|
|
4883
|
+
__metadata("design:paramtypes", [Array]),
|
|
4223
4884
|
__metadata("design:returntype", Promise)
|
|
4224
|
-
], SystemDatabase.prototype, "
|
|
4885
|
+
], SystemDatabase.prototype, "deleteWorkflowsInternal", null);
|
|
4225
4886
|
__decorate([
|
|
4226
4887
|
dbRetry()
|
|
4227
4888
|
// A missing row normally means the workflow has not been inserted yet, so
|
|
@@ -4255,19 +4916,19 @@ __decorate([
|
|
|
4255
4916
|
__decorate([
|
|
4256
4917
|
dbRetry(),
|
|
4257
4918
|
__metadata("design:type", Function),
|
|
4258
|
-
__metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String]),
|
|
4919
|
+
__metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String, String]),
|
|
4259
4920
|
__metadata("design:returntype", Promise)
|
|
4260
4921
|
], SystemDatabase.prototype, "send", null);
|
|
4261
4922
|
__decorate([
|
|
4262
4923
|
dbRetry(),
|
|
4263
4924
|
__metadata("design:type", Function),
|
|
4264
|
-
__metadata("design:paramtypes", [String, String, Object, Object, Object, String]),
|
|
4925
|
+
__metadata("design:paramtypes", [String, String, Object, Object, Object, String, String]),
|
|
4265
4926
|
__metadata("design:returntype", Promise)
|
|
4266
4927
|
], SystemDatabase.prototype, "sendFromStep", null);
|
|
4267
4928
|
__decorate([
|
|
4268
4929
|
dbRetry(),
|
|
4269
4930
|
__metadata("design:type", Function),
|
|
4270
|
-
__metadata("design:paramtypes", [String, Object, Object, Object, String]),
|
|
4931
|
+
__metadata("design:paramtypes", [String, Object, Object, Object, String, String]),
|
|
4271
4932
|
__metadata("design:returntype", Promise)
|
|
4272
4933
|
], SystemDatabase.prototype, "sendDirectStandalone", null);
|
|
4273
4934
|
__decorate([
|
|
@@ -4321,7 +4982,7 @@ __decorate([
|
|
|
4321
4982
|
__decorate([
|
|
4322
4983
|
dbRetry(),
|
|
4323
4984
|
__metadata("design:type", Function),
|
|
4324
|
-
__metadata("design:paramtypes", [Object]),
|
|
4985
|
+
__metadata("design:paramtypes", [Object, Object]),
|
|
4325
4986
|
__metadata("design:returntype", Promise)
|
|
4326
4987
|
], SystemDatabase.prototype, "listWorkflows", null);
|
|
4327
4988
|
__decorate([
|