@dbos-inc/dbos-sdk 4.26.3-preview → 4.26.5-preview
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/schemas/system_db_schema.d.ts +5 -0
- package/dist/schemas/system_db_schema.d.ts.map +1 -1
- package/dist/schemas/system_db_schema.js.map +1 -1
- package/dist/src/adminserver.js +1 -1
- package/dist/src/adminserver.js.map +1 -1
- package/dist/src/cli/cli.d.ts.map +1 -1
- package/dist/src/cli/cli.js +64 -0
- package/dist/src/cli/cli.js.map +1 -1
- package/dist/src/cli/schema.d.ts.map +1 -1
- package/dist/src/cli/schema.js +26 -16
- package/dist/src/cli/schema.js.map +1 -1
- package/dist/src/client.d.ts +62 -4
- package/dist/src/client.d.ts.map +1 -1
- package/dist/src/client.js +58 -8
- package/dist/src/client.js.map +1 -1
- package/dist/src/conductor/conductor.d.ts.map +1 -1
- package/dist/src/conductor/conductor.js +13 -2
- package/dist/src/conductor/conductor.js.map +1 -1
- package/dist/src/conductor/protocol.d.ts +16 -2
- package/dist/src/conductor/protocol.d.ts.map +1 -1
- package/dist/src/conductor/protocol.js +10 -2
- package/dist/src/conductor/protocol.js.map +1 -1
- package/dist/src/context.d.ts +1 -0
- package/dist/src/context.d.ts.map +1 -1
- package/dist/src/context.js +2 -1
- package/dist/src/context.js.map +1 -1
- package/dist/src/dbos-executor.d.ts +1 -0
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js +11 -2
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts +41 -7
- package/dist/src/dbos.d.ts.map +1 -1
- package/dist/src/dbos.js +61 -4
- package/dist/src/dbos.js.map +1 -1
- package/dist/src/debouncer.d.ts +4 -0
- package/dist/src/debouncer.d.ts.map +1 -1
- package/dist/src/debouncer.js +32 -8
- package/dist/src/debouncer.js.map +1 -1
- package/dist/src/enqueue_options.d.ts +59 -0
- package/dist/src/enqueue_options.d.ts.map +1 -0
- package/dist/src/enqueue_options.js +61 -0
- package/dist/src/enqueue_options.js.map +1 -0
- package/dist/src/enqueue_workflow.d.ts +12 -0
- package/dist/src/enqueue_workflow.d.ts.map +1 -0
- package/dist/src/enqueue_workflow.js +91 -0
- package/dist/src/enqueue_workflow.js.map +1 -0
- package/dist/src/index.d.ts +2 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/scheduler/scheduler.d.ts +1 -0
- package/dist/src/scheduler/scheduler.d.ts.map +1 -1
- package/dist/src/scheduler/scheduler.js +13 -5
- package/dist/src/scheduler/scheduler.js.map +1 -1
- package/dist/src/sysdb_migrations/internal/migrations.d.ts +2 -1
- package/dist/src/sysdb_migrations/internal/migrations.d.ts.map +1 -1
- package/dist/src/sysdb_migrations/internal/migrations.js +186 -2
- package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
- package/dist/src/sysdb_migrations/migration_runner.d.ts +5 -0
- package/dist/src/sysdb_migrations/migration_runner.d.ts.map +1 -1
- package/dist/src/sysdb_migrations/migration_runner.js +48 -10
- package/dist/src/sysdb_migrations/migration_runner.js.map +1 -1
- package/dist/src/system_database.d.ts +65 -10
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +599 -175
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/utils.d.ts +1 -0
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +1 -0
- package/dist/src/utils.js.map +1 -1
- package/dist/src/wfqueue.d.ts +2 -0
- package/dist/src/wfqueue.d.ts.map +1 -1
- package/dist/src/wfqueue.js +5 -1
- package/dist/src/wfqueue.js.map +1 -1
- package/dist/src/workflow.d.ts +2 -0
- package/dist/src/workflow.d.ts.map +1 -1
- package/dist/src/workflow.js.map +1 -1
- package/dist/src/workflow_management.d.ts.map +1 -1
- package/dist/src/workflow_management.js +4 -9
- package/dist/src/workflow_management.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
|
@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
9
9
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.SystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
12
|
+
exports.SystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
13
13
|
const dbos_executor_1 = require("./dbos-executor");
|
|
14
14
|
const pg_1 = require("pg");
|
|
15
15
|
const error_1 = require("./error");
|
|
@@ -38,6 +38,8 @@ exports.DBOS_WORKFLOW_EVENTS_CHANNEL = 'dbos_workflow_events_channel';
|
|
|
38
38
|
exports.DBOS_STREAMS_CHANNEL = 'dbos_streams_channel';
|
|
39
39
|
// Interval for coalescing LISTEN/NOTIFY notifications off the write path; caps the rate of notifying commits regardless of write throughput.
|
|
40
40
|
exports.DEFAULT_NOTIFICATION_COALESCE_MS = 10;
|
|
41
|
+
// Workflows re-owned per transaction by a rename. Matches the GC default.
|
|
42
|
+
exports.DEFAULT_RENAME_BATCH_SIZE = 10_000;
|
|
41
43
|
const QUEUE_COLUMN_BY_FIELD = {
|
|
42
44
|
concurrency: 'concurrency',
|
|
43
45
|
workerConcurrency: 'worker_concurrency',
|
|
@@ -47,6 +49,7 @@ const QUEUE_COLUMN_BY_FIELD = {
|
|
|
47
49
|
partitionQueue: 'partition_queue',
|
|
48
50
|
pollingIntervalSec: 'polling_interval_sec',
|
|
49
51
|
};
|
|
52
|
+
const QUEUE_COLUMNS = 'name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec, priority_enabled, partition_queue, polling_interval_sec, application_name';
|
|
50
53
|
function queueRecordFromRow(row) {
|
|
51
54
|
return {
|
|
52
55
|
name: row.name,
|
|
@@ -57,6 +60,7 @@ function queueRecordFromRow(row) {
|
|
|
57
60
|
priorityEnabled: row.priority_enabled,
|
|
58
61
|
partitionQueue: row.partition_queue,
|
|
59
62
|
pollingIntervalSec: row.polling_interval_sec,
|
|
63
|
+
applicationName: row.application_name ?? undefined,
|
|
60
64
|
};
|
|
61
65
|
}
|
|
62
66
|
/** The statements granting permissions on all entities in the system schema to a role. */
|
|
@@ -172,6 +176,32 @@ class NotificationMap {
|
|
|
172
176
|
}
|
|
173
177
|
}
|
|
174
178
|
}
|
|
179
|
+
function mapVersionInfo(row) {
|
|
180
|
+
return {
|
|
181
|
+
versionId: row.version_id,
|
|
182
|
+
versionName: row.version_name,
|
|
183
|
+
versionTimestamp: Number(row.version_timestamp),
|
|
184
|
+
createdAt: Number(row.created_at),
|
|
185
|
+
applicationName: row.application_name ?? undefined,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
const SCHEDULE_COLUMNS = 'schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name';
|
|
189
|
+
function mapWorkflowSchedule(row) {
|
|
190
|
+
return {
|
|
191
|
+
scheduleId: row.schedule_id,
|
|
192
|
+
scheduleName: row.schedule_name,
|
|
193
|
+
workflowName: row.workflow_name,
|
|
194
|
+
workflowClassName: row.workflow_class_name,
|
|
195
|
+
schedule: row.schedule,
|
|
196
|
+
status: row.status,
|
|
197
|
+
context: row.context,
|
|
198
|
+
lastFiredAt: row.last_fired_at ?? null,
|
|
199
|
+
automaticBackfill: !!row.automatic_backfill,
|
|
200
|
+
cronTimezone: row.cron_timezone ?? null,
|
|
201
|
+
queueName: row.queue_name ?? null,
|
|
202
|
+
applicationName: row.application_name ?? undefined,
|
|
203
|
+
};
|
|
204
|
+
}
|
|
175
205
|
function mapWorkflowStatus(row) {
|
|
176
206
|
return {
|
|
177
207
|
workflowUUID: row.workflow_uuid,
|
|
@@ -209,6 +239,7 @@ function mapWorkflowStatus(row) {
|
|
|
209
239
|
scheduleName: row.schedule_name ?? undefined,
|
|
210
240
|
debounceDeadlineEpochMS: row.debounce_deadline_epoch_ms ? Number(row.debounce_deadline_epoch_ms) : undefined,
|
|
211
241
|
isDebounced: row.is_debounced ?? false,
|
|
242
|
+
applicationName: row.application_name ?? undefined,
|
|
212
243
|
};
|
|
213
244
|
}
|
|
214
245
|
// SQLSTATE classes/codes that are generally safe to retry
|
|
@@ -364,6 +395,7 @@ class SystemDatabase {
|
|
|
364
395
|
systemDatabaseUrl;
|
|
365
396
|
logger;
|
|
366
397
|
serializer;
|
|
398
|
+
appName;
|
|
367
399
|
// ==================== Lifecycle ====================
|
|
368
400
|
pool;
|
|
369
401
|
schemaName;
|
|
@@ -411,10 +443,13 @@ class SystemDatabase {
|
|
|
411
443
|
runningWorkflowMap = new Map(); // Map from workflowID to workflow promise, queue name and partition key
|
|
412
444
|
// Per-partition-key created_at cursors: keep per-key queue order monotonic across batches
|
|
413
445
|
#batchCreatedAtCursors = new Map();
|
|
414
|
-
constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS
|
|
446
|
+
constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS,
|
|
447
|
+
// The application this handle acts for; undefined writes unclaimed rows.
|
|
448
|
+
appName) {
|
|
415
449
|
this.systemDatabaseUrl = systemDatabaseUrl;
|
|
416
450
|
this.logger = logger;
|
|
417
451
|
this.serializer = serializer;
|
|
452
|
+
this.appName = appName;
|
|
418
453
|
this.schemaName = schemaName;
|
|
419
454
|
this.shouldUseDBNotifications = useListenNotify;
|
|
420
455
|
this.notificationCoalesceMs = notificationCoalesceMs;
|
|
@@ -448,6 +483,60 @@ class SystemDatabase {
|
|
|
448
483
|
getSerializer() {
|
|
449
484
|
return this.serializer;
|
|
450
485
|
}
|
|
486
|
+
// ==================== Application Ownership ====================
|
|
487
|
+
/**
|
|
488
|
+
* A predicate matching rows owned by these applications plus unclaimed ones, which belong to
|
|
489
|
+
* every application; unset or empty matches everything. Appends its bind parameter to `params`.
|
|
490
|
+
*/
|
|
491
|
+
#appNameFilter(column, value, params) {
|
|
492
|
+
// An empty name is no name: it is not a value any application could be configured with.
|
|
493
|
+
const names = !value ? [] : Array.isArray(value) ? value : [value];
|
|
494
|
+
if (names.length === 0)
|
|
495
|
+
return 'TRUE';
|
|
496
|
+
params.push(names);
|
|
497
|
+
return `(${column} = ANY($${params.length}) OR ${column} IS NULL)`;
|
|
498
|
+
}
|
|
499
|
+
/**
|
|
500
|
+
* The filter above defaulted to this handle's own application: an unset filter scopes to what
|
|
501
|
+
* this application owns, not to every application's rows. A handle with no application of its
|
|
502
|
+
* own still matches every one.
|
|
503
|
+
*/
|
|
504
|
+
#observabilityFilter(column, value, params) {
|
|
505
|
+
return this.#appNameFilter(column, value ?? this.appName, params);
|
|
506
|
+
}
|
|
507
|
+
/**
|
|
508
|
+
* The version name a dequeue treats as latest: this application's own plus unclaimed
|
|
509
|
+
* ones, so a named peer's deploy does not demote this one.
|
|
510
|
+
*/
|
|
511
|
+
async #latestApplicationVersionName(client) {
|
|
512
|
+
const params = [];
|
|
513
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
514
|
+
const { rows } = await client.query(`SELECT version_name
|
|
515
|
+
FROM "${this.schemaName}".application_versions
|
|
516
|
+
WHERE ${scope}
|
|
517
|
+
ORDER BY version_timestamp DESC LIMIT 1`, params);
|
|
518
|
+
return rows[0]?.version_name;
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* The owner to persist when writing a row that may already exist. A nameless writer
|
|
522
|
+
* leaves the owner intact; a named one collides only with a different name.
|
|
523
|
+
*/
|
|
524
|
+
async #resolveRowOwner(client, table, keyColumn, name, owner, kind) {
|
|
525
|
+
const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${keyColumn} = $1`, [name]);
|
|
526
|
+
const current = rows[0]?.application_name ?? null;
|
|
527
|
+
if (current === null)
|
|
528
|
+
return owner;
|
|
529
|
+
if (owner === undefined || current === owner)
|
|
530
|
+
return current;
|
|
531
|
+
// A version name is computed or pinned, so "pick another" is config advice, not a rename.
|
|
532
|
+
const takeANewName = kind === 'Application version'
|
|
533
|
+
? `set a distinct applicationVersion for '${owner}'`
|
|
534
|
+
: `give '${owner}' a different ${kind.toLowerCase()} name`;
|
|
535
|
+
throw new error_1.DBOSError(`${kind} '${name}' is already registered by application '${current}' in this system database. ` +
|
|
536
|
+
`${kind} names must be unique across applications sharing a system database. ` +
|
|
537
|
+
`Either ${takeANewName}, or, if '${current}' was renamed to '${owner}', ` +
|
|
538
|
+
`re-own its rows first with dbos rename-application`);
|
|
539
|
+
}
|
|
451
540
|
async init() {
|
|
452
541
|
await ensureSystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
|
|
453
542
|
if (this.shouldUseDBNotifications) {
|
|
@@ -458,8 +547,11 @@ class SystemDatabase {
|
|
|
458
547
|
}
|
|
459
548
|
}
|
|
460
549
|
async destroy() {
|
|
550
|
+
// Set synchronously, before any await, so no reconnect is scheduled or published after this point.
|
|
551
|
+
this.#notificationsStopped = true;
|
|
461
552
|
if (this.reconnectTimeout) {
|
|
462
553
|
clearTimeout(this.reconnectTimeout);
|
|
554
|
+
this.reconnectTimeout = null;
|
|
463
555
|
}
|
|
464
556
|
// Stop the notifier and await its final flush before the pool closes.
|
|
465
557
|
this.#notifierActive = false;
|
|
@@ -469,12 +561,7 @@ class SystemDatabase {
|
|
|
469
561
|
this.#notifierLoop = undefined;
|
|
470
562
|
}
|
|
471
563
|
if (this.notificationsClient) {
|
|
472
|
-
|
|
473
|
-
this.notificationsClient.release(true);
|
|
474
|
-
}
|
|
475
|
-
catch (e) {
|
|
476
|
-
this.logger.warn(`Error ending notifications client: ${String(e)}`);
|
|
477
|
-
}
|
|
564
|
+
this.#retireNotificationsClient(this.notificationsClient);
|
|
478
565
|
}
|
|
479
566
|
await this.pool.end();
|
|
480
567
|
}
|
|
@@ -658,6 +745,7 @@ class SystemDatabase {
|
|
|
658
745
|
'delay_until_epoch_ms',
|
|
659
746
|
'attributes',
|
|
660
747
|
'schedule_name',
|
|
748
|
+
'application_name',
|
|
661
749
|
];
|
|
662
750
|
const client = await this.pool.connect();
|
|
663
751
|
try {
|
|
@@ -675,7 +763,7 @@ class SystemDatabase {
|
|
|
675
763
|
tuples.push(`(${columns.map(() => `$${paramIdx++}`).join(', ')})`);
|
|
676
764
|
params.push(status.workflowUUID, status.status, status.workflowName,
|
|
677
765
|
// For cross-language compatibility, these MUST be NULL in the database when not set
|
|
678
|
-
status.workflowClassName === '' ? null : status.workflowClassName, status.workflowConfigName === '' ? null : status.workflowConfigName, status.queueName ?? null, status.authenticatedUser, status.assumedRole, JSON.stringify(status.authenticatedRoles), JSON.stringify(status.request), status.executorId, status.applicationVersion ?? null, status.applicationID, createdAt, 0, createdAt, status.timeoutMS ?? null, status.deadlineEpochMS ?? null, status.input, null, status.priority, status.queuePartitionKey ?? null, status.parentWorkflowID ?? null, status.serialization, null, status.delayUntilEpochMS ?? null, status.attributes ? JSON.stringify(status.attributes) : null, status.scheduleName ?? null);
|
|
766
|
+
status.workflowClassName === '' ? null : status.workflowClassName, status.workflowConfigName === '' ? null : status.workflowConfigName, status.queueName ?? null, status.authenticatedUser, status.assumedRole, JSON.stringify(status.authenticatedRoles), JSON.stringify(status.request), status.executorId, status.applicationVersion ?? null, status.applicationID, createdAt, 0, createdAt, status.timeoutMS ?? null, status.deadlineEpochMS ?? null, status.input, null, status.priority, status.queuePartitionKey ?? null, status.parentWorkflowID ?? null, status.serialization, null, status.delayUntilEpochMS ?? null, status.attributes ? JSON.stringify(status.attributes) : null, status.scheduleName ?? null, status.applicationName ?? null);
|
|
679
767
|
}
|
|
680
768
|
const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
|
|
681
769
|
VALUES ${tuples.join(', ')}
|
|
@@ -734,15 +822,28 @@ class SystemDatabase {
|
|
|
734
822
|
return rowCount > 0;
|
|
735
823
|
}
|
|
736
824
|
async getPendingWorkflows(executorID, appVersion) {
|
|
825
|
+
const params = [workflow_1.StatusString.PENDING, executorID, appVersion];
|
|
826
|
+
// executor_id defaults to "local", so it collides across applications.
|
|
827
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
737
828
|
const getWorkflows = await this.pool.query(`SELECT workflow_uuid
|
|
738
829
|
FROM "${this.schemaName}".workflow_status
|
|
739
|
-
WHERE status=$1 AND executor_id=$2 AND application_version=$3
|
|
830
|
+
WHERE status=$1 AND executor_id=$2 AND application_version=$3 AND ${scope}`, params);
|
|
740
831
|
return getWorkflows.rows.map((i) => ({
|
|
741
832
|
workflowUUID: i.workflow_uuid,
|
|
742
833
|
}));
|
|
743
834
|
}
|
|
744
835
|
// Recovery re-enqueues rather than executing directly so the queue's atomic dequeue admits exactly one runner, and the executor ID predicate rejects sweeps for rows a live executor has already claimed.
|
|
745
836
|
async reenqueueWorkflowsForRecovery(executorID, appVersion, recoveryQueueName) {
|
|
837
|
+
const params = [
|
|
838
|
+
workflow_1.StatusString.ENQUEUED,
|
|
839
|
+
Date.now(),
|
|
840
|
+
recoveryQueueName,
|
|
841
|
+
workflow_1.StatusString.PENDING,
|
|
842
|
+
executorID,
|
|
843
|
+
appVersion,
|
|
844
|
+
];
|
|
845
|
+
// executor_id defaults to "local", so it collides across applications.
|
|
846
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
746
847
|
const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
747
848
|
SET started_at_epoch_ms = NULL,
|
|
748
849
|
status = $1,
|
|
@@ -751,7 +852,8 @@ class SystemDatabase {
|
|
|
751
852
|
WHERE status = $4
|
|
752
853
|
AND executor_id = $5
|
|
753
854
|
AND application_version = $6
|
|
754
|
-
|
|
855
|
+
AND ${scope}
|
|
856
|
+
RETURNING workflow_uuid`, params);
|
|
755
857
|
return result.rows.map((row) => row.workflow_uuid);
|
|
756
858
|
}
|
|
757
859
|
async getWorkflowStatus(workflowID, callerID, callerFN) {
|
|
@@ -877,9 +979,9 @@ class SystemDatabase {
|
|
|
877
979
|
// Insert a patchmarker
|
|
878
980
|
const dn = Date.now();
|
|
879
981
|
await this.pool.query(`INSERT INTO ${this.schemaName}.operation_outputs
|
|
880
|
-
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms)
|
|
881
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
882
|
-
ON CONFLICT DO NOTHING;`, [workflowID, functionID, null, null, patchName, null, dn, dn]);
|
|
982
|
+
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name)
|
|
983
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
984
|
+
ON CONFLICT DO NOTHING;`, [workflowID, functionID, null, null, patchName, null, dn, dn, this.appName ?? null]);
|
|
883
985
|
return { isPatched: true, hasEntry: true };
|
|
884
986
|
}
|
|
885
987
|
// ==================== Workflow Management ====================
|
|
@@ -937,8 +1039,9 @@ class SystemDatabase {
|
|
|
937
1039
|
* Extend an existing debounced DELAYED workflow's delay and update its inputs, atomically.
|
|
938
1040
|
* The new delay is capped at the workflow's debounce_deadline_epoch_ms, if one is set.
|
|
939
1041
|
* Matching on workflow name and class ensures a debounce-key collision between different
|
|
940
|
-
* workflows never overwrites another workflow's inputs.
|
|
941
|
-
*
|
|
1042
|
+
* workflows never overwrites another workflow's inputs. The bounce acts for `params.applicationName`:
|
|
1043
|
+
* it extends only that application's holders plus unclaimed ones, claiming those. If nothing matched,
|
|
1044
|
+
* returns the current holder (or that the key is unheld) so the caller can start fresh or surface a conflict.
|
|
942
1045
|
* Runs on `client` if given, joining its transaction (e.g. a transactional step's);
|
|
943
1046
|
* otherwise in its own retried transaction.
|
|
944
1047
|
*/
|
|
@@ -966,17 +1069,7 @@ class SystemDatabase {
|
|
|
966
1069
|
}
|
|
967
1070
|
async #debounceDelayedWorkflowInternal(client, params) {
|
|
968
1071
|
const classNameOrNull = params.workflowClassName === '' ? null : params.workflowClassName;
|
|
969
|
-
const
|
|
970
|
-
SET delay_until_epoch_ms = CASE
|
|
971
|
-
WHEN debounce_deadline_epoch_ms IS NOT NULL AND debounce_deadline_epoch_ms < $1
|
|
972
|
-
THEN debounce_deadline_epoch_ms
|
|
973
|
-
ELSE $1
|
|
974
|
-
END,
|
|
975
|
-
inputs = $2, serialization = $3, updated_at = $4
|
|
976
|
-
WHERE name = $5 AND class_name IS NOT DISTINCT FROM $6
|
|
977
|
-
AND queue_name = $7 AND deduplication_id = $8
|
|
978
|
-
AND status = $9 AND is_debounced = TRUE
|
|
979
|
-
RETURNING workflow_uuid`, [
|
|
1072
|
+
const updateParams = [
|
|
980
1073
|
params.delayUntilEpochMS,
|
|
981
1074
|
params.input,
|
|
982
1075
|
params.serialization,
|
|
@@ -986,7 +1079,24 @@ class SystemDatabase {
|
|
|
986
1079
|
params.queueName,
|
|
987
1080
|
params.deduplicationID,
|
|
988
1081
|
workflow_1.StatusString.DELAYED,
|
|
989
|
-
|
|
1082
|
+
params.applicationName ?? null,
|
|
1083
|
+
];
|
|
1084
|
+
// Never extend a workflow the target application doesn't own; falls through to the holder below.
|
|
1085
|
+
const ownScope = this.#appNameFilter('application_name', params.applicationName, updateParams);
|
|
1086
|
+
const updated = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1087
|
+
SET delay_until_epoch_ms = CASE
|
|
1088
|
+
WHEN debounce_deadline_epoch_ms IS NOT NULL AND debounce_deadline_epoch_ms < $1
|
|
1089
|
+
THEN debounce_deadline_epoch_ms
|
|
1090
|
+
ELSE $1
|
|
1091
|
+
END,
|
|
1092
|
+
inputs = $2, serialization = $3, updated_at = $4,
|
|
1093
|
+
-- Claim it for the target, as its dequeue would: left unclaimed, every peer coalesces onto the one workflow and the last inputs win.
|
|
1094
|
+
application_name = COALESCE(application_name, $10)
|
|
1095
|
+
WHERE name = $5 AND class_name IS NOT DISTINCT FROM $6
|
|
1096
|
+
AND queue_name = $7 AND deduplication_id = $8
|
|
1097
|
+
AND status = $9 AND is_debounced = TRUE
|
|
1098
|
+
AND ${ownScope}
|
|
1099
|
+
RETURNING workflow_uuid`, updateParams);
|
|
990
1100
|
if (updated.rows.length > 0) {
|
|
991
1101
|
return {
|
|
992
1102
|
bouncedWorkflowID: updated.rows[0].workflow_uuid,
|
|
@@ -994,10 +1104,11 @@ class SystemDatabase {
|
|
|
994
1104
|
holderIsDebounced: false,
|
|
995
1105
|
holderWorkflowName: null,
|
|
996
1106
|
holderWorkflowClassName: null,
|
|
1107
|
+
holderApplicationName: null,
|
|
997
1108
|
};
|
|
998
1109
|
}
|
|
999
|
-
//
|
|
1000
|
-
const holder = await client.query(`SELECT workflow_uuid, is_debounced, name, class_name
|
|
1110
|
+
// Unscoped, so a holder that blocked the update above is reportable.
|
|
1111
|
+
const holder = await client.query(`SELECT workflow_uuid, is_debounced, name, class_name, application_name
|
|
1001
1112
|
FROM "${this.schemaName}".workflow_status
|
|
1002
1113
|
WHERE queue_name = $1 AND deduplication_id = $2`, [params.queueName, params.deduplicationID]);
|
|
1003
1114
|
if (holder.rows.length === 0) {
|
|
@@ -1007,6 +1118,7 @@ class SystemDatabase {
|
|
|
1007
1118
|
holderIsDebounced: false,
|
|
1008
1119
|
holderWorkflowName: null,
|
|
1009
1120
|
holderWorkflowClassName: null,
|
|
1121
|
+
holderApplicationName: null,
|
|
1010
1122
|
};
|
|
1011
1123
|
}
|
|
1012
1124
|
return {
|
|
@@ -1015,6 +1127,7 @@ class SystemDatabase {
|
|
|
1015
1127
|
holderIsDebounced: holder.rows[0].is_debounced ?? false,
|
|
1016
1128
|
holderWorkflowName: holder.rows[0].name,
|
|
1017
1129
|
holderWorkflowClassName: holder.rows[0].class_name ?? null,
|
|
1130
|
+
holderApplicationName: holder.rows[0].application_name ?? null,
|
|
1018
1131
|
};
|
|
1019
1132
|
}
|
|
1020
1133
|
// Get the immediate (one-level) child workflow IDs for a set of workflows.
|
|
@@ -1126,7 +1239,7 @@ class SystemDatabase {
|
|
|
1126
1239
|
// Fetch the status of all original workflows inside the transaction.
|
|
1127
1240
|
const { rows: statusRows } = await client.query(`SELECT workflow_uuid, name, class_name, config_name, application_id,
|
|
1128
1241
|
authenticated_user, authenticated_roles, assumed_role, inputs, serialization,
|
|
1129
|
-
request, application_version, attributes
|
|
1242
|
+
request, application_version, attributes, application_name
|
|
1130
1243
|
FROM "${this.schemaName}".workflow_status
|
|
1131
1244
|
WHERE workflow_uuid = ANY($1)`, [originalWorkflowIDs]);
|
|
1132
1245
|
const statusByID = new Map(statusRows.map((r) => [r.workflow_uuid, r]));
|
|
@@ -1155,10 +1268,16 @@ class SystemDatabase {
|
|
|
1155
1268
|
'forked_from',
|
|
1156
1269
|
'serialization',
|
|
1157
1270
|
'attributes',
|
|
1271
|
+
'application_name',
|
|
1158
1272
|
];
|
|
1159
1273
|
if (options.timeoutMS !== undefined) {
|
|
1160
1274
|
insertCols.push('workflow_timeout_ms');
|
|
1161
1275
|
}
|
|
1276
|
+
// One owner per fork, shared by its status row and its copied steps: the source's, or this application claiming an unclaimed one.
|
|
1277
|
+
const forkOwners = new Map(originalWorkflowIDs.map((origID, i) => [
|
|
1278
|
+
forkedWorkflowIDs[i],
|
|
1279
|
+
statusByID.get(origID).application_name ?? this.appName ?? null,
|
|
1280
|
+
]));
|
|
1162
1281
|
const valuesPlaceholders = [];
|
|
1163
1282
|
const params = [];
|
|
1164
1283
|
let paramIdx = 1;
|
|
@@ -1168,7 +1287,7 @@ class SystemDatabase {
|
|
|
1168
1287
|
const ws = statusByID.get(origID);
|
|
1169
1288
|
const placeholders = insertCols.map(() => `$${paramIdx++}`).join(', ');
|
|
1170
1289
|
valuesPlaceholders.push(`(${placeholders})`);
|
|
1171
|
-
params.push(forkID, workflow_1.StatusString.ENQUEUED, ws.name, ws.class_name ?? null, ws.config_name ?? null, queueName, ws.authenticated_user, ws.assumed_role, ws.authenticated_roles, ws.request, options.applicationVersion ?? ws.application_version ?? null, ws.application_id, ws.inputs, options.queuePartitionKey ?? null, origID, ws.serialization, ws.attributes ? JSON.stringify(ws.attributes) : null);
|
|
1290
|
+
params.push(forkID, workflow_1.StatusString.ENQUEUED, ws.name, ws.class_name ?? null, ws.config_name ?? null, queueName, ws.authenticated_user, ws.assumed_role, ws.authenticated_roles, ws.request, options.applicationVersion ?? ws.application_version ?? null, ws.application_id, ws.inputs, options.queuePartitionKey ?? null, origID, ws.serialization, ws.attributes ? JSON.stringify(ws.attributes) : null, forkOwners.get(forkID) ?? null);
|
|
1172
1291
|
if (options.timeoutMS !== undefined) {
|
|
1173
1292
|
params.push(options.timeoutMS);
|
|
1174
1293
|
}
|
|
@@ -1187,11 +1306,12 @@ class SystemDatabase {
|
|
|
1187
1306
|
const mappingParams = [];
|
|
1188
1307
|
let mIdx = 1;
|
|
1189
1308
|
for (const m of forkMappings) {
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1309
|
+
// Cast: an unclaimed fork makes owner a bare NULL the VALUES list cannot type.
|
|
1310
|
+
mappingValues.push(`($${mIdx}::text, $${mIdx + 1}::text, $${mIdx + 2}::int, $${mIdx + 3}::text)`);
|
|
1311
|
+
mappingParams.push(m.origID, m.forkID, m.startStep, forkOwners.get(m.forkID) ?? null);
|
|
1312
|
+
mIdx += 4;
|
|
1193
1313
|
}
|
|
1194
|
-
const mappingCTE = `WITH mapping(orig_id, fork_id, start_step) AS (VALUES ${mappingValues.join(', ')})`;
|
|
1314
|
+
const mappingCTE = `WITH mapping(orig_id, fork_id, start_step, owner) AS (VALUES ${mappingValues.join(', ')})`;
|
|
1195
1315
|
// Build the child_workflow_id expression, applying replacements if provided.
|
|
1196
1316
|
let childWfExpr = 'oo.child_workflow_id';
|
|
1197
1317
|
const ooParams = [...mappingParams];
|
|
@@ -1207,8 +1327,8 @@ class SystemDatabase {
|
|
|
1207
1327
|
// Copy operation outputs
|
|
1208
1328
|
await client.query(`${mappingCTE}
|
|
1209
1329
|
INSERT INTO "${this.schemaName}".operation_outputs
|
|
1210
|
-
(workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms)
|
|
1211
|
-
SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms
|
|
1330
|
+
(workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name)
|
|
1331
|
+
SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner
|
|
1212
1332
|
FROM mapping m
|
|
1213
1333
|
JOIN "${this.schemaName}".operation_outputs oo
|
|
1214
1334
|
ON oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
|
|
@@ -1276,7 +1396,7 @@ class SystemDatabase {
|
|
|
1276
1396
|
deduplication_id, inputs, priority, queue_partition_key, forked_from,
|
|
1277
1397
|
parent_workflow_id, serialization, delay_until_epoch_ms,
|
|
1278
1398
|
was_forked_from, rate_limited, completed_at, attributes, schedule_name,
|
|
1279
|
-
debounce_deadline_epoch_ms, is_debounced
|
|
1399
|
+
debounce_deadline_epoch_ms, is_debounced, application_name
|
|
1280
1400
|
FROM "${this.schemaName}".workflow_status
|
|
1281
1401
|
WHERE workflow_uuid = $1`, [wfID]);
|
|
1282
1402
|
if (statusResult.rows.length === 0) {
|
|
@@ -1287,7 +1407,7 @@ class SystemDatabase {
|
|
|
1287
1407
|
const outputsResult = await client.query(`SELECT
|
|
1288
1408
|
workflow_uuid, function_id, function_name, output, error,
|
|
1289
1409
|
child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
|
|
1290
|
-
serialization
|
|
1410
|
+
serialization, application_name
|
|
1291
1411
|
FROM "${this.schemaName}".operation_outputs
|
|
1292
1412
|
WHERE workflow_uuid = $1`, [wfID]);
|
|
1293
1413
|
// Export workflow_events
|
|
@@ -1332,8 +1452,8 @@ class SystemDatabase {
|
|
|
1332
1452
|
deduplication_id, inputs, priority, queue_partition_key, forked_from,
|
|
1333
1453
|
parent_workflow_id, serialization, delay_until_epoch_ms,
|
|
1334
1454
|
was_forked_from, rate_limited, completed_at, attributes, schedule_name,
|
|
1335
|
-
debounce_deadline_epoch_ms, is_debounced
|
|
1336
|
-
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)`, [
|
|
1455
|
+
debounce_deadline_epoch_ms, is_debounced, application_name
|
|
1456
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37)`, [
|
|
1337
1457
|
status.workflow_uuid,
|
|
1338
1458
|
status.status,
|
|
1339
1459
|
status.name,
|
|
@@ -1372,14 +1492,15 @@ class SystemDatabase {
|
|
|
1372
1492
|
status.schedule_name ?? null,
|
|
1373
1493
|
status.debounce_deadline_epoch_ms ?? null,
|
|
1374
1494
|
status.is_debounced ?? false,
|
|
1495
|
+
status.application_name ?? null,
|
|
1375
1496
|
]);
|
|
1376
1497
|
// Import operation_outputs
|
|
1377
1498
|
for (const output of workflow.operation_outputs) {
|
|
1378
1499
|
await client.query(`INSERT INTO "${this.schemaName}".operation_outputs (
|
|
1379
1500
|
workflow_uuid, function_id, function_name, output, error,
|
|
1380
1501
|
child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
|
|
1381
|
-
serialization
|
|
1382
|
-
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [
|
|
1502
|
+
serialization, application_name
|
|
1503
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, [
|
|
1383
1504
|
output.workflow_uuid,
|
|
1384
1505
|
output.function_id,
|
|
1385
1506
|
output.function_name,
|
|
@@ -1389,6 +1510,7 @@ class SystemDatabase {
|
|
|
1389
1510
|
output.started_at_epoch_ms,
|
|
1390
1511
|
output.completed_at_epoch_ms,
|
|
1391
1512
|
output.serialization,
|
|
1513
|
+
output.application_name ?? null,
|
|
1392
1514
|
]);
|
|
1393
1515
|
}
|
|
1394
1516
|
// Import workflow_events
|
|
@@ -2092,10 +2214,13 @@ class SystemDatabase {
|
|
|
2092
2214
|
// Transition workflows from DELAYED to ENQUEUED when their delay has expired.
|
|
2093
2215
|
// For debounced workflows, clear the deduplication ID in the same atomic update: it is a
|
|
2094
2216
|
// debounce key held only while DELAYED, so a later same-key debounce starts a fresh workflow.
|
|
2217
|
+
const params = [workflow_1.StatusString.ENQUEUED, Date.now(), workflow_1.StatusString.DELAYED];
|
|
2218
|
+
// Only what this application would dequeue: a peer's debounce key is not ours to clear.
|
|
2219
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
2095
2220
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2096
2221
|
SET status = $1, updated_at = $2,
|
|
2097
2222
|
deduplication_id = CASE WHEN is_debounced THEN NULL ELSE deduplication_id END
|
|
2098
|
-
WHERE status = $3 AND delay_until_epoch_ms <= $2
|
|
2223
|
+
WHERE status = $3 AND delay_until_epoch_ms <= $2 AND ${scope}`, params);
|
|
2099
2224
|
}
|
|
2100
2225
|
async getDeduplicatedWorkflow(queueName, deduplicationID) {
|
|
2101
2226
|
const { rows } = await this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
@@ -2107,18 +2232,21 @@ class SystemDatabase {
|
|
|
2107
2232
|
}
|
|
2108
2233
|
async getQueuePartitions(queueName) {
|
|
2109
2234
|
// Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v2, so cost scales with the number of partitions rather than the backlog depth.
|
|
2235
|
+
const params = [queueName, workflow_1.StatusString.ENQUEUED];
|
|
2236
|
+
// Only partitions this application can actually dequeue from.
|
|
2237
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
2110
2238
|
const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
|
|
2111
2239
|
(SELECT MIN(queue_partition_key) AS pk
|
|
2112
2240
|
FROM "${this.schemaName}".workflow_status
|
|
2113
|
-
WHERE queue_name = $1 AND status = $2 AND queue_partition_key IS NOT NULL)
|
|
2241
|
+
WHERE queue_name = $1 AND status = $2 AND queue_partition_key IS NOT NULL AND ${scope})
|
|
2114
2242
|
UNION ALL
|
|
2115
2243
|
(SELECT (SELECT MIN(queue_partition_key)
|
|
2116
2244
|
FROM "${this.schemaName}".workflow_status
|
|
2117
|
-
WHERE queue_name = $1 AND status = $2 AND queue_partition_key > partitions.pk)
|
|
2245
|
+
WHERE queue_name = $1 AND status = $2 AND queue_partition_key > partitions.pk AND ${scope})
|
|
2118
2246
|
FROM partitions
|
|
2119
2247
|
WHERE partitions.pk IS NOT NULL)
|
|
2120
2248
|
)
|
|
2121
|
-
SELECT pk FROM partitions WHERE pk IS NOT NULL`,
|
|
2249
|
+
SELECT pk FROM partitions WHERE pk IS NOT NULL`, params);
|
|
2122
2250
|
return rows.map((row) => row.pk);
|
|
2123
2251
|
}
|
|
2124
2252
|
async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey) {
|
|
@@ -2152,11 +2280,14 @@ class SystemDatabase {
|
|
|
2152
2280
|
startTimeMs - limiterPeriodMS,
|
|
2153
2281
|
...partitionParams,
|
|
2154
2282
|
];
|
|
2283
|
+
// Count only what this application would dequeue, matching the select below.
|
|
2284
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
2155
2285
|
const countResult = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
2156
2286
|
WHERE queue_name = $1
|
|
2157
2287
|
AND rate_limited = TRUE
|
|
2158
2288
|
AND status NOT IN ($2, $3)
|
|
2159
2289
|
AND started_at_epoch_ms > $4
|
|
2290
|
+
AND ${scope}
|
|
2160
2291
|
${partitionFilter.replace('$PARTITION', '$5')}`, params);
|
|
2161
2292
|
numRecentQueries = Number(countResult.rows[0].count);
|
|
2162
2293
|
if (numRecentQueries >= queue.rateLimit.limitPerPeriod) {
|
|
@@ -2175,9 +2306,10 @@ class SystemDatabase {
|
|
|
2175
2306
|
if (queue.concurrency !== undefined) {
|
|
2176
2307
|
// Global concurrency still requires a DB query since other workers may be running workflows too.
|
|
2177
2308
|
const params = [queue.name, workflow_1.StatusString.PENDING, ...partitionParams];
|
|
2309
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
2178
2310
|
const runningTasksResult = await client.query(`SELECT COUNT(*) as task_count
|
|
2179
2311
|
FROM "${this.schemaName}".workflow_status
|
|
2180
|
-
WHERE queue_name = $1 AND status = $2
|
|
2312
|
+
WHERE queue_name = $1 AND status = $2 AND ${scope}
|
|
2181
2313
|
${partitionFilter.replace('$PARTITION', '$3')}`, params);
|
|
2182
2314
|
const totalRunningTasks = Number(runningTasksResult.rows[0]?.task_count ?? 0);
|
|
2183
2315
|
if (totalRunningTasks > queue.concurrency) {
|
|
@@ -2192,10 +2324,7 @@ class SystemDatabase {
|
|
|
2192
2324
|
return claimedIDs;
|
|
2193
2325
|
}
|
|
2194
2326
|
// Retrieve the first max_tasks workflows in the queue.
|
|
2195
|
-
const
|
|
2196
|
-
FROM "${this.schemaName}".application_versions
|
|
2197
|
-
ORDER BY version_timestamp DESC LIMIT 1`);
|
|
2198
|
-
const latestVersion = latestVersionResult.rows[0]?.version_name;
|
|
2327
|
+
const latestVersion = await this.#latestApplicationVersionName(client);
|
|
2199
2328
|
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
2200
2329
|
const versionClause = isLatestVersion
|
|
2201
2330
|
? '(application_version = $3 OR application_version IS NULL)'
|
|
@@ -2203,12 +2332,14 @@ class SystemDatabase {
|
|
|
2203
2332
|
const lockMode = queue.concurrency ? 'FOR UPDATE NOWAIT' : 'FOR UPDATE SKIP LOCKED';
|
|
2204
2333
|
const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
|
|
2205
2334
|
const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
|
|
2335
|
+
const selectScope = this.#appNameFilter('application_name', this.appName, selectParams);
|
|
2206
2336
|
const selectQuery = `
|
|
2207
2337
|
SELECT workflow_uuid
|
|
2208
2338
|
FROM "${this.schemaName}".workflow_status
|
|
2209
2339
|
WHERE status = $1
|
|
2210
2340
|
AND queue_name = $2
|
|
2211
2341
|
AND ${versionClause}
|
|
2342
|
+
AND ${selectScope}
|
|
2212
2343
|
${partitionFilter.replace('$PARTITION', '$4')}
|
|
2213
2344
|
ORDER BY priority ASC, created_at ASC
|
|
2214
2345
|
${limitClause}
|
|
@@ -2229,26 +2360,32 @@ class SystemDatabase {
|
|
|
2229
2360
|
// Start the functions by marking them as pending and updating their executor IDs.
|
|
2230
2361
|
// Only claim the workflow if the UPDATE actually transitioned an ENQUEUED row —
|
|
2231
2362
|
// otherwise another worker won the race and we must not re-dispatch it.
|
|
2363
|
+
const updateParams = [
|
|
2364
|
+
workflow_1.StatusString.PENDING,
|
|
2365
|
+
executorID,
|
|
2366
|
+
appVersion,
|
|
2367
|
+
startTimeMs,
|
|
2368
|
+
queue.rateLimit !== undefined,
|
|
2369
|
+
id,
|
|
2370
|
+
workflow_1.StatusString.ENQUEUED,
|
|
2371
|
+
// Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
|
|
2372
|
+
this.appName ?? null,
|
|
2373
|
+
];
|
|
2374
|
+
// Re-check ownership alongside status, as the partitioned claim guard does.
|
|
2375
|
+
const claimScope = this.#appNameFilter('application_name', this.appName, updateParams);
|
|
2232
2376
|
const updateRes = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2233
2377
|
SET status = $1,
|
|
2234
2378
|
executor_id = $2,
|
|
2235
2379
|
application_version = $3,
|
|
2236
2380
|
started_at_epoch_ms = $4,
|
|
2237
2381
|
rate_limited = $5,
|
|
2382
|
+
application_name = COALESCE(application_name, $8),
|
|
2238
2383
|
workflow_deadline_epoch_ms = CASE
|
|
2239
2384
|
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
|
|
2240
2385
|
THEN (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
|
|
2241
2386
|
ELSE workflow_deadline_epoch_ms
|
|
2242
2387
|
END
|
|
2243
|
-
WHERE workflow_uuid = $6 AND status = $7`,
|
|
2244
|
-
workflow_1.StatusString.PENDING,
|
|
2245
|
-
executorID,
|
|
2246
|
-
appVersion,
|
|
2247
|
-
startTimeMs,
|
|
2248
|
-
queue.rateLimit !== undefined,
|
|
2249
|
-
id,
|
|
2250
|
-
workflow_1.StatusString.ENQUEUED,
|
|
2251
|
-
]);
|
|
2388
|
+
WHERE workflow_uuid = $6 AND status = $7 AND ${claimScope}`, updateParams);
|
|
2252
2389
|
if ((updateRes.rowCount ?? 0) > 0) {
|
|
2253
2390
|
claimedIDs.push(id);
|
|
2254
2391
|
}
|
|
@@ -2277,23 +2414,28 @@ class SystemDatabase {
|
|
|
2277
2414
|
const client = await this.pool.connect();
|
|
2278
2415
|
try {
|
|
2279
2416
|
await client.query('BEGIN');
|
|
2280
|
-
const
|
|
2281
|
-
FROM "${this.schemaName}".application_versions
|
|
2282
|
-
ORDER BY version_timestamp DESC LIMIT 1`);
|
|
2283
|
-
const latestVersion = latestVersionResult.rows[0]?.version_name;
|
|
2417
|
+
const latestVersion = await this.#latestApplicationVersionName(client);
|
|
2284
2418
|
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
2285
2419
|
const versionClause = (n) => isLatestVersion
|
|
2286
2420
|
? `(application_version = $${n} OR application_version IS NULL)`
|
|
2287
2421
|
: `application_version = $${n}`;
|
|
2422
|
+
const candidateParams = [
|
|
2423
|
+
queue.name,
|
|
2424
|
+
workflow_1.StatusString.ENQUEUED,
|
|
2425
|
+
appVersion,
|
|
2426
|
+
workflow_1.StatusString.PENDING,
|
|
2427
|
+
this.partitionedDequeueSweepCap,
|
|
2428
|
+
];
|
|
2429
|
+
const candidateScope = this.#appNameFilter('application_name', this.appName, candidateParams);
|
|
2288
2430
|
// Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
|
|
2289
2431
|
const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
|
|
2290
2432
|
(SELECT MIN(queue_partition_key) AS pk
|
|
2291
2433
|
FROM "${this.schemaName}".workflow_status
|
|
2292
|
-
WHERE queue_name = $1 AND status = $2 AND queue_partition_key IS NOT NULL)
|
|
2434
|
+
WHERE queue_name = $1 AND status = $2 AND queue_partition_key IS NOT NULL AND ${candidateScope})
|
|
2293
2435
|
UNION ALL
|
|
2294
2436
|
(SELECT (SELECT MIN(queue_partition_key)
|
|
2295
2437
|
FROM "${this.schemaName}".workflow_status
|
|
2296
|
-
WHERE queue_name = $1 AND status = $2 AND queue_partition_key > partitions.pk)
|
|
2438
|
+
WHERE queue_name = $1 AND status = $2 AND queue_partition_key > partitions.pk AND ${candidateScope})
|
|
2297
2439
|
FROM partitions
|
|
2298
2440
|
WHERE partitions.pk IS NOT NULL)
|
|
2299
2441
|
)
|
|
@@ -2306,12 +2448,13 @@ class SystemDatabase {
|
|
|
2306
2448
|
WHERE queue_name = $1 AND status = $2
|
|
2307
2449
|
AND queue_partition_key = partitions.pk
|
|
2308
2450
|
AND ${versionClause(3)}
|
|
2451
|
+
AND ${candidateScope}
|
|
2309
2452
|
-- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
|
|
2310
2453
|
ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
|
|
2311
2454
|
LIMIT 1
|
|
2312
2455
|
) head ON TRUE
|
|
2313
2456
|
WHERE partitions.pk IS NOT NULL
|
|
2314
|
-
--
|
|
2457
|
+
-- Unscoped by design: a mutual-exclusion probe must block on any owner's row.
|
|
2315
2458
|
AND NOT EXISTS (
|
|
2316
2459
|
SELECT 1
|
|
2317
2460
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -2319,7 +2462,7 @@ class SystemDatabase {
|
|
|
2319
2462
|
AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
|
|
2320
2463
|
)
|
|
2321
2464
|
ORDER BY partitions.pk ASC
|
|
2322
|
-
LIMIT $5`,
|
|
2465
|
+
LIMIT $5`, candidateParams);
|
|
2323
2466
|
const candidateIDs = candidateResult.rows.map((row) => row.workflow_uuid);
|
|
2324
2467
|
if (candidateIDs.length === 0) {
|
|
2325
2468
|
await client.query('COMMIT');
|
|
@@ -2327,16 +2470,19 @@ class SystemDatabase {
|
|
|
2327
2470
|
}
|
|
2328
2471
|
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
|
|
2329
2472
|
// Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
|
|
2330
|
-
const claimGuard = (ids, status, name, version) => `workflow_uuid = ANY($${ids}::text[])
|
|
2473
|
+
const claimGuard = (ids, status, name, version, scope) => `workflow_uuid = ANY($${ids}::text[])
|
|
2331
2474
|
AND status = $${status}
|
|
2332
2475
|
AND queue_name = $${name}
|
|
2333
2476
|
AND queue_partition_key IS NOT NULL
|
|
2334
|
-
AND ${versionClause(version)}
|
|
2477
|
+
AND ${versionClause(version)}
|
|
2478
|
+
AND ${scope}`;
|
|
2479
|
+
const lockedParams = [candidateIDs, workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
|
|
2480
|
+
const lockedScope = this.#appNameFilter('application_name', this.appName, lockedParams);
|
|
2335
2481
|
// Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
|
|
2336
2482
|
const lockedResult = await client.query(`SELECT workflow_uuid
|
|
2337
2483
|
FROM "${this.schemaName}".workflow_status
|
|
2338
|
-
WHERE ${claimGuard(1, 2, 3, 4)}
|
|
2339
|
-
FOR UPDATE SKIP LOCKED`,
|
|
2484
|
+
WHERE ${claimGuard(1, 2, 3, 4, lockedScope)}
|
|
2485
|
+
FOR UPDATE SKIP LOCKED`, lockedParams);
|
|
2340
2486
|
const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
|
|
2341
2487
|
// Preserve partition order for submission.
|
|
2342
2488
|
const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
|
|
@@ -2344,6 +2490,18 @@ class SystemDatabase {
|
|
|
2344
2490
|
await client.query('COMMIT');
|
|
2345
2491
|
return [];
|
|
2346
2492
|
}
|
|
2493
|
+
const flipParams = [
|
|
2494
|
+
workflow_1.StatusString.PENDING,
|
|
2495
|
+
executorID,
|
|
2496
|
+
claimIDs,
|
|
2497
|
+
workflow_1.StatusString.ENQUEUED,
|
|
2498
|
+
queue.name,
|
|
2499
|
+
appVersion,
|
|
2500
|
+
startTimeMs,
|
|
2501
|
+
// Claim the row, as the unpartitioned dequeue does.
|
|
2502
|
+
this.appName ?? null,
|
|
2503
|
+
];
|
|
2504
|
+
const flipScope = this.#appNameFilter('application_name', this.appName, flipParams);
|
|
2347
2505
|
// Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
|
|
2348
2506
|
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2349
2507
|
SET status = $1,
|
|
@@ -2351,13 +2509,14 @@ class SystemDatabase {
|
|
|
2351
2509
|
application_version = $6,
|
|
2352
2510
|
started_at_epoch_ms = $7,
|
|
2353
2511
|
rate_limited = FALSE,
|
|
2512
|
+
application_name = COALESCE(application_name, $8),
|
|
2354
2513
|
workflow_deadline_epoch_ms = CASE
|
|
2355
2514
|
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
|
|
2356
2515
|
THEN (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
|
|
2357
2516
|
ELSE workflow_deadline_epoch_ms
|
|
2358
2517
|
END
|
|
2359
|
-
WHERE ${claimGuard(3, 4, 5, 6)}
|
|
2360
|
-
RETURNING workflow_uuid`,
|
|
2518
|
+
WHERE ${claimGuard(3, 4, 5, 6, flipScope)}
|
|
2519
|
+
RETURNING workflow_uuid`, flipParams);
|
|
2361
2520
|
await client.query('COMMIT');
|
|
2362
2521
|
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
2363
2522
|
const claimedIDs = claimIDs.filter((id) => flippedIDs.has(id));
|
|
@@ -2408,6 +2567,7 @@ class SystemDatabase {
|
|
|
2408
2567
|
'schedule_name',
|
|
2409
2568
|
'debounce_deadline_epoch_ms',
|
|
2410
2569
|
'is_debounced',
|
|
2570
|
+
'application_name',
|
|
2411
2571
|
];
|
|
2412
2572
|
input.loadInput = input.loadInput ?? true;
|
|
2413
2573
|
input.loadOutput = input.loadOutput ?? true;
|
|
@@ -2452,6 +2612,15 @@ class SystemDatabase {
|
|
|
2452
2612
|
addFilter('name', input.workflowName);
|
|
2453
2613
|
addFilter('queue_name', input.queueName);
|
|
2454
2614
|
addFilter('schedule_name', input.scheduleName);
|
|
2615
|
+
// A workflow ID is a global address, so an ID-keyed read is an identity read: it takes an
|
|
2616
|
+
// explicit filter but is never defaulted to this one. Otherwise unset scopes to this
|
|
2617
|
+
// application, as on every other observability query.
|
|
2618
|
+
// Falsy, not just undefined: a Conductor request carries an omitted list as JSON null.
|
|
2619
|
+
const idKeyed = (input.workflowIDs?.length ?? 0) > 0;
|
|
2620
|
+
whereClauses.push(idKeyed
|
|
2621
|
+
? this.#appNameFilter('application_name', input.applicationName, params)
|
|
2622
|
+
: this.#observabilityFilter('application_name', input.applicationName, params));
|
|
2623
|
+
paramCounter = params.length + 1;
|
|
2455
2624
|
if (input.workflow_id_prefix) {
|
|
2456
2625
|
if (Array.isArray(input.workflow_id_prefix)) {
|
|
2457
2626
|
const likeClauses = input.workflow_id_prefix.map((_, i) => `workflow_uuid LIKE $${paramCounter + i}`);
|
|
@@ -2554,6 +2723,7 @@ class SystemDatabase {
|
|
|
2554
2723
|
['queue_name', input.groupByQueueName ?? false, 'queue_name'],
|
|
2555
2724
|
['executor_id', input.groupByExecutorId ?? false, 'executor_id'],
|
|
2556
2725
|
['application_version', input.groupByApplicationVersion ?? false, 'application_version'],
|
|
2726
|
+
['application_name', input.groupByApplicationName ?? false, 'application_name'],
|
|
2557
2727
|
];
|
|
2558
2728
|
const groupNames = [];
|
|
2559
2729
|
const groupColumns = [];
|
|
@@ -2624,6 +2794,9 @@ class SystemDatabase {
|
|
|
2624
2794
|
addFilter('forked_from', input.forkedFrom);
|
|
2625
2795
|
addFilter('parent_workflow_id', input.parentWorkflowID);
|
|
2626
2796
|
addFilter('schedule_name', input.scheduleName);
|
|
2797
|
+
// Unset scopes to this application, as on every other observability query.
|
|
2798
|
+
whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
|
|
2799
|
+
paramIdx = params.length + 1;
|
|
2627
2800
|
// Only workflows that are actively enqueued.
|
|
2628
2801
|
if (input.queuesOnly) {
|
|
2629
2802
|
whereClauses.push(`queue_name IS NOT NULL`);
|
|
@@ -2786,6 +2959,9 @@ class SystemDatabase {
|
|
|
2786
2959
|
params.push(new Date(input.completedBefore).getTime());
|
|
2787
2960
|
paramIdx++;
|
|
2788
2961
|
}
|
|
2962
|
+
// Unset scopes to this application, as on every other observability query.
|
|
2963
|
+
whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
|
|
2964
|
+
paramIdx = params.length + 1;
|
|
2789
2965
|
const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
|
|
2790
2966
|
const groupByClause = groupColumns.join(', ');
|
|
2791
2967
|
const selectClause = [...groupSelectColumns, ...selectColumns].join(', ');
|
|
@@ -2814,10 +2990,13 @@ class SystemDatabase {
|
|
|
2814
2990
|
async garbageCollect(cutoffEpochTimestampMs, rowsThreshold) {
|
|
2815
2991
|
if (rowsThreshold !== undefined && rowsThreshold !== null) {
|
|
2816
2992
|
// Get the created_at timestamp of the rows_threshold newest row
|
|
2993
|
+
const params = [rowsThreshold - 1];
|
|
2994
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
2817
2995
|
const result = await this.pool.query(`SELECT created_at
|
|
2818
2996
|
FROM "${this.schemaName}".workflow_status
|
|
2997
|
+
WHERE ${scope}
|
|
2819
2998
|
ORDER BY created_at DESC
|
|
2820
|
-
LIMIT 1 OFFSET $1`,
|
|
2999
|
+
LIMIT 1 OFFSET $1`, params);
|
|
2821
3000
|
if (result.rows.length > 0) {
|
|
2822
3001
|
const rowsBasedCutoff = result.rows[0].created_at;
|
|
2823
3002
|
// Use the more restrictive cutoff (higher timestamp = more recent = more deletion)
|
|
@@ -2831,21 +3010,51 @@ class SystemDatabase {
|
|
|
2831
3010
|
if (cutoffEpochTimestampMs === undefined || cutoffEpochTimestampMs === null) {
|
|
2832
3011
|
return;
|
|
2833
3012
|
}
|
|
3013
|
+
const deleteParams = [
|
|
3014
|
+
cutoffEpochTimestampMs,
|
|
3015
|
+
workflow_1.StatusString.PENDING,
|
|
3016
|
+
workflow_1.StatusString.ENQUEUED,
|
|
3017
|
+
workflow_1.StatusString.DELAYED,
|
|
3018
|
+
];
|
|
3019
|
+
// Unclaimed rows included: excluding them would leak pre-upgrade rows forever.
|
|
3020
|
+
const deleteScope = this.#appNameFilter('application_name', this.appName, deleteParams);
|
|
2834
3021
|
// Delete all workflows older than cutoff that are NOT PENDING, ENQUEUED, or DELAYED
|
|
2835
3022
|
await this.pool.query(`DELETE FROM "${this.schemaName}".workflow_status
|
|
2836
3023
|
WHERE created_at < $1
|
|
2837
|
-
AND status NOT IN ($2, $3, $4)
|
|
3024
|
+
AND status NOT IN ($2, $3, $4)
|
|
3025
|
+
AND ${deleteScope}`, deleteParams);
|
|
2838
3026
|
return;
|
|
2839
3027
|
}
|
|
2840
|
-
|
|
3028
|
+
/**
|
|
3029
|
+
* IDs of this application's in-flight workflows created at or before the cutoff.
|
|
3030
|
+
* Claiming-scoped, so an upgrade still times out its own unclaimed workflows.
|
|
3031
|
+
*/
|
|
3032
|
+
async listTimedOutWorkflowIds(cutoffEpochTimestampMs) {
|
|
3033
|
+
const params = [
|
|
3034
|
+
cutoffEpochTimestampMs,
|
|
3035
|
+
workflow_1.StatusString.PENDING,
|
|
3036
|
+
workflow_1.StatusString.ENQUEUED,
|
|
3037
|
+
workflow_1.StatusString.DELAYED,
|
|
3038
|
+
];
|
|
3039
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
3040
|
+
const { rows } = await this.pool.query(`SELECT workflow_uuid
|
|
3041
|
+
FROM "${this.schemaName}".workflow_status
|
|
3042
|
+
WHERE created_at <= $1
|
|
3043
|
+
AND status IN ($2, $3, $4)
|
|
3044
|
+
AND ${scope}`, params);
|
|
3045
|
+
return rows.map((row) => row.workflow_uuid);
|
|
3046
|
+
}
|
|
3047
|
+
async getMetrics(startTime, endTime, applicationName) {
|
|
2841
3048
|
const startEpochMs = new Date(startTime).getTime();
|
|
2842
3049
|
const endEpochMs = new Date(endTime).getTime();
|
|
2843
3050
|
const metrics = [];
|
|
2844
3051
|
// Query workflow metrics
|
|
3052
|
+
const workflowParams = [startEpochMs, endEpochMs];
|
|
3053
|
+
const workflowScope = this.#observabilityFilter('application_name', applicationName, workflowParams);
|
|
2845
3054
|
const workflowResult = await this.pool.query(`SELECT name, COUNT(workflow_uuid) as count
|
|
2846
3055
|
FROM "${this.schemaName}".workflow_status
|
|
2847
|
-
WHERE created_at >= $1 AND created_at < $2
|
|
2848
|
-
GROUP BY name`,
|
|
3056
|
+
WHERE created_at >= $1 AND created_at < $2 AND ${workflowScope}
|
|
3057
|
+
GROUP BY name`, workflowParams);
|
|
2849
3058
|
for (const row of workflowResult.rows) {
|
|
2850
3059
|
metrics.push({
|
|
2851
3060
|
metricType: 'workflow_count',
|
|
@@ -2854,10 +3063,12 @@ class SystemDatabase {
|
|
|
2854
3063
|
});
|
|
2855
3064
|
}
|
|
2856
3065
|
// Query step metrics
|
|
3066
|
+
const stepParams = [startEpochMs, endEpochMs];
|
|
3067
|
+
const stepScope = this.#observabilityFilter('application_name', applicationName, stepParams);
|
|
2857
3068
|
const stepResult = await this.pool.query(`SELECT function_name, COUNT(*) as count
|
|
2858
3069
|
FROM "${this.schemaName}".operation_outputs
|
|
2859
|
-
WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2
|
|
2860
|
-
GROUP BY function_name`,
|
|
3070
|
+
WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2 AND ${stepScope}
|
|
3071
|
+
GROUP BY function_name`, stepParams);
|
|
2861
3072
|
for (const row of stepResult.rows) {
|
|
2862
3073
|
metrics.push({
|
|
2863
3074
|
metricType: 'step_count',
|
|
@@ -2870,10 +3081,11 @@ class SystemDatabase {
|
|
|
2870
3081
|
// ==================== Scheduling ====================
|
|
2871
3082
|
async createSchedule(schedule, client) {
|
|
2872
3083
|
const q = client ?? this.pool;
|
|
3084
|
+
const owner = await this.#resolveRowOwner(q, 'workflow_schedules', 'schedule_name', schedule.scheduleName, schedule.applicationName, 'Schedule');
|
|
2873
3085
|
try {
|
|
2874
3086
|
await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
2875
|
-
(schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name)
|
|
2876
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, [
|
|
3087
|
+
(schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name)
|
|
3088
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, [
|
|
2877
3089
|
schedule.scheduleId,
|
|
2878
3090
|
schedule.scheduleName,
|
|
2879
3091
|
schedule.workflowName,
|
|
@@ -2885,6 +3097,7 @@ class SystemDatabase {
|
|
|
2885
3097
|
schedule.automaticBackfill,
|
|
2886
3098
|
schedule.cronTimezone,
|
|
2887
3099
|
schedule.queueName,
|
|
3100
|
+
owner ?? null,
|
|
2888
3101
|
]);
|
|
2889
3102
|
}
|
|
2890
3103
|
catch (e) {
|
|
@@ -2894,6 +3107,10 @@ class SystemDatabase {
|
|
|
2894
3107
|
throw e;
|
|
2895
3108
|
}
|
|
2896
3109
|
}
|
|
3110
|
+
/**
|
|
3111
|
+
* List only schedules owned by these applications, plus unclaimed ones.
|
|
3112
|
+
* By default, only list this application's schedules.
|
|
3113
|
+
*/
|
|
2897
3114
|
async listSchedules(filters, client) {
|
|
2898
3115
|
const q = client ?? this.pool;
|
|
2899
3116
|
const conditions = [];
|
|
@@ -2925,45 +3142,23 @@ class SystemDatabase {
|
|
|
2925
3142
|
});
|
|
2926
3143
|
conditions.push(`(${likeClauses.join(' OR ')})`);
|
|
2927
3144
|
}
|
|
3145
|
+
// Unset scopes to this application, as on every other observability query.
|
|
3146
|
+
conditions.push(this.#observabilityFilter('application_name', filters?.applicationName, params));
|
|
3147
|
+
paramIdx = params.length + 1;
|
|
2928
3148
|
const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
|
|
2929
|
-
const result = await q.query(`SELECT
|
|
3149
|
+
const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
|
|
2930
3150
|
FROM "${this.schemaName}".workflow_schedules${where}
|
|
2931
3151
|
ORDER BY schedule_name`, params);
|
|
2932
|
-
return result.rows.map((row) => (
|
|
2933
|
-
scheduleId: row.schedule_id,
|
|
2934
|
-
scheduleName: row.schedule_name,
|
|
2935
|
-
workflowName: row.workflow_name,
|
|
2936
|
-
workflowClassName: row.workflow_class_name,
|
|
2937
|
-
schedule: row.schedule,
|
|
2938
|
-
status: row.status,
|
|
2939
|
-
context: row.context,
|
|
2940
|
-
lastFiredAt: row.last_fired_at ?? null,
|
|
2941
|
-
automaticBackfill: !!row.automatic_backfill,
|
|
2942
|
-
cronTimezone: row.cron_timezone ?? null,
|
|
2943
|
-
queueName: row.queue_name ?? null,
|
|
2944
|
-
}));
|
|
3152
|
+
return result.rows.map((row) => mapWorkflowSchedule(row));
|
|
2945
3153
|
}
|
|
2946
3154
|
async getSchedule(name, client) {
|
|
2947
3155
|
const q = client ?? this.pool;
|
|
2948
|
-
const result = await q.query(`SELECT
|
|
3156
|
+
const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
|
|
2949
3157
|
FROM "${this.schemaName}".workflow_schedules
|
|
2950
3158
|
WHERE schedule_name = $1`, [name]);
|
|
2951
3159
|
if (result.rows.length === 0)
|
|
2952
3160
|
return null;
|
|
2953
|
-
|
|
2954
|
-
return {
|
|
2955
|
-
scheduleId: row.schedule_id,
|
|
2956
|
-
scheduleName: row.schedule_name,
|
|
2957
|
-
workflowName: row.workflow_name,
|
|
2958
|
-
workflowClassName: row.workflow_class_name,
|
|
2959
|
-
schedule: row.schedule,
|
|
2960
|
-
status: row.status,
|
|
2961
|
-
context: row.context,
|
|
2962
|
-
lastFiredAt: row.last_fired_at ?? null,
|
|
2963
|
-
automaticBackfill: !!row.automatic_backfill,
|
|
2964
|
-
cronTimezone: row.cron_timezone ?? null,
|
|
2965
|
-
queueName: row.queue_name ?? null,
|
|
2966
|
-
};
|
|
3161
|
+
return mapWorkflowSchedule(result.rows[0]);
|
|
2967
3162
|
}
|
|
2968
3163
|
async deleteSchedule(name, client) {
|
|
2969
3164
|
const q = client ?? this.pool;
|
|
@@ -3019,10 +3214,11 @@ class SystemDatabase {
|
|
|
3019
3214
|
try {
|
|
3020
3215
|
await client.query('BEGIN');
|
|
3021
3216
|
for (const sched of schedules) {
|
|
3217
|
+
const owner = await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
3022
3218
|
// Upsert on schedule_name; on conflict, preserve schedule_id and runtime state (status, last_fired_at) and update only the declared definition fields, so an unchanged re-apply is a no-op.
|
|
3023
3219
|
await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
3024
|
-
(schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name)
|
|
3025
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
|
3220
|
+
(schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name)
|
|
3221
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
|
3026
3222
|
ON CONFLICT (schedule_name) DO UPDATE SET
|
|
3027
3223
|
workflow_name = EXCLUDED.workflow_name,
|
|
3028
3224
|
workflow_class_name = EXCLUDED.workflow_class_name,
|
|
@@ -3030,7 +3226,9 @@ class SystemDatabase {
|
|
|
3030
3226
|
context = EXCLUDED.context,
|
|
3031
3227
|
automatic_backfill = EXCLUDED.automatic_backfill,
|
|
3032
3228
|
cron_timezone = EXCLUDED.cron_timezone,
|
|
3033
|
-
queue_name = EXCLUDED.queue_name
|
|
3229
|
+
queue_name = EXCLUDED.queue_name,
|
|
3230
|
+
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it took.
|
|
3231
|
+
application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, [
|
|
3034
3232
|
sched.scheduleId,
|
|
3035
3233
|
sched.scheduleName,
|
|
3036
3234
|
sched.workflowName,
|
|
@@ -3042,7 +3240,10 @@ class SystemDatabase {
|
|
|
3042
3240
|
sched.automaticBackfill,
|
|
3043
3241
|
sched.cronTimezone,
|
|
3044
3242
|
sched.queueName,
|
|
3243
|
+
owner ?? null,
|
|
3045
3244
|
]);
|
|
3245
|
+
// Read back, since the guard above is silent about why it declined to claim.
|
|
3246
|
+
await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
3046
3247
|
}
|
|
3047
3248
|
await client.query('COMMIT');
|
|
3048
3249
|
}
|
|
@@ -3055,56 +3256,109 @@ class SystemDatabase {
|
|
|
3055
3256
|
}
|
|
3056
3257
|
}
|
|
3057
3258
|
// ==================== Application Versions ====================
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3259
|
+
/**
|
|
3260
|
+
* Register this version, claiming the row if nobody owns it yet so a pinned version
|
|
3261
|
+
* does not stay unclaimed. A peer's name is a collision, which is why this throws.
|
|
3262
|
+
*/
|
|
3263
|
+
async createApplicationVersion(versionName, applicationName) {
|
|
3264
|
+
const owner = applicationName ?? this.appName;
|
|
3265
|
+
const client = await this.pool.connect();
|
|
3266
|
+
try {
|
|
3267
|
+
await client.query('BEGIN');
|
|
3268
|
+
// Claim a pre-upgrade row in place, so the version is not recreated or retimed.
|
|
3269
|
+
const claimed = await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3270
|
+
SET application_name = $1
|
|
3271
|
+
WHERE version_name = $2 AND application_name IS NULL`, [owner ?? null, versionName]);
|
|
3272
|
+
if ((claimed.rowCount ?? 0) === 0) {
|
|
3273
|
+
// Targetless DO NOTHING: names no arbiter, so it survives version_name's uniqueness being dropped while still absorbing a concurrent registrar.
|
|
3274
|
+
await client.query(`INSERT INTO "${this.schemaName}".application_versions (version_id, version_name, application_name)
|
|
3275
|
+
VALUES ($1, $2, $3)
|
|
3276
|
+
ON CONFLICT DO NOTHING`, [(0, crypto_1.randomUUID)(), versionName, owner ?? null]);
|
|
3277
|
+
}
|
|
3278
|
+
// Read back, since the writes above are silent about why they declined to claim.
|
|
3279
|
+
await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
|
|
3280
|
+
await client.query('COMMIT');
|
|
3281
|
+
}
|
|
3282
|
+
catch (e) {
|
|
3283
|
+
await client.query('ROLLBACK');
|
|
3284
|
+
throw e;
|
|
3285
|
+
}
|
|
3286
|
+
finally {
|
|
3287
|
+
client.release();
|
|
3288
|
+
}
|
|
3063
3289
|
}
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3290
|
+
/**
|
|
3291
|
+
* Promote a version to latest. Promoting a peer's is a collision, not a retiming;
|
|
3292
|
+
* promotion claims an unclaimed row, which would otherwise be every peer's latest.
|
|
3293
|
+
*/
|
|
3294
|
+
async updateApplicationVersionTimestamp(versionName, newTimestamp, applicationName) {
|
|
3295
|
+
const owner = applicationName ?? this.appName;
|
|
3296
|
+
const client = await this.pool.connect();
|
|
3297
|
+
try {
|
|
3298
|
+
await client.query('BEGIN');
|
|
3299
|
+
const resolved = await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
|
|
3300
|
+
// Scoped to the row this writer resolved to: once version_name is no longer globally unique, a bare name match would retime every peer's version.
|
|
3301
|
+
const scope = resolved === undefined ? 'application_name IS NULL' : '(application_name = $3 OR application_name IS NULL)';
|
|
3302
|
+
const params = [newTimestamp, versionName];
|
|
3303
|
+
if (resolved !== undefined)
|
|
3304
|
+
params.push(resolved);
|
|
3305
|
+
await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3306
|
+
SET version_timestamp = $1, application_name = ${resolved === undefined ? 'application_name' : '$3'}
|
|
3307
|
+
WHERE version_name = $2 AND ${scope}`, params);
|
|
3308
|
+
await client.query('COMMIT');
|
|
3309
|
+
}
|
|
3310
|
+
catch (e) {
|
|
3311
|
+
await client.query('ROLLBACK');
|
|
3312
|
+
throw e;
|
|
3313
|
+
}
|
|
3314
|
+
finally {
|
|
3315
|
+
client.release();
|
|
3316
|
+
}
|
|
3068
3317
|
}
|
|
3069
3318
|
async listApplicationVersions() {
|
|
3070
|
-
const
|
|
3319
|
+
const params = [];
|
|
3320
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
3321
|
+
const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
|
|
3071
3322
|
FROM "${this.schemaName}".application_versions
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
versionName: r.version_name,
|
|
3076
|
-
versionTimestamp: Number(r.version_timestamp),
|
|
3077
|
-
createdAt: Number(r.created_at),
|
|
3078
|
-
}));
|
|
3323
|
+
WHERE ${scope}
|
|
3324
|
+
ORDER BY version_timestamp DESC`, params);
|
|
3325
|
+
return rows.map(mapVersionInfo);
|
|
3079
3326
|
}
|
|
3080
|
-
|
|
3081
|
-
|
|
3327
|
+
/**
|
|
3328
|
+
* The latest version registered by an application. Defaults to this handle's, so a
|
|
3329
|
+
* caller acting for another one — firing its schedule — must name it.
|
|
3330
|
+
*/
|
|
3331
|
+
async getLatestApplicationVersion(applicationName) {
|
|
3332
|
+
const owner = applicationName ?? this.appName;
|
|
3333
|
+
const params = [];
|
|
3334
|
+
const scope = this.#appNameFilter('application_name', owner, params);
|
|
3335
|
+
const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
|
|
3082
3336
|
FROM "${this.schemaName}".application_versions
|
|
3337
|
+
WHERE ${scope}
|
|
3083
3338
|
ORDER BY version_timestamp DESC
|
|
3084
|
-
LIMIT 1
|
|
3339
|
+
LIMIT 1`, params);
|
|
3085
3340
|
if (rows.length === 0) {
|
|
3086
3341
|
throw new error_1.DBOSInitializationError('No application versions found');
|
|
3087
3342
|
}
|
|
3088
|
-
|
|
3089
|
-
return {
|
|
3090
|
-
versionId: r.version_id,
|
|
3091
|
-
versionName: r.version_name,
|
|
3092
|
-
versionTimestamp: Number(r.version_timestamp),
|
|
3093
|
-
createdAt: Number(r.created_at),
|
|
3094
|
-
};
|
|
3343
|
+
return mapVersionInfo(rows[0]);
|
|
3095
3344
|
}
|
|
3096
3345
|
// ==================== Queues ====================
|
|
3097
3346
|
async getQueue(name) {
|
|
3098
|
-
const { rows } = await this.pool.query(`SELECT
|
|
3099
|
-
priority_enabled, partition_queue, polling_interval_sec
|
|
3347
|
+
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3100
3348
|
FROM "${this.schemaName}".queues
|
|
3101
3349
|
WHERE name = $1`, [name]);
|
|
3102
3350
|
return rows.length === 0 ? null : queueRecordFromRow(rows[0]);
|
|
3103
3351
|
}
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3352
|
+
/**
|
|
3353
|
+
* List only queues owned by these applications, plus unclaimed ones.
|
|
3354
|
+
* By default, only list this application's queues.
|
|
3355
|
+
*/
|
|
3356
|
+
async listQueues(applicationName) {
|
|
3357
|
+
const params = [];
|
|
3358
|
+
const scope = this.#observabilityFilter('application_name', applicationName, params);
|
|
3359
|
+
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3360
|
+
FROM "${this.schemaName}".queues
|
|
3361
|
+
WHERE ${scope}`, params);
|
|
3108
3362
|
return rows.map(queueRecordFromRow);
|
|
3109
3363
|
}
|
|
3110
3364
|
async deleteQueue(name) {
|
|
@@ -3140,16 +3394,21 @@ class SystemDatabase {
|
|
|
3140
3394
|
priority_enabled = EXCLUDED.priority_enabled,
|
|
3141
3395
|
partition_queue = EXCLUDED.partition_queue,
|
|
3142
3396
|
polling_interval_sec = EXCLUDED.polling_interval_sec,
|
|
3143
|
-
updated_at = EXCLUDED.updated_at
|
|
3397
|
+
updated_at = EXCLUDED.updated_at,
|
|
3398
|
+
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it just took.
|
|
3399
|
+
application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
|
|
3144
3400
|
: `ON CONFLICT (name) DO NOTHING`;
|
|
3401
|
+
const owner = record.applicationName ?? this.appName;
|
|
3145
3402
|
const client = await this.pool.connect();
|
|
3146
3403
|
try {
|
|
3147
3404
|
await client.query('BEGIN');
|
|
3148
3405
|
const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE name = $1`, [record.name]);
|
|
3406
|
+
// A name collision is a conflict in every mode: the name is the queue's address.
|
|
3407
|
+
const resolvedOwner = await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
3149
3408
|
await client.query(`INSERT INTO "${this.schemaName}".queues
|
|
3150
3409
|
(name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
|
|
3151
|
-
priority_enabled, partition_queue, polling_interval_sec, updated_at)
|
|
3152
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
3410
|
+
priority_enabled, partition_queue, polling_interval_sec, updated_at, application_name)
|
|
3411
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
|
3153
3412
|
${onConflict}`, [
|
|
3154
3413
|
record.name,
|
|
3155
3414
|
record.concurrency,
|
|
@@ -3160,7 +3419,10 @@ class SystemDatabase {
|
|
|
3160
3419
|
record.partitionQueue,
|
|
3161
3420
|
record.pollingIntervalSec,
|
|
3162
3421
|
now,
|
|
3422
|
+
resolvedOwner ?? null,
|
|
3163
3423
|
]);
|
|
3424
|
+
// Read back, since the guard above is silent about why it declined to claim.
|
|
3425
|
+
await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
3164
3426
|
await client.query('COMMIT');
|
|
3165
3427
|
return existed.rowCount === 0;
|
|
3166
3428
|
}
|
|
@@ -3172,6 +3434,123 @@ class SystemDatabase {
|
|
|
3172
3434
|
client.release();
|
|
3173
3435
|
}
|
|
3174
3436
|
}
|
|
3437
|
+
// ==================== Application Rename ====================
|
|
3438
|
+
/**
|
|
3439
|
+
* Rows a rename moves: an application's own, unclaimed ones, or both. Unlike
|
|
3440
|
+
* the claiming scope, unclaimed rows are not implied; they move only when asked.
|
|
3441
|
+
*/
|
|
3442
|
+
#renameSource(oldName, adoptUnclaimedRows, params) {
|
|
3443
|
+
const clauses = [];
|
|
3444
|
+
if (oldName !== undefined) {
|
|
3445
|
+
params.push(oldName);
|
|
3446
|
+
clauses.push(`application_name = $${params.length}`);
|
|
3447
|
+
}
|
|
3448
|
+
if (adoptUnclaimedRows) {
|
|
3449
|
+
clauses.push('application_name IS NULL');
|
|
3450
|
+
}
|
|
3451
|
+
// Callers validate that at least one source is named.
|
|
3452
|
+
return `(${clauses.join(' OR ')})`;
|
|
3453
|
+
}
|
|
3454
|
+
/**
|
|
3455
|
+
* Re-own a table's rows in half-open key ranges, so a long history neither moves in
|
|
3456
|
+
* one transaction nor rescans what it already moved; a re-run resumes.
|
|
3457
|
+
*/
|
|
3458
|
+
async #renameRowsInBatches(table, keyColumn, oldName, newName, batchSize, adoptUnclaimedRows) {
|
|
3459
|
+
if (batchSize === undefined) {
|
|
3460
|
+
const params = [newName];
|
|
3461
|
+
const predicate = this.#renameSource(oldName, adoptUnclaimedRows, params);
|
|
3462
|
+
const res = await this.pool.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${predicate}`, params);
|
|
3463
|
+
return res.rowCount ?? 0;
|
|
3464
|
+
}
|
|
3465
|
+
let total = 0;
|
|
3466
|
+
// Ranges, not LIMIT: a LIMIT repages every row already moved, and an IN list of keys plans as a whole-table hash join.
|
|
3467
|
+
let watermark = undefined;
|
|
3468
|
+
for (;;) {
|
|
3469
|
+
const boundParams = [];
|
|
3470
|
+
const predicate = this.#renameSource(oldName, adoptUnclaimedRows, boundParams);
|
|
3471
|
+
let scope = predicate;
|
|
3472
|
+
if (watermark !== undefined) {
|
|
3473
|
+
boundParams.push(watermark);
|
|
3474
|
+
scope = `${predicate} AND ${keyColumn} > $${boundParams.length}`;
|
|
3475
|
+
}
|
|
3476
|
+
// The batchSize-th matching key bounds this range; distinct, so a key's rows are never split across batches.
|
|
3477
|
+
const upperResult = await this.pool.query(`SELECT DISTINCT ${keyColumn} FROM "${this.schemaName}".${table}
|
|
3478
|
+
WHERE ${scope} ORDER BY ${keyColumn} LIMIT 1 OFFSET ${batchSize - 1}`, boundParams);
|
|
3479
|
+
const upper = upperResult.rows[0]?.[keyColumn];
|
|
3480
|
+
const updateParams = [newName];
|
|
3481
|
+
const updatePredicate = this.#renameSource(oldName, adoptUnclaimedRows, updateParams);
|
|
3482
|
+
let batch = updatePredicate;
|
|
3483
|
+
if (upper !== undefined) {
|
|
3484
|
+
if (watermark !== undefined) {
|
|
3485
|
+
updateParams.push(watermark);
|
|
3486
|
+
batch = `${batch} AND ${keyColumn} > $${updateParams.length}`;
|
|
3487
|
+
}
|
|
3488
|
+
updateParams.push(upper);
|
|
3489
|
+
batch = `${batch} AND ${keyColumn} <= $${updateParams.length}`;
|
|
3490
|
+
}
|
|
3491
|
+
// The final batch drops the watermark, so rows that appeared below it still move.
|
|
3492
|
+
const res = await this.pool.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${batch}`, updateParams);
|
|
3493
|
+
total += res.rowCount ?? 0;
|
|
3494
|
+
// Fewer than a full batch remained, so that update took the rest.
|
|
3495
|
+
if (upper === undefined)
|
|
3496
|
+
return total;
|
|
3497
|
+
watermark = upper;
|
|
3498
|
+
}
|
|
3499
|
+
}
|
|
3500
|
+
/**
|
|
3501
|
+
* Give `newName` ownership of rows `oldName` holds, of unclaimed rows, or of both.
|
|
3502
|
+
* The renamed application must be stopped, or its dequeues race this.
|
|
3503
|
+
*/
|
|
3504
|
+
async renameApplication(oldName, newName, options = {}) {
|
|
3505
|
+
const adoptUnclaimedRows = options.adoptUnclaimedRows ?? false;
|
|
3506
|
+
const batchSize = options.batchSize === null ? undefined : (options.batchSize ?? exports.DEFAULT_RENAME_BATCH_SIZE);
|
|
3507
|
+
if (oldName !== undefined && oldName === '') {
|
|
3508
|
+
throw new error_1.DBOSError("The application's previous name cannot be empty.");
|
|
3509
|
+
}
|
|
3510
|
+
if (oldName === undefined && !adoptUnclaimedRows) {
|
|
3511
|
+
throw new error_1.DBOSError('Nothing to re-own: name the application to rename, adopt unclaimed rows, or both.');
|
|
3512
|
+
}
|
|
3513
|
+
if (oldName === newName) {
|
|
3514
|
+
throw new error_1.DBOSError(`Application '${newName}' already holds that name; nothing to rename.`);
|
|
3515
|
+
}
|
|
3516
|
+
// A NaN survives a bare `< 1` test and would only fail once it reached SQL, leaving the rename half-applied.
|
|
3517
|
+
if (batchSize !== undefined && (!Number.isInteger(batchSize) || batchSize < 1)) {
|
|
3518
|
+
throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
|
|
3519
|
+
}
|
|
3520
|
+
// Never a merge: queue, schedule, and version names are globally unique whatever their owner, so this cannot collide.
|
|
3521
|
+
const client = await this.pool.connect();
|
|
3522
|
+
let queues, schedules, versions, inFlight;
|
|
3523
|
+
try {
|
|
3524
|
+
await client.query('BEGIN');
|
|
3525
|
+
const move = async (table, statuses) => {
|
|
3526
|
+
const params = [newName];
|
|
3527
|
+
let where = this.#renameSource(oldName, adoptUnclaimedRows, params);
|
|
3528
|
+
if (statuses !== undefined) {
|
|
3529
|
+
params.push(statuses);
|
|
3530
|
+
where = `${where} AND status = ANY($${params.length})`;
|
|
3531
|
+
}
|
|
3532
|
+
const res = await client.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${where}`, params);
|
|
3533
|
+
return res.rowCount ?? 0;
|
|
3534
|
+
};
|
|
3535
|
+
// A half-owned application dequeues work whose version row it can no longer see, so these move together.
|
|
3536
|
+
queues = await move('queues');
|
|
3537
|
+
schedules = await move('workflow_schedules');
|
|
3538
|
+
versions = await move('application_versions');
|
|
3539
|
+
inFlight = await move('workflow_status', [workflow_1.StatusString.PENDING, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED]);
|
|
3540
|
+
await client.query('COMMIT');
|
|
3541
|
+
}
|
|
3542
|
+
catch (e) {
|
|
3543
|
+
await client.query('ROLLBACK');
|
|
3544
|
+
throw e;
|
|
3545
|
+
}
|
|
3546
|
+
finally {
|
|
3547
|
+
client.release();
|
|
3548
|
+
}
|
|
3549
|
+
// Only terminal rows are left to match, and they scope observability and GC alone, so they may lag behind the commit above.
|
|
3550
|
+
const terminal = await this.#renameRowsInBatches('workflow_status', 'workflow_uuid', oldName, newName, batchSize, adoptUnclaimedRows);
|
|
3551
|
+
const steps = await this.#renameRowsInBatches('operation_outputs', 'workflow_uuid', oldName, newName, batchSize, adoptUnclaimedRows);
|
|
3552
|
+
return { queues, schedules, versions, workflows: inFlight + terminal, steps };
|
|
3553
|
+
}
|
|
3175
3554
|
// ==================== Internal ====================
|
|
3176
3555
|
async insertWorkflowStatus(client, initStatus, ownerXid, incrementAttempts = false) {
|
|
3177
3556
|
try {
|
|
@@ -3206,8 +3585,9 @@ class SystemDatabase {
|
|
|
3206
3585
|
attributes,
|
|
3207
3586
|
schedule_name,
|
|
3208
3587
|
debounce_deadline_epoch_ms,
|
|
3209
|
-
is_debounced
|
|
3210
|
-
|
|
3588
|
+
is_debounced,
|
|
3589
|
+
application_name
|
|
3590
|
+
) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $26, $27, $28, $29, $30, $31, $32, $33)
|
|
3211
3591
|
ON CONFLICT (workflow_uuid)
|
|
3212
3592
|
DO UPDATE SET
|
|
3213
3593
|
recovery_attempts = CASE
|
|
@@ -3255,6 +3635,8 @@ class SystemDatabase {
|
|
|
3255
3635
|
initStatus.scheduleName ?? null,
|
|
3256
3636
|
initStatus.debounceDeadlineEpochMS ?? null,
|
|
3257
3637
|
initStatus.isDebounced ?? false,
|
|
3638
|
+
// Absent from the conflict update: a re-enqueue must not re-own a claimed row.
|
|
3639
|
+
initStatus.applicationName ?? null,
|
|
3258
3640
|
]);
|
|
3259
3641
|
if (rows.length === 0) {
|
|
3260
3642
|
throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
|
|
@@ -3342,8 +3724,8 @@ class SystemDatabase {
|
|
|
3342
3724
|
async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
3343
3725
|
try {
|
|
3344
3726
|
const out = await client.query(`INSERT INTO ${this.schemaName}.operation_outputs
|
|
3345
|
-
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization)
|
|
3346
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
3727
|
+
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization, application_name)
|
|
3728
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
|
3347
3729
|
ON CONFLICT (workflow_uuid, function_id) DO UPDATE
|
|
3348
3730
|
SET completed_at_epoch_ms = operation_outputs.completed_at_epoch_ms
|
|
3349
3731
|
RETURNING completed_at_epoch_ms;`, [
|
|
@@ -3356,6 +3738,8 @@ class SystemDatabase {
|
|
|
3356
3738
|
startTimeEpochMs,
|
|
3357
3739
|
endTimeEpochMs,
|
|
3358
3740
|
options.serialization ?? null,
|
|
3741
|
+
// Mirrors the parent: only the running application records its steps.
|
|
3742
|
+
this.appName ?? null,
|
|
3359
3743
|
]);
|
|
3360
3744
|
if (checkConflict &&
|
|
3361
3745
|
(out?.rowCount ?? 0) > 0 &&
|
|
@@ -3457,10 +3841,35 @@ class SystemDatabase {
|
|
|
3457
3841
|
* workflow listener by resolving its promise.
|
|
3458
3842
|
*/
|
|
3459
3843
|
reconnectTimeout = null;
|
|
3844
|
+
#notificationsStopped = false;
|
|
3845
|
+
// Disown the client and release it once; a late socket error must not re-enter this path and release again.
|
|
3846
|
+
#retireNotificationsClient(client) {
|
|
3847
|
+
if (this.notificationsClient === client) {
|
|
3848
|
+
this.notificationsClient = null;
|
|
3849
|
+
}
|
|
3850
|
+
client.removeAllListeners();
|
|
3851
|
+
// Errors can still arrive while release() tears the connection down; a bare emit would crash the process.
|
|
3852
|
+
client.on('error', () => { });
|
|
3853
|
+
try {
|
|
3854
|
+
client.release(true);
|
|
3855
|
+
}
|
|
3856
|
+
catch (e) {
|
|
3857
|
+
this.logger.warn(`Error releasing notifications client: ${String(e)}`);
|
|
3858
|
+
}
|
|
3859
|
+
}
|
|
3860
|
+
// Shutdown can begin during any await in the setup below; releasing the client instead of carrying on
|
|
3861
|
+
// keeps pool.end() from waiting on a connection that will never be published.
|
|
3862
|
+
#abandonIfStopped(client) {
|
|
3863
|
+
if (!this.#notificationsStopped) {
|
|
3864
|
+
return false;
|
|
3865
|
+
}
|
|
3866
|
+
this.#retireNotificationsClient(client);
|
|
3867
|
+
return true;
|
|
3868
|
+
}
|
|
3460
3869
|
async #listenForNotifications() {
|
|
3461
3870
|
const connect = async () => {
|
|
3462
3871
|
const reconnect = () => {
|
|
3463
|
-
if (this.reconnectTimeout) {
|
|
3872
|
+
if (this.reconnectTimeout || this.#notificationsStopped) {
|
|
3464
3873
|
return;
|
|
3465
3874
|
}
|
|
3466
3875
|
this.reconnectTimeout = setTimeout(async () => {
|
|
@@ -3468,12 +3877,24 @@ class SystemDatabase {
|
|
|
3468
3877
|
await connect();
|
|
3469
3878
|
}, 1000);
|
|
3470
3879
|
};
|
|
3471
|
-
let
|
|
3880
|
+
let acquired = null;
|
|
3472
3881
|
try {
|
|
3473
|
-
client = await this.pool.connect();
|
|
3882
|
+
const client = await this.pool.connect();
|
|
3883
|
+
acquired = client;
|
|
3884
|
+
if (this.#abandonIfStopped(client))
|
|
3885
|
+
return;
|
|
3886
|
+
// Catch errors during setup
|
|
3887
|
+
const setup = { error: null };
|
|
3888
|
+
const onSetupError = (err) => {
|
|
3889
|
+
setup.error = err;
|
|
3890
|
+
};
|
|
3891
|
+
client.on('error', onSetupError);
|
|
3474
3892
|
await client.query(`LISTEN ${exports.DBOS_NOTIFICATIONS_CHANNEL};`);
|
|
3475
3893
|
await client.query(`LISTEN ${exports.DBOS_WORKFLOW_EVENTS_CHANNEL};`);
|
|
3476
3894
|
await client.query(`LISTEN ${exports.DBOS_STREAMS_CHANNEL};`);
|
|
3895
|
+
// The self-test's NOTIFY needs a second client, which can queue forever on an ending pool.
|
|
3896
|
+
if (this.#abandonIfStopped(client))
|
|
3897
|
+
return;
|
|
3477
3898
|
// Self-test: verify LISTEN actually works by sending a NOTIFY and checking it arrives.
|
|
3478
3899
|
// If a transaction-mode pooler (e.g. PgBouncer pool_mode=transaction) is in the path,
|
|
3479
3900
|
// LISTEN succeeds but the subscription is silently lost when the backend is released.
|
|
@@ -3485,10 +3906,16 @@ class SystemDatabase {
|
|
|
3485
3906
|
};
|
|
3486
3907
|
client.on('notification', onSelfTest);
|
|
3487
3908
|
await this.pool.query("NOTIFY dbos_notifications_channel, 'dbos_listen_selftest'");
|
|
3488
|
-
for (let i = 0; i < 30 && !selfTestReceived; i++) {
|
|
3909
|
+
for (let i = 0; i < 30 && !selfTestReceived && !setup.error && !this.#notificationsStopped; i++) {
|
|
3489
3910
|
await new Promise((r) => setTimeout(r, 100));
|
|
3490
3911
|
}
|
|
3491
3912
|
client.removeListener('notification', onSelfTest);
|
|
3913
|
+
// Both checked before the warning below, so an abandoned self-test is not reported as a pooler problem.
|
|
3914
|
+
if (this.#abandonIfStopped(client))
|
|
3915
|
+
return;
|
|
3916
|
+
if (setup.error) {
|
|
3917
|
+
throw setup.error;
|
|
3918
|
+
}
|
|
3492
3919
|
if (!selfTestReceived) {
|
|
3493
3920
|
this.logger.warn('LISTEN/NOTIFY self-test failed: notification was not received within 3 seconds. ' +
|
|
3494
3921
|
'This typically means the connection is going through a transaction-mode pooler ' +
|
|
@@ -3508,28 +3935,19 @@ class SystemDatabase {
|
|
|
3508
3935
|
this.streamsMap.callCallbacks(msg.payload);
|
|
3509
3936
|
}
|
|
3510
3937
|
};
|
|
3938
|
+
client.removeListener('error', onSetupError);
|
|
3511
3939
|
client.on('notification', handler);
|
|
3512
3940
|
client.on('error', (err) => {
|
|
3513
3941
|
this.logger.warn(`Error in notifications client: ${err}`);
|
|
3514
|
-
|
|
3515
|
-
this.notificationsClient = null;
|
|
3516
|
-
}
|
|
3517
|
-
if (client) {
|
|
3518
|
-
client.removeAllListeners();
|
|
3519
|
-
// Errors can still arrive while release() tears the connection down; a bare emit would crash the process.
|
|
3520
|
-
client.on('error', () => { });
|
|
3521
|
-
client.release(true);
|
|
3522
|
-
}
|
|
3942
|
+
this.#retireNotificationsClient(client);
|
|
3523
3943
|
reconnect();
|
|
3524
3944
|
});
|
|
3525
3945
|
this.notificationsClient = client;
|
|
3526
3946
|
}
|
|
3527
3947
|
catch (error) {
|
|
3528
3948
|
this.logger.warn(`Error in notifications listener: ${String(error)}`);
|
|
3529
|
-
if (
|
|
3530
|
-
|
|
3531
|
-
client.on('error', () => { });
|
|
3532
|
-
client.release(true);
|
|
3949
|
+
if (acquired) {
|
|
3950
|
+
this.#retireNotificationsClient(acquired);
|
|
3533
3951
|
}
|
|
3534
3952
|
reconnect();
|
|
3535
3953
|
}
|
|
@@ -3697,7 +4115,13 @@ __decorate([
|
|
|
3697
4115
|
__decorate([
|
|
3698
4116
|
dbRetry(),
|
|
3699
4117
|
__metadata("design:type", Function),
|
|
3700
|
-
__metadata("design:paramtypes", [
|
|
4118
|
+
__metadata("design:paramtypes", [Number]),
|
|
4119
|
+
__metadata("design:returntype", Promise)
|
|
4120
|
+
], SystemDatabase.prototype, "listTimedOutWorkflowIds", null);
|
|
4121
|
+
__decorate([
|
|
4122
|
+
dbRetry(),
|
|
4123
|
+
__metadata("design:type", Function),
|
|
4124
|
+
__metadata("design:paramtypes", [String, String, Array]),
|
|
3701
4125
|
__metadata("design:returntype", Promise)
|
|
3702
4126
|
], SystemDatabase.prototype, "getMetrics", null);
|
|
3703
4127
|
//# sourceMappingURL=system_database.js.map
|