@dbos-inc/dbos-sdk 4.25.14 → 4.26.4-preview
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/schemas/system_db_schema.d.ts +5 -0
- package/dist/schemas/system_db_schema.d.ts.map +1 -1
- package/dist/schemas/system_db_schema.js.map +1 -1
- package/dist/src/adminserver.js +1 -1
- package/dist/src/adminserver.js.map +1 -1
- package/dist/src/cli/cli.d.ts.map +1 -1
- package/dist/src/cli/cli.js +64 -0
- package/dist/src/cli/cli.js.map +1 -1
- package/dist/src/cli/schema.d.ts.map +1 -1
- package/dist/src/cli/schema.js +26 -16
- package/dist/src/cli/schema.js.map +1 -1
- package/dist/src/client.d.ts +62 -4
- package/dist/src/client.d.ts.map +1 -1
- package/dist/src/client.js +58 -8
- package/dist/src/client.js.map +1 -1
- package/dist/src/conductor/conductor.d.ts.map +1 -1
- package/dist/src/conductor/conductor.js +13 -2
- package/dist/src/conductor/conductor.js.map +1 -1
- package/dist/src/conductor/protocol.d.ts +16 -2
- package/dist/src/conductor/protocol.d.ts.map +1 -1
- package/dist/src/conductor/protocol.js +10 -2
- package/dist/src/conductor/protocol.js.map +1 -1
- package/dist/src/context.d.ts +1 -0
- package/dist/src/context.d.ts.map +1 -1
- package/dist/src/context.js +2 -1
- package/dist/src/context.js.map +1 -1
- package/dist/src/dbos-executor.d.ts +1 -0
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js +128 -64
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts +41 -7
- package/dist/src/dbos.d.ts.map +1 -1
- package/dist/src/dbos.js +61 -4
- package/dist/src/dbos.js.map +1 -1
- package/dist/src/debouncer.d.ts +4 -0
- package/dist/src/debouncer.d.ts.map +1 -1
- package/dist/src/debouncer.js +32 -8
- package/dist/src/debouncer.js.map +1 -1
- package/dist/src/enqueue_options.d.ts +59 -0
- package/dist/src/enqueue_options.d.ts.map +1 -0
- package/dist/src/enqueue_options.js +61 -0
- package/dist/src/enqueue_options.js.map +1 -0
- package/dist/src/enqueue_workflow.d.ts +12 -0
- package/dist/src/enqueue_workflow.d.ts.map +1 -0
- package/dist/src/enqueue_workflow.js +91 -0
- package/dist/src/enqueue_workflow.js.map +1 -0
- package/dist/src/index.d.ts +2 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/scheduler/scheduler.d.ts +1 -0
- package/dist/src/scheduler/scheduler.d.ts.map +1 -1
- package/dist/src/scheduler/scheduler.js +13 -5
- package/dist/src/scheduler/scheduler.js.map +1 -1
- package/dist/src/sysdb_migrations/internal/migrations.d.ts +2 -1
- package/dist/src/sysdb_migrations/internal/migrations.d.ts.map +1 -1
- package/dist/src/sysdb_migrations/internal/migrations.js +186 -2
- package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
- package/dist/src/sysdb_migrations/migration_runner.d.ts +5 -0
- package/dist/src/sysdb_migrations/migration_runner.d.ts.map +1 -1
- package/dist/src/sysdb_migrations/migration_runner.js +48 -10
- package/dist/src/sysdb_migrations/migration_runner.js.map +1 -1
- package/dist/src/system_database.d.ts +68 -13
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +602 -200
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/utils.d.ts +1 -0
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +1 -0
- package/dist/src/utils.js.map +1 -1
- package/dist/src/wfqueue.d.ts +2 -0
- package/dist/src/wfqueue.d.ts.map +1 -1
- package/dist/src/wfqueue.js +5 -1
- package/dist/src/wfqueue.js.map +1 -1
- package/dist/src/workflow.d.ts +2 -0
- package/dist/src/workflow.d.ts.map +1 -1
- package/dist/src/workflow.js.map +1 -1
- package/dist/src/workflow_management.d.ts.map +1 -1
- package/dist/src/workflow_management.js +4 -9
- package/dist/src/workflow_management.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
|
@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
9
9
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.SystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
12
|
+
exports.SystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
13
13
|
const dbos_executor_1 = require("./dbos-executor");
|
|
14
14
|
const pg_1 = require("pg");
|
|
15
15
|
const error_1 = require("./error");
|
|
@@ -38,6 +38,8 @@ exports.DBOS_WORKFLOW_EVENTS_CHANNEL = 'dbos_workflow_events_channel';
|
|
|
38
38
|
exports.DBOS_STREAMS_CHANNEL = 'dbos_streams_channel';
|
|
39
39
|
// Interval for coalescing LISTEN/NOTIFY notifications off the write path; caps the rate of notifying commits regardless of write throughput.
|
|
40
40
|
exports.DEFAULT_NOTIFICATION_COALESCE_MS = 10;
|
|
41
|
+
// Workflows re-owned per transaction by a rename. Matches the GC default.
|
|
42
|
+
exports.DEFAULT_RENAME_BATCH_SIZE = 10_000;
|
|
41
43
|
const QUEUE_COLUMN_BY_FIELD = {
|
|
42
44
|
concurrency: 'concurrency',
|
|
43
45
|
workerConcurrency: 'worker_concurrency',
|
|
@@ -47,6 +49,7 @@ const QUEUE_COLUMN_BY_FIELD = {
|
|
|
47
49
|
partitionQueue: 'partition_queue',
|
|
48
50
|
pollingIntervalSec: 'polling_interval_sec',
|
|
49
51
|
};
|
|
52
|
+
const QUEUE_COLUMNS = 'name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec, priority_enabled, partition_queue, polling_interval_sec, application_name';
|
|
50
53
|
function queueRecordFromRow(row) {
|
|
51
54
|
return {
|
|
52
55
|
name: row.name,
|
|
@@ -57,6 +60,7 @@ function queueRecordFromRow(row) {
|
|
|
57
60
|
priorityEnabled: row.priority_enabled,
|
|
58
61
|
partitionQueue: row.partition_queue,
|
|
59
62
|
pollingIntervalSec: row.polling_interval_sec,
|
|
63
|
+
applicationName: row.application_name ?? undefined,
|
|
60
64
|
};
|
|
61
65
|
}
|
|
62
66
|
/** The statements granting permissions on all entities in the system schema to a role. */
|
|
@@ -172,6 +176,32 @@ class NotificationMap {
|
|
|
172
176
|
}
|
|
173
177
|
}
|
|
174
178
|
}
|
|
179
|
+
function mapVersionInfo(row) {
|
|
180
|
+
return {
|
|
181
|
+
versionId: row.version_id,
|
|
182
|
+
versionName: row.version_name,
|
|
183
|
+
versionTimestamp: Number(row.version_timestamp),
|
|
184
|
+
createdAt: Number(row.created_at),
|
|
185
|
+
applicationName: row.application_name ?? undefined,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
const SCHEDULE_COLUMNS = 'schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name';
|
|
189
|
+
function mapWorkflowSchedule(row) {
|
|
190
|
+
return {
|
|
191
|
+
scheduleId: row.schedule_id,
|
|
192
|
+
scheduleName: row.schedule_name,
|
|
193
|
+
workflowName: row.workflow_name,
|
|
194
|
+
workflowClassName: row.workflow_class_name,
|
|
195
|
+
schedule: row.schedule,
|
|
196
|
+
status: row.status,
|
|
197
|
+
context: row.context,
|
|
198
|
+
lastFiredAt: row.last_fired_at ?? null,
|
|
199
|
+
automaticBackfill: !!row.automatic_backfill,
|
|
200
|
+
cronTimezone: row.cron_timezone ?? null,
|
|
201
|
+
queueName: row.queue_name ?? null,
|
|
202
|
+
applicationName: row.application_name ?? undefined,
|
|
203
|
+
};
|
|
204
|
+
}
|
|
175
205
|
function mapWorkflowStatus(row) {
|
|
176
206
|
return {
|
|
177
207
|
workflowUUID: row.workflow_uuid,
|
|
@@ -209,6 +239,7 @@ function mapWorkflowStatus(row) {
|
|
|
209
239
|
scheduleName: row.schedule_name ?? undefined,
|
|
210
240
|
debounceDeadlineEpochMS: row.debounce_deadline_epoch_ms ? Number(row.debounce_deadline_epoch_ms) : undefined,
|
|
211
241
|
isDebounced: row.is_debounced ?? false,
|
|
242
|
+
applicationName: row.application_name ?? undefined,
|
|
212
243
|
};
|
|
213
244
|
}
|
|
214
245
|
// SQLSTATE classes/codes that are generally safe to retry
|
|
@@ -364,6 +395,7 @@ class SystemDatabase {
|
|
|
364
395
|
systemDatabaseUrl;
|
|
365
396
|
logger;
|
|
366
397
|
serializer;
|
|
398
|
+
appName;
|
|
367
399
|
// ==================== Lifecycle ====================
|
|
368
400
|
pool;
|
|
369
401
|
schemaName;
|
|
@@ -411,10 +443,13 @@ class SystemDatabase {
|
|
|
411
443
|
runningWorkflowMap = new Map(); // Map from workflowID to workflow promise, queue name and partition key
|
|
412
444
|
// Per-partition-key created_at cursors: keep per-key queue order monotonic across batches
|
|
413
445
|
#batchCreatedAtCursors = new Map();
|
|
414
|
-
constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS
|
|
446
|
+
constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS,
|
|
447
|
+
// The application this handle acts for; undefined writes unclaimed rows.
|
|
448
|
+
appName) {
|
|
415
449
|
this.systemDatabaseUrl = systemDatabaseUrl;
|
|
416
450
|
this.logger = logger;
|
|
417
451
|
this.serializer = serializer;
|
|
452
|
+
this.appName = appName;
|
|
418
453
|
this.schemaName = schemaName;
|
|
419
454
|
this.shouldUseDBNotifications = useListenNotify;
|
|
420
455
|
this.notificationCoalesceMs = notificationCoalesceMs;
|
|
@@ -448,6 +483,60 @@ class SystemDatabase {
|
|
|
448
483
|
getSerializer() {
|
|
449
484
|
return this.serializer;
|
|
450
485
|
}
|
|
486
|
+
// ==================== Application Ownership ====================
|
|
487
|
+
/**
|
|
488
|
+
* A predicate matching rows owned by these applications plus unclaimed ones, which belong to
|
|
489
|
+
* every application; unset or empty matches everything. Appends its bind parameter to `params`.
|
|
490
|
+
*/
|
|
491
|
+
#appNameFilter(column, value, params) {
|
|
492
|
+
// An empty name is no name: it is not a value any application could be configured with.
|
|
493
|
+
const names = !value ? [] : Array.isArray(value) ? value : [value];
|
|
494
|
+
if (names.length === 0)
|
|
495
|
+
return 'TRUE';
|
|
496
|
+
params.push(names);
|
|
497
|
+
return `(${column} = ANY($${params.length}) OR ${column} IS NULL)`;
|
|
498
|
+
}
|
|
499
|
+
/**
|
|
500
|
+
* The filter above defaulted to this handle's own application: an unset filter scopes to what
|
|
501
|
+
* this application owns, not to every application's rows. A handle with no application of its
|
|
502
|
+
* own still matches every one.
|
|
503
|
+
*/
|
|
504
|
+
#observabilityFilter(column, value, params) {
|
|
505
|
+
return this.#appNameFilter(column, value ?? this.appName, params);
|
|
506
|
+
}
|
|
507
|
+
/**
|
|
508
|
+
* The version name a dequeue treats as latest: this application's own plus unclaimed
|
|
509
|
+
* ones, so a named peer's deploy does not demote this one.
|
|
510
|
+
*/
|
|
511
|
+
async #latestApplicationVersionName(client) {
|
|
512
|
+
const params = [];
|
|
513
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
514
|
+
const { rows } = await client.query(`SELECT version_name
|
|
515
|
+
FROM "${this.schemaName}".application_versions
|
|
516
|
+
WHERE ${scope}
|
|
517
|
+
ORDER BY version_timestamp DESC LIMIT 1`, params);
|
|
518
|
+
return rows[0]?.version_name;
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* The owner to persist when writing a row that may already exist. A nameless writer
|
|
522
|
+
* leaves the owner intact; a named one collides only with a different name.
|
|
523
|
+
*/
|
|
524
|
+
async #resolveRowOwner(client, table, keyColumn, name, owner, kind) {
|
|
525
|
+
const { rows } = await client.query(`SELECT application_name FROM "${this.schemaName}".${table} WHERE ${keyColumn} = $1`, [name]);
|
|
526
|
+
const current = rows[0]?.application_name ?? null;
|
|
527
|
+
if (current === null)
|
|
528
|
+
return owner;
|
|
529
|
+
if (owner === undefined || current === owner)
|
|
530
|
+
return current;
|
|
531
|
+
// A version name is computed or pinned, so "pick another" is config advice, not a rename.
|
|
532
|
+
const takeANewName = kind === 'Application version'
|
|
533
|
+
? `set a distinct applicationVersion for '${owner}'`
|
|
534
|
+
: `give '${owner}' a different ${kind.toLowerCase()} name`;
|
|
535
|
+
throw new error_1.DBOSError(`${kind} '${name}' is already registered by application '${current}' in this system database. ` +
|
|
536
|
+
`${kind} names must be unique across applications sharing a system database. ` +
|
|
537
|
+
`Either ${takeANewName}, or, if '${current}' was renamed to '${owner}', ` +
|
|
538
|
+
`re-own its rows first with dbos rename-application`);
|
|
539
|
+
}
|
|
451
540
|
async init() {
|
|
452
541
|
await ensureSystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
|
|
453
542
|
if (this.shouldUseDBNotifications) {
|
|
@@ -658,6 +747,7 @@ class SystemDatabase {
|
|
|
658
747
|
'delay_until_epoch_ms',
|
|
659
748
|
'attributes',
|
|
660
749
|
'schedule_name',
|
|
750
|
+
'application_name',
|
|
661
751
|
];
|
|
662
752
|
const client = await this.pool.connect();
|
|
663
753
|
try {
|
|
@@ -675,7 +765,7 @@ class SystemDatabase {
|
|
|
675
765
|
tuples.push(`(${columns.map(() => `$${paramIdx++}`).join(', ')})`);
|
|
676
766
|
params.push(status.workflowUUID, status.status, status.workflowName,
|
|
677
767
|
// For cross-language compatibility, these MUST be NULL in the database when not set
|
|
678
|
-
status.workflowClassName === '' ? null : status.workflowClassName, status.workflowConfigName === '' ? null : status.workflowConfigName, status.queueName ?? null, status.authenticatedUser, status.assumedRole, JSON.stringify(status.authenticatedRoles), JSON.stringify(status.request), status.executorId, status.applicationVersion ?? null, status.applicationID, createdAt, 0, createdAt, status.timeoutMS ?? null, status.deadlineEpochMS ?? null, status.input, null, status.priority, status.queuePartitionKey ?? null, status.parentWorkflowID ?? null, status.serialization, null, status.delayUntilEpochMS ?? null, status.attributes ? JSON.stringify(status.attributes) : null, status.scheduleName ?? null);
|
|
768
|
+
status.workflowClassName === '' ? null : status.workflowClassName, status.workflowConfigName === '' ? null : status.workflowConfigName, status.queueName ?? null, status.authenticatedUser, status.assumedRole, JSON.stringify(status.authenticatedRoles), JSON.stringify(status.request), status.executorId, status.applicationVersion ?? null, status.applicationID, createdAt, 0, createdAt, status.timeoutMS ?? null, status.deadlineEpochMS ?? null, status.input, null, status.priority, status.queuePartitionKey ?? null, status.parentWorkflowID ?? null, status.serialization, null, status.delayUntilEpochMS ?? null, status.attributes ? JSON.stringify(status.attributes) : null, status.scheduleName ?? null, status.applicationName ?? null);
|
|
679
769
|
}
|
|
680
770
|
const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
|
|
681
771
|
VALUES ${tuples.join(', ')}
|
|
@@ -699,7 +789,7 @@ class SystemDatabase {
|
|
|
699
789
|
async recordWorkflowOutput(workflowID, status) {
|
|
700
790
|
const client = await this.pool.connect();
|
|
701
791
|
try {
|
|
702
|
-
await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output });
|
|
792
|
+
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output });
|
|
703
793
|
}
|
|
704
794
|
finally {
|
|
705
795
|
client.release();
|
|
@@ -708,47 +798,54 @@ class SystemDatabase {
|
|
|
708
798
|
async recordWorkflowError(workflowID, status) {
|
|
709
799
|
const client = await this.pool.connect();
|
|
710
800
|
try {
|
|
711
|
-
await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error });
|
|
801
|
+
return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error });
|
|
712
802
|
}
|
|
713
803
|
finally {
|
|
714
804
|
client.release();
|
|
715
805
|
}
|
|
716
806
|
}
|
|
717
|
-
// Record a workflow's terminal outcome (SUCCESS or ERROR),
|
|
718
|
-
// the
|
|
719
|
-
//
|
|
720
|
-
// workflow is
|
|
721
|
-
//
|
|
807
|
+
// Record a workflow's terminal outcome (SUCCESS or ERROR), reporting whether
|
|
808
|
+
// the write landed. The write applies only to a PENDING row: a run owns its
|
|
809
|
+
// workflow's outcome exactly as long as the row says that run is what the
|
|
810
|
+
// workflow is doing. (Note: this does not prevent a write when another
|
|
811
|
+
// concurrent execution is already running and the status is PENDING. However,
|
|
812
|
+
// both executions should be deterministic and idempotent.)
|
|
813
|
+
//
|
|
814
|
+
// Returning false means the row was CANCELLED, dead-lettered, already
|
|
815
|
+
// terminal, handed to another execution (ENQUEUED/DELAYED, e.g. by a
|
|
816
|
+
// concurrent resume), or deleted; the caller resolves which by awaiting the
|
|
817
|
+
// recorded outcome.
|
|
722
818
|
async #recordWorkflowOutcome(client, workflowID, status, outcome) {
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
throwOnFailure: false,
|
|
730
|
-
});
|
|
731
|
-
cancelled = (await this.getWorkflowStatusValue(client, workflowID)) === workflow_1.StatusString.CANCELLED;
|
|
732
|
-
await client.query('COMMIT');
|
|
733
|
-
}
|
|
734
|
-
catch (e) {
|
|
735
|
-
await client.query('ROLLBACK');
|
|
736
|
-
throw e;
|
|
737
|
-
}
|
|
738
|
-
if (cancelled) {
|
|
739
|
-
throw new error_1.DBOSWorkflowCancelledError(workflowID);
|
|
740
|
-
}
|
|
819
|
+
const rowCount = await this.updateWorkflowStatus(client, workflowID, status, {
|
|
820
|
+
update: { ...outcome, resetDeduplicationID: true, setCompletedAt: true },
|
|
821
|
+
where: { status: workflow_1.StatusString.PENDING },
|
|
822
|
+
throwOnFailure: false,
|
|
823
|
+
});
|
|
824
|
+
return rowCount > 0;
|
|
741
825
|
}
|
|
742
826
|
async getPendingWorkflows(executorID, appVersion) {
|
|
827
|
+
const params = [workflow_1.StatusString.PENDING, executorID, appVersion];
|
|
828
|
+
// executor_id defaults to "local", so it collides across applications.
|
|
829
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
743
830
|
const getWorkflows = await this.pool.query(`SELECT workflow_uuid
|
|
744
831
|
FROM "${this.schemaName}".workflow_status
|
|
745
|
-
WHERE status=$1 AND executor_id=$2 AND application_version=$3
|
|
832
|
+
WHERE status=$1 AND executor_id=$2 AND application_version=$3 AND ${scope}`, params);
|
|
746
833
|
return getWorkflows.rows.map((i) => ({
|
|
747
834
|
workflowUUID: i.workflow_uuid,
|
|
748
835
|
}));
|
|
749
836
|
}
|
|
750
837
|
// Recovery re-enqueues rather than executing directly so the queue's atomic dequeue admits exactly one runner, and the executor ID predicate rejects sweeps for rows a live executor has already claimed.
|
|
751
838
|
async reenqueueWorkflowsForRecovery(executorID, appVersion, recoveryQueueName) {
|
|
839
|
+
const params = [
|
|
840
|
+
workflow_1.StatusString.ENQUEUED,
|
|
841
|
+
Date.now(),
|
|
842
|
+
recoveryQueueName,
|
|
843
|
+
workflow_1.StatusString.PENDING,
|
|
844
|
+
executorID,
|
|
845
|
+
appVersion,
|
|
846
|
+
];
|
|
847
|
+
// executor_id defaults to "local", so it collides across applications.
|
|
848
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
752
849
|
const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
753
850
|
SET started_at_epoch_ms = NULL,
|
|
754
851
|
status = $1,
|
|
@@ -757,7 +854,8 @@ class SystemDatabase {
|
|
|
757
854
|
WHERE status = $4
|
|
758
855
|
AND executor_id = $5
|
|
759
856
|
AND application_version = $6
|
|
760
|
-
|
|
857
|
+
AND ${scope}
|
|
858
|
+
RETURNING workflow_uuid`, params);
|
|
761
859
|
return result.rows.map((row) => row.workflow_uuid);
|
|
762
860
|
}
|
|
763
861
|
async getWorkflowStatus(workflowID, callerID, callerFN) {
|
|
@@ -883,9 +981,9 @@ class SystemDatabase {
|
|
|
883
981
|
// Insert a patchmarker
|
|
884
982
|
const dn = Date.now();
|
|
885
983
|
await this.pool.query(`INSERT INTO ${this.schemaName}.operation_outputs
|
|
886
|
-
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms)
|
|
887
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
888
|
-
ON CONFLICT DO NOTHING;`, [workflowID, functionID, null, null, patchName, null, dn, dn]);
|
|
984
|
+
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name)
|
|
985
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
986
|
+
ON CONFLICT DO NOTHING;`, [workflowID, functionID, null, null, patchName, null, dn, dn, this.appName ?? null]);
|
|
889
987
|
return { isPatched: true, hasEntry: true };
|
|
890
988
|
}
|
|
891
989
|
// ==================== Workflow Management ====================
|
|
@@ -943,8 +1041,9 @@ class SystemDatabase {
|
|
|
943
1041
|
* Extend an existing debounced DELAYED workflow's delay and update its inputs, atomically.
|
|
944
1042
|
* The new delay is capped at the workflow's debounce_deadline_epoch_ms, if one is set.
|
|
945
1043
|
* Matching on workflow name and class ensures a debounce-key collision between different
|
|
946
|
-
* workflows never overwrites another workflow's inputs.
|
|
947
|
-
*
|
|
1044
|
+
* workflows never overwrites another workflow's inputs. The bounce acts for `params.applicationName`:
|
|
1045
|
+
* it extends only that application's holders plus unclaimed ones, claiming those. If nothing matched,
|
|
1046
|
+
* returns the current holder (or that the key is unheld) so the caller can start fresh or surface a conflict.
|
|
948
1047
|
* Runs on `client` if given, joining its transaction (e.g. a transactional step's);
|
|
949
1048
|
* otherwise in its own retried transaction.
|
|
950
1049
|
*/
|
|
@@ -972,17 +1071,7 @@ class SystemDatabase {
|
|
|
972
1071
|
}
|
|
973
1072
|
async #debounceDelayedWorkflowInternal(client, params) {
|
|
974
1073
|
const classNameOrNull = params.workflowClassName === '' ? null : params.workflowClassName;
|
|
975
|
-
const
|
|
976
|
-
SET delay_until_epoch_ms = CASE
|
|
977
|
-
WHEN debounce_deadline_epoch_ms IS NOT NULL AND debounce_deadline_epoch_ms < $1
|
|
978
|
-
THEN debounce_deadline_epoch_ms
|
|
979
|
-
ELSE $1
|
|
980
|
-
END,
|
|
981
|
-
inputs = $2, serialization = $3, updated_at = $4
|
|
982
|
-
WHERE name = $5 AND class_name IS NOT DISTINCT FROM $6
|
|
983
|
-
AND queue_name = $7 AND deduplication_id = $8
|
|
984
|
-
AND status = $9 AND is_debounced = TRUE
|
|
985
|
-
RETURNING workflow_uuid`, [
|
|
1074
|
+
const updateParams = [
|
|
986
1075
|
params.delayUntilEpochMS,
|
|
987
1076
|
params.input,
|
|
988
1077
|
params.serialization,
|
|
@@ -992,7 +1081,24 @@ class SystemDatabase {
|
|
|
992
1081
|
params.queueName,
|
|
993
1082
|
params.deduplicationID,
|
|
994
1083
|
workflow_1.StatusString.DELAYED,
|
|
995
|
-
|
|
1084
|
+
params.applicationName ?? null,
|
|
1085
|
+
];
|
|
1086
|
+
// Never extend a workflow the target application doesn't own; falls through to the holder below.
|
|
1087
|
+
const ownScope = this.#appNameFilter('application_name', params.applicationName, updateParams);
|
|
1088
|
+
const updated = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
1089
|
+
SET delay_until_epoch_ms = CASE
|
|
1090
|
+
WHEN debounce_deadline_epoch_ms IS NOT NULL AND debounce_deadline_epoch_ms < $1
|
|
1091
|
+
THEN debounce_deadline_epoch_ms
|
|
1092
|
+
ELSE $1
|
|
1093
|
+
END,
|
|
1094
|
+
inputs = $2, serialization = $3, updated_at = $4,
|
|
1095
|
+
-- Claim it for the target, as its dequeue would: left unclaimed, every peer coalesces onto the one workflow and the last inputs win.
|
|
1096
|
+
application_name = COALESCE(application_name, $10)
|
|
1097
|
+
WHERE name = $5 AND class_name IS NOT DISTINCT FROM $6
|
|
1098
|
+
AND queue_name = $7 AND deduplication_id = $8
|
|
1099
|
+
AND status = $9 AND is_debounced = TRUE
|
|
1100
|
+
AND ${ownScope}
|
|
1101
|
+
RETURNING workflow_uuid`, updateParams);
|
|
996
1102
|
if (updated.rows.length > 0) {
|
|
997
1103
|
return {
|
|
998
1104
|
bouncedWorkflowID: updated.rows[0].workflow_uuid,
|
|
@@ -1000,10 +1106,11 @@ class SystemDatabase {
|
|
|
1000
1106
|
holderIsDebounced: false,
|
|
1001
1107
|
holderWorkflowName: null,
|
|
1002
1108
|
holderWorkflowClassName: null,
|
|
1109
|
+
holderApplicationName: null,
|
|
1003
1110
|
};
|
|
1004
1111
|
}
|
|
1005
|
-
//
|
|
1006
|
-
const holder = await client.query(`SELECT workflow_uuid, is_debounced, name, class_name
|
|
1112
|
+
// Unscoped, so a holder that blocked the update above is reportable.
|
|
1113
|
+
const holder = await client.query(`SELECT workflow_uuid, is_debounced, name, class_name, application_name
|
|
1007
1114
|
FROM "${this.schemaName}".workflow_status
|
|
1008
1115
|
WHERE queue_name = $1 AND deduplication_id = $2`, [params.queueName, params.deduplicationID]);
|
|
1009
1116
|
if (holder.rows.length === 0) {
|
|
@@ -1013,6 +1120,7 @@ class SystemDatabase {
|
|
|
1013
1120
|
holderIsDebounced: false,
|
|
1014
1121
|
holderWorkflowName: null,
|
|
1015
1122
|
holderWorkflowClassName: null,
|
|
1123
|
+
holderApplicationName: null,
|
|
1016
1124
|
};
|
|
1017
1125
|
}
|
|
1018
1126
|
return {
|
|
@@ -1021,6 +1129,7 @@ class SystemDatabase {
|
|
|
1021
1129
|
holderIsDebounced: holder.rows[0].is_debounced ?? false,
|
|
1022
1130
|
holderWorkflowName: holder.rows[0].name,
|
|
1023
1131
|
holderWorkflowClassName: holder.rows[0].class_name ?? null,
|
|
1132
|
+
holderApplicationName: holder.rows[0].application_name ?? null,
|
|
1024
1133
|
};
|
|
1025
1134
|
}
|
|
1026
1135
|
// Get the immediate (one-level) child workflow IDs for a set of workflows.
|
|
@@ -1132,7 +1241,7 @@ class SystemDatabase {
|
|
|
1132
1241
|
// Fetch the status of all original workflows inside the transaction.
|
|
1133
1242
|
const { rows: statusRows } = await client.query(`SELECT workflow_uuid, name, class_name, config_name, application_id,
|
|
1134
1243
|
authenticated_user, authenticated_roles, assumed_role, inputs, serialization,
|
|
1135
|
-
request, application_version, attributes
|
|
1244
|
+
request, application_version, attributes, application_name
|
|
1136
1245
|
FROM "${this.schemaName}".workflow_status
|
|
1137
1246
|
WHERE workflow_uuid = ANY($1)`, [originalWorkflowIDs]);
|
|
1138
1247
|
const statusByID = new Map(statusRows.map((r) => [r.workflow_uuid, r]));
|
|
@@ -1161,10 +1270,16 @@ class SystemDatabase {
|
|
|
1161
1270
|
'forked_from',
|
|
1162
1271
|
'serialization',
|
|
1163
1272
|
'attributes',
|
|
1273
|
+
'application_name',
|
|
1164
1274
|
];
|
|
1165
1275
|
if (options.timeoutMS !== undefined) {
|
|
1166
1276
|
insertCols.push('workflow_timeout_ms');
|
|
1167
1277
|
}
|
|
1278
|
+
// One owner per fork, shared by its status row and its copied steps: the source's, or this application claiming an unclaimed one.
|
|
1279
|
+
const forkOwners = new Map(originalWorkflowIDs.map((origID, i) => [
|
|
1280
|
+
forkedWorkflowIDs[i],
|
|
1281
|
+
statusByID.get(origID).application_name ?? this.appName ?? null,
|
|
1282
|
+
]));
|
|
1168
1283
|
const valuesPlaceholders = [];
|
|
1169
1284
|
const params = [];
|
|
1170
1285
|
let paramIdx = 1;
|
|
@@ -1174,7 +1289,7 @@ class SystemDatabase {
|
|
|
1174
1289
|
const ws = statusByID.get(origID);
|
|
1175
1290
|
const placeholders = insertCols.map(() => `$${paramIdx++}`).join(', ');
|
|
1176
1291
|
valuesPlaceholders.push(`(${placeholders})`);
|
|
1177
|
-
params.push(forkID, workflow_1.StatusString.ENQUEUED, ws.name, ws.class_name ?? null, ws.config_name ?? null, queueName, ws.authenticated_user, ws.assumed_role, ws.authenticated_roles, ws.request, options.applicationVersion ?? ws.application_version ?? null, ws.application_id, ws.inputs, options.queuePartitionKey ?? null, origID, ws.serialization, ws.attributes ? JSON.stringify(ws.attributes) : null);
|
|
1292
|
+
params.push(forkID, workflow_1.StatusString.ENQUEUED, ws.name, ws.class_name ?? null, ws.config_name ?? null, queueName, ws.authenticated_user, ws.assumed_role, ws.authenticated_roles, ws.request, options.applicationVersion ?? ws.application_version ?? null, ws.application_id, ws.inputs, options.queuePartitionKey ?? null, origID, ws.serialization, ws.attributes ? JSON.stringify(ws.attributes) : null, forkOwners.get(forkID) ?? null);
|
|
1178
1293
|
if (options.timeoutMS !== undefined) {
|
|
1179
1294
|
params.push(options.timeoutMS);
|
|
1180
1295
|
}
|
|
@@ -1193,11 +1308,12 @@ class SystemDatabase {
|
|
|
1193
1308
|
const mappingParams = [];
|
|
1194
1309
|
let mIdx = 1;
|
|
1195
1310
|
for (const m of forkMappings) {
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1311
|
+
// Cast: an unclaimed fork makes owner a bare NULL the VALUES list cannot type.
|
|
1312
|
+
mappingValues.push(`($${mIdx}::text, $${mIdx + 1}::text, $${mIdx + 2}::int, $${mIdx + 3}::text)`);
|
|
1313
|
+
mappingParams.push(m.origID, m.forkID, m.startStep, forkOwners.get(m.forkID) ?? null);
|
|
1314
|
+
mIdx += 4;
|
|
1199
1315
|
}
|
|
1200
|
-
const mappingCTE = `WITH mapping(orig_id, fork_id, start_step) AS (VALUES ${mappingValues.join(', ')})`;
|
|
1316
|
+
const mappingCTE = `WITH mapping(orig_id, fork_id, start_step, owner) AS (VALUES ${mappingValues.join(', ')})`;
|
|
1201
1317
|
// Build the child_workflow_id expression, applying replacements if provided.
|
|
1202
1318
|
let childWfExpr = 'oo.child_workflow_id';
|
|
1203
1319
|
const ooParams = [...mappingParams];
|
|
@@ -1213,8 +1329,8 @@ class SystemDatabase {
|
|
|
1213
1329
|
// Copy operation outputs
|
|
1214
1330
|
await client.query(`${mappingCTE}
|
|
1215
1331
|
INSERT INTO "${this.schemaName}".operation_outputs
|
|
1216
|
-
(workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms)
|
|
1217
|
-
SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms
|
|
1332
|
+
(workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name)
|
|
1333
|
+
SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner
|
|
1218
1334
|
FROM mapping m
|
|
1219
1335
|
JOIN "${this.schemaName}".operation_outputs oo
|
|
1220
1336
|
ON oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
|
|
@@ -1282,7 +1398,7 @@ class SystemDatabase {
|
|
|
1282
1398
|
deduplication_id, inputs, priority, queue_partition_key, forked_from,
|
|
1283
1399
|
parent_workflow_id, serialization, delay_until_epoch_ms,
|
|
1284
1400
|
was_forked_from, rate_limited, completed_at, attributes, schedule_name,
|
|
1285
|
-
debounce_deadline_epoch_ms, is_debounced
|
|
1401
|
+
debounce_deadline_epoch_ms, is_debounced, application_name
|
|
1286
1402
|
FROM "${this.schemaName}".workflow_status
|
|
1287
1403
|
WHERE workflow_uuid = $1`, [wfID]);
|
|
1288
1404
|
if (statusResult.rows.length === 0) {
|
|
@@ -1293,7 +1409,7 @@ class SystemDatabase {
|
|
|
1293
1409
|
const outputsResult = await client.query(`SELECT
|
|
1294
1410
|
workflow_uuid, function_id, function_name, output, error,
|
|
1295
1411
|
child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
|
|
1296
|
-
serialization
|
|
1412
|
+
serialization, application_name
|
|
1297
1413
|
FROM "${this.schemaName}".operation_outputs
|
|
1298
1414
|
WHERE workflow_uuid = $1`, [wfID]);
|
|
1299
1415
|
// Export workflow_events
|
|
@@ -1338,8 +1454,8 @@ class SystemDatabase {
|
|
|
1338
1454
|
deduplication_id, inputs, priority, queue_partition_key, forked_from,
|
|
1339
1455
|
parent_workflow_id, serialization, delay_until_epoch_ms,
|
|
1340
1456
|
was_forked_from, rate_limited, completed_at, attributes, schedule_name,
|
|
1341
|
-
debounce_deadline_epoch_ms, is_debounced
|
|
1342
|
-
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)`, [
|
|
1457
|
+
debounce_deadline_epoch_ms, is_debounced, application_name
|
|
1458
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37)`, [
|
|
1343
1459
|
status.workflow_uuid,
|
|
1344
1460
|
status.status,
|
|
1345
1461
|
status.name,
|
|
@@ -1378,14 +1494,15 @@ class SystemDatabase {
|
|
|
1378
1494
|
status.schedule_name ?? null,
|
|
1379
1495
|
status.debounce_deadline_epoch_ms ?? null,
|
|
1380
1496
|
status.is_debounced ?? false,
|
|
1497
|
+
status.application_name ?? null,
|
|
1381
1498
|
]);
|
|
1382
1499
|
// Import operation_outputs
|
|
1383
1500
|
for (const output of workflow.operation_outputs) {
|
|
1384
1501
|
await client.query(`INSERT INTO "${this.schemaName}".operation_outputs (
|
|
1385
1502
|
workflow_uuid, function_id, function_name, output, error,
|
|
1386
1503
|
child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
|
|
1387
|
-
serialization
|
|
1388
|
-
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [
|
|
1504
|
+
serialization, application_name
|
|
1505
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, [
|
|
1389
1506
|
output.workflow_uuid,
|
|
1390
1507
|
output.function_id,
|
|
1391
1508
|
output.function_name,
|
|
@@ -1395,6 +1512,7 @@ class SystemDatabase {
|
|
|
1395
1512
|
output.started_at_epoch_ms,
|
|
1396
1513
|
output.completed_at_epoch_ms,
|
|
1397
1514
|
output.serialization,
|
|
1515
|
+
output.application_name ?? null,
|
|
1398
1516
|
]);
|
|
1399
1517
|
}
|
|
1400
1518
|
// Import workflow_events
|
|
@@ -1487,7 +1605,12 @@ class SystemDatabase {
|
|
|
1487
1605
|
async #checkIfCanceledLimited(workflowID) {
|
|
1488
1606
|
await this.#pollWithLimiter(() => this.#checkIfCanceled(this.pool, workflowID));
|
|
1489
1607
|
}
|
|
1490
|
-
|
|
1608
|
+
// A missing row normally means the workflow has not been inserted yet, so
|
|
1609
|
+
// polling for it is correct. Callers that know the row must already exist
|
|
1610
|
+
// (e.g. a run parking on an outcome it just failed to write) pass
|
|
1611
|
+
// `failIfMissing` to fail fast with DBOSNonExistentWorkflowError instead of
|
|
1612
|
+
// polling forever.
|
|
1613
|
+
async awaitWorkflowResult(workflowID, timeoutSeconds, callerID, timerFuncID, pollingIntervalMs, failIfMissing) {
|
|
1491
1614
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
1492
1615
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
1493
1616
|
const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
|
|
@@ -1499,33 +1622,37 @@ class SystemDatabase {
|
|
|
1499
1622
|
while (true) {
|
|
1500
1623
|
if (callerID)
|
|
1501
1624
|
await this.#checkIfCanceledLimited(callerID);
|
|
1625
|
+
let rows;
|
|
1502
1626
|
try {
|
|
1503
|
-
|
|
1504
|
-
WHERE workflow_uuid=$1`, [workflowID]));
|
|
1505
|
-
if (rows.length > 0) {
|
|
1506
|
-
const status = rows[0].status;
|
|
1507
|
-
if (status === workflow_1.StatusString.SUCCESS) {
|
|
1508
|
-
return { output: rows[0].output, serialization: rows[0].serialization };
|
|
1509
|
-
}
|
|
1510
|
-
else if (status === workflow_1.StatusString.ERROR) {
|
|
1511
|
-
return { error: rows[0].error, serialization: rows[0].serialization };
|
|
1512
|
-
}
|
|
1513
|
-
else if (status === workflow_1.StatusString.CANCELLED) {
|
|
1514
|
-
return { cancelled: true };
|
|
1515
|
-
}
|
|
1516
|
-
else if (status === workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED) {
|
|
1517
|
-
return { maxRecoveryAttemptsExceeded: true };
|
|
1518
|
-
}
|
|
1519
|
-
else {
|
|
1520
|
-
// Status is not actionable
|
|
1521
|
-
}
|
|
1522
|
-
}
|
|
1627
|
+
({ rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT status, output, error, serialization FROM "${this.schemaName}".workflow_status
|
|
1628
|
+
WHERE workflow_uuid=$1`, [workflowID])));
|
|
1523
1629
|
}
|
|
1524
1630
|
catch (e) {
|
|
1525
1631
|
const err = e;
|
|
1526
1632
|
this.logger.error(`Exception from system database: ${err}`, err);
|
|
1527
1633
|
throw err;
|
|
1528
1634
|
}
|
|
1635
|
+
if (rows.length > 0) {
|
|
1636
|
+
const status = rows[0].status;
|
|
1637
|
+
if (status === workflow_1.StatusString.SUCCESS) {
|
|
1638
|
+
return { output: rows[0].output, serialization: rows[0].serialization };
|
|
1639
|
+
}
|
|
1640
|
+
else if (status === workflow_1.StatusString.ERROR) {
|
|
1641
|
+
return { error: rows[0].error, serialization: rows[0].serialization };
|
|
1642
|
+
}
|
|
1643
|
+
else if (status === workflow_1.StatusString.CANCELLED) {
|
|
1644
|
+
return { cancelled: true };
|
|
1645
|
+
}
|
|
1646
|
+
else if (status === workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED) {
|
|
1647
|
+
return { maxRecoveryAttemptsExceeded: true };
|
|
1648
|
+
}
|
|
1649
|
+
else {
|
|
1650
|
+
// Status is not actionable
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
else if (failIfMissing) {
|
|
1654
|
+
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
|
|
1655
|
+
}
|
|
1529
1656
|
const ct = Date.now();
|
|
1530
1657
|
if (finishTime && ct > finishTime)
|
|
1531
1658
|
return undefined; // Time's up
|
|
@@ -2089,10 +2216,13 @@ class SystemDatabase {
|
|
|
2089
2216
|
// Transition workflows from DELAYED to ENQUEUED when their delay has expired.
|
|
2090
2217
|
// For debounced workflows, clear the deduplication ID in the same atomic update: it is a
|
|
2091
2218
|
// debounce key held only while DELAYED, so a later same-key debounce starts a fresh workflow.
|
|
2219
|
+
const params = [workflow_1.StatusString.ENQUEUED, Date.now(), workflow_1.StatusString.DELAYED];
|
|
2220
|
+
// Only what this application would dequeue: a peer's debounce key is not ours to clear.
|
|
2221
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
2092
2222
|
await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2093
2223
|
SET status = $1, updated_at = $2,
|
|
2094
2224
|
deduplication_id = CASE WHEN is_debounced THEN NULL ELSE deduplication_id END
|
|
2095
|
-
WHERE status = $3 AND delay_until_epoch_ms <= $2
|
|
2225
|
+
WHERE status = $3 AND delay_until_epoch_ms <= $2 AND ${scope}`, params);
|
|
2096
2226
|
}
|
|
2097
2227
|
async getDeduplicatedWorkflow(queueName, deduplicationID) {
|
|
2098
2228
|
const { rows } = await this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
@@ -2104,18 +2234,21 @@ class SystemDatabase {
|
|
|
2104
2234
|
}
|
|
2105
2235
|
async getQueuePartitions(queueName) {
|
|
2106
2236
|
// Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v2, so cost scales with the number of partitions rather than the backlog depth.
|
|
2237
|
+
const params = [queueName, workflow_1.StatusString.ENQUEUED];
|
|
2238
|
+
// Only partitions this application can actually dequeue from.
|
|
2239
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
2107
2240
|
const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
|
|
2108
2241
|
(SELECT MIN(queue_partition_key) AS pk
|
|
2109
2242
|
FROM "${this.schemaName}".workflow_status
|
|
2110
|
-
WHERE queue_name = $1 AND status = $2 AND queue_partition_key IS NOT NULL)
|
|
2243
|
+
WHERE queue_name = $1 AND status = $2 AND queue_partition_key IS NOT NULL AND ${scope})
|
|
2111
2244
|
UNION ALL
|
|
2112
2245
|
(SELECT (SELECT MIN(queue_partition_key)
|
|
2113
2246
|
FROM "${this.schemaName}".workflow_status
|
|
2114
|
-
WHERE queue_name = $1 AND status = $2 AND queue_partition_key > partitions.pk)
|
|
2247
|
+
WHERE queue_name = $1 AND status = $2 AND queue_partition_key > partitions.pk AND ${scope})
|
|
2115
2248
|
FROM partitions
|
|
2116
2249
|
WHERE partitions.pk IS NOT NULL)
|
|
2117
2250
|
)
|
|
2118
|
-
SELECT pk FROM partitions WHERE pk IS NOT NULL`,
|
|
2251
|
+
SELECT pk FROM partitions WHERE pk IS NOT NULL`, params);
|
|
2119
2252
|
return rows.map((row) => row.pk);
|
|
2120
2253
|
}
|
|
2121
2254
|
async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey) {
|
|
@@ -2149,11 +2282,14 @@ class SystemDatabase {
|
|
|
2149
2282
|
startTimeMs - limiterPeriodMS,
|
|
2150
2283
|
...partitionParams,
|
|
2151
2284
|
];
|
|
2285
|
+
// Count only what this application would dequeue, matching the select below.
|
|
2286
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
2152
2287
|
const countResult = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
|
|
2153
2288
|
WHERE queue_name = $1
|
|
2154
2289
|
AND rate_limited = TRUE
|
|
2155
2290
|
AND status NOT IN ($2, $3)
|
|
2156
2291
|
AND started_at_epoch_ms > $4
|
|
2292
|
+
AND ${scope}
|
|
2157
2293
|
${partitionFilter.replace('$PARTITION', '$5')}`, params);
|
|
2158
2294
|
numRecentQueries = Number(countResult.rows[0].count);
|
|
2159
2295
|
if (numRecentQueries >= queue.rateLimit.limitPerPeriod) {
|
|
@@ -2172,9 +2308,10 @@ class SystemDatabase {
|
|
|
2172
2308
|
if (queue.concurrency !== undefined) {
|
|
2173
2309
|
// Global concurrency still requires a DB query since other workers may be running workflows too.
|
|
2174
2310
|
const params = [queue.name, workflow_1.StatusString.PENDING, ...partitionParams];
|
|
2311
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
2175
2312
|
const runningTasksResult = await client.query(`SELECT COUNT(*) as task_count
|
|
2176
2313
|
FROM "${this.schemaName}".workflow_status
|
|
2177
|
-
WHERE queue_name = $1 AND status = $2
|
|
2314
|
+
WHERE queue_name = $1 AND status = $2 AND ${scope}
|
|
2178
2315
|
${partitionFilter.replace('$PARTITION', '$3')}`, params);
|
|
2179
2316
|
const totalRunningTasks = Number(runningTasksResult.rows[0]?.task_count ?? 0);
|
|
2180
2317
|
if (totalRunningTasks > queue.concurrency) {
|
|
@@ -2189,10 +2326,7 @@ class SystemDatabase {
|
|
|
2189
2326
|
return claimedIDs;
|
|
2190
2327
|
}
|
|
2191
2328
|
// Retrieve the first max_tasks workflows in the queue.
|
|
2192
|
-
const
|
|
2193
|
-
FROM "${this.schemaName}".application_versions
|
|
2194
|
-
ORDER BY version_timestamp DESC LIMIT 1`);
|
|
2195
|
-
const latestVersion = latestVersionResult.rows[0]?.version_name;
|
|
2329
|
+
const latestVersion = await this.#latestApplicationVersionName(client);
|
|
2196
2330
|
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
2197
2331
|
const versionClause = isLatestVersion
|
|
2198
2332
|
? '(application_version = $3 OR application_version IS NULL)'
|
|
@@ -2200,12 +2334,14 @@ class SystemDatabase {
|
|
|
2200
2334
|
const lockMode = queue.concurrency ? 'FOR UPDATE NOWAIT' : 'FOR UPDATE SKIP LOCKED';
|
|
2201
2335
|
const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
|
|
2202
2336
|
const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
|
|
2337
|
+
const selectScope = this.#appNameFilter('application_name', this.appName, selectParams);
|
|
2203
2338
|
const selectQuery = `
|
|
2204
2339
|
SELECT workflow_uuid
|
|
2205
2340
|
FROM "${this.schemaName}".workflow_status
|
|
2206
2341
|
WHERE status = $1
|
|
2207
2342
|
AND queue_name = $2
|
|
2208
2343
|
AND ${versionClause}
|
|
2344
|
+
AND ${selectScope}
|
|
2209
2345
|
${partitionFilter.replace('$PARTITION', '$4')}
|
|
2210
2346
|
ORDER BY priority ASC, created_at ASC
|
|
2211
2347
|
${limitClause}
|
|
@@ -2226,26 +2362,32 @@ class SystemDatabase {
|
|
|
2226
2362
|
// Start the functions by marking them as pending and updating their executor IDs.
|
|
2227
2363
|
// Only claim the workflow if the UPDATE actually transitioned an ENQUEUED row —
|
|
2228
2364
|
// otherwise another worker won the race and we must not re-dispatch it.
|
|
2365
|
+
const updateParams = [
|
|
2366
|
+
workflow_1.StatusString.PENDING,
|
|
2367
|
+
executorID,
|
|
2368
|
+
appVersion,
|
|
2369
|
+
startTimeMs,
|
|
2370
|
+
queue.rateLimit !== undefined,
|
|
2371
|
+
id,
|
|
2372
|
+
workflow_1.StatusString.ENQUEUED,
|
|
2373
|
+
// Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
|
|
2374
|
+
this.appName ?? null,
|
|
2375
|
+
];
|
|
2376
|
+
// Re-check ownership alongside status, as the partitioned claim guard does.
|
|
2377
|
+
const claimScope = this.#appNameFilter('application_name', this.appName, updateParams);
|
|
2229
2378
|
const updateRes = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2230
2379
|
SET status = $1,
|
|
2231
2380
|
executor_id = $2,
|
|
2232
2381
|
application_version = $3,
|
|
2233
2382
|
started_at_epoch_ms = $4,
|
|
2234
2383
|
rate_limited = $5,
|
|
2384
|
+
application_name = COALESCE(application_name, $8),
|
|
2235
2385
|
workflow_deadline_epoch_ms = CASE
|
|
2236
2386
|
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
|
|
2237
2387
|
THEN (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
|
|
2238
2388
|
ELSE workflow_deadline_epoch_ms
|
|
2239
2389
|
END
|
|
2240
|
-
WHERE workflow_uuid = $6 AND status = $7`,
|
|
2241
|
-
workflow_1.StatusString.PENDING,
|
|
2242
|
-
executorID,
|
|
2243
|
-
appVersion,
|
|
2244
|
-
startTimeMs,
|
|
2245
|
-
queue.rateLimit !== undefined,
|
|
2246
|
-
id,
|
|
2247
|
-
workflow_1.StatusString.ENQUEUED,
|
|
2248
|
-
]);
|
|
2390
|
+
WHERE workflow_uuid = $6 AND status = $7 AND ${claimScope}`, updateParams);
|
|
2249
2391
|
if ((updateRes.rowCount ?? 0) > 0) {
|
|
2250
2392
|
claimedIDs.push(id);
|
|
2251
2393
|
}
|
|
@@ -2274,23 +2416,28 @@ class SystemDatabase {
|
|
|
2274
2416
|
const client = await this.pool.connect();
|
|
2275
2417
|
try {
|
|
2276
2418
|
await client.query('BEGIN');
|
|
2277
|
-
const
|
|
2278
|
-
FROM "${this.schemaName}".application_versions
|
|
2279
|
-
ORDER BY version_timestamp DESC LIMIT 1`);
|
|
2280
|
-
const latestVersion = latestVersionResult.rows[0]?.version_name;
|
|
2419
|
+
const latestVersion = await this.#latestApplicationVersionName(client);
|
|
2281
2420
|
const isLatestVersion = latestVersion === undefined || latestVersion === appVersion;
|
|
2282
2421
|
const versionClause = (n) => isLatestVersion
|
|
2283
2422
|
? `(application_version = $${n} OR application_version IS NULL)`
|
|
2284
2423
|
: `application_version = $${n}`;
|
|
2424
|
+
const candidateParams = [
|
|
2425
|
+
queue.name,
|
|
2426
|
+
workflow_1.StatusString.ENQUEUED,
|
|
2427
|
+
appVersion,
|
|
2428
|
+
workflow_1.StatusString.PENDING,
|
|
2429
|
+
this.partitionedDequeueSweepCap,
|
|
2430
|
+
];
|
|
2431
|
+
const candidateScope = this.#appNameFilter('application_name', this.appName, candidateParams);
|
|
2285
2432
|
// Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
|
|
2286
2433
|
const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
|
|
2287
2434
|
(SELECT MIN(queue_partition_key) AS pk
|
|
2288
2435
|
FROM "${this.schemaName}".workflow_status
|
|
2289
|
-
WHERE queue_name = $1 AND status = $2 AND queue_partition_key IS NOT NULL)
|
|
2436
|
+
WHERE queue_name = $1 AND status = $2 AND queue_partition_key IS NOT NULL AND ${candidateScope})
|
|
2290
2437
|
UNION ALL
|
|
2291
2438
|
(SELECT (SELECT MIN(queue_partition_key)
|
|
2292
2439
|
FROM "${this.schemaName}".workflow_status
|
|
2293
|
-
WHERE queue_name = $1 AND status = $2 AND queue_partition_key > partitions.pk)
|
|
2440
|
+
WHERE queue_name = $1 AND status = $2 AND queue_partition_key > partitions.pk AND ${candidateScope})
|
|
2294
2441
|
FROM partitions
|
|
2295
2442
|
WHERE partitions.pk IS NOT NULL)
|
|
2296
2443
|
)
|
|
@@ -2303,12 +2450,13 @@ class SystemDatabase {
|
|
|
2303
2450
|
WHERE queue_name = $1 AND status = $2
|
|
2304
2451
|
AND queue_partition_key = partitions.pk
|
|
2305
2452
|
AND ${versionClause(3)}
|
|
2453
|
+
AND ${candidateScope}
|
|
2306
2454
|
-- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
|
|
2307
2455
|
ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
|
|
2308
2456
|
LIMIT 1
|
|
2309
2457
|
) head ON TRUE
|
|
2310
2458
|
WHERE partitions.pk IS NOT NULL
|
|
2311
|
-
--
|
|
2459
|
+
-- Unscoped by design: a mutual-exclusion probe must block on any owner's row.
|
|
2312
2460
|
AND NOT EXISTS (
|
|
2313
2461
|
SELECT 1
|
|
2314
2462
|
FROM "${this.schemaName}".workflow_status
|
|
@@ -2316,7 +2464,7 @@ class SystemDatabase {
|
|
|
2316
2464
|
AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
|
|
2317
2465
|
)
|
|
2318
2466
|
ORDER BY partitions.pk ASC
|
|
2319
|
-
LIMIT $5`,
|
|
2467
|
+
LIMIT $5`, candidateParams);
|
|
2320
2468
|
const candidateIDs = candidateResult.rows.map((row) => row.workflow_uuid);
|
|
2321
2469
|
if (candidateIDs.length === 0) {
|
|
2322
2470
|
await client.query('COMMIT');
|
|
@@ -2324,16 +2472,19 @@ class SystemDatabase {
|
|
|
2324
2472
|
}
|
|
2325
2473
|
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
|
|
2326
2474
|
// Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
|
|
2327
|
-
const claimGuard = (ids, status, name, version) => `workflow_uuid = ANY($${ids}::text[])
|
|
2475
|
+
const claimGuard = (ids, status, name, version, scope) => `workflow_uuid = ANY($${ids}::text[])
|
|
2328
2476
|
AND status = $${status}
|
|
2329
2477
|
AND queue_name = $${name}
|
|
2330
2478
|
AND queue_partition_key IS NOT NULL
|
|
2331
|
-
AND ${versionClause(version)}
|
|
2479
|
+
AND ${versionClause(version)}
|
|
2480
|
+
AND ${scope}`;
|
|
2481
|
+
const lockedParams = [candidateIDs, workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
|
|
2482
|
+
const lockedScope = this.#appNameFilter('application_name', this.appName, lockedParams);
|
|
2332
2483
|
// Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
|
|
2333
2484
|
const lockedResult = await client.query(`SELECT workflow_uuid
|
|
2334
2485
|
FROM "${this.schemaName}".workflow_status
|
|
2335
|
-
WHERE ${claimGuard(1, 2, 3, 4)}
|
|
2336
|
-
FOR UPDATE SKIP LOCKED`,
|
|
2486
|
+
WHERE ${claimGuard(1, 2, 3, 4, lockedScope)}
|
|
2487
|
+
FOR UPDATE SKIP LOCKED`, lockedParams);
|
|
2337
2488
|
const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
|
|
2338
2489
|
// Preserve partition order for submission.
|
|
2339
2490
|
const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
|
|
@@ -2341,6 +2492,18 @@ class SystemDatabase {
|
|
|
2341
2492
|
await client.query('COMMIT');
|
|
2342
2493
|
return [];
|
|
2343
2494
|
}
|
|
2495
|
+
const flipParams = [
|
|
2496
|
+
workflow_1.StatusString.PENDING,
|
|
2497
|
+
executorID,
|
|
2498
|
+
claimIDs,
|
|
2499
|
+
workflow_1.StatusString.ENQUEUED,
|
|
2500
|
+
queue.name,
|
|
2501
|
+
appVersion,
|
|
2502
|
+
startTimeMs,
|
|
2503
|
+
// Claim the row, as the unpartitioned dequeue does.
|
|
2504
|
+
this.appName ?? null,
|
|
2505
|
+
];
|
|
2506
|
+
const flipScope = this.#appNameFilter('application_name', this.appName, flipParams);
|
|
2344
2507
|
// Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
|
|
2345
2508
|
const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
|
|
2346
2509
|
SET status = $1,
|
|
@@ -2348,13 +2511,14 @@ class SystemDatabase {
|
|
|
2348
2511
|
application_version = $6,
|
|
2349
2512
|
started_at_epoch_ms = $7,
|
|
2350
2513
|
rate_limited = FALSE,
|
|
2514
|
+
application_name = COALESCE(application_name, $8),
|
|
2351
2515
|
workflow_deadline_epoch_ms = CASE
|
|
2352
2516
|
WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
|
|
2353
2517
|
THEN (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
|
|
2354
2518
|
ELSE workflow_deadline_epoch_ms
|
|
2355
2519
|
END
|
|
2356
|
-
WHERE ${claimGuard(3, 4, 5, 6)}
|
|
2357
|
-
RETURNING workflow_uuid`,
|
|
2520
|
+
WHERE ${claimGuard(3, 4, 5, 6, flipScope)}
|
|
2521
|
+
RETURNING workflow_uuid`, flipParams);
|
|
2358
2522
|
await client.query('COMMIT');
|
|
2359
2523
|
const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
|
|
2360
2524
|
const claimedIDs = claimIDs.filter((id) => flippedIDs.has(id));
|
|
@@ -2405,6 +2569,7 @@ class SystemDatabase {
|
|
|
2405
2569
|
'schedule_name',
|
|
2406
2570
|
'debounce_deadline_epoch_ms',
|
|
2407
2571
|
'is_debounced',
|
|
2572
|
+
'application_name',
|
|
2408
2573
|
];
|
|
2409
2574
|
input.loadInput = input.loadInput ?? true;
|
|
2410
2575
|
input.loadOutput = input.loadOutput ?? true;
|
|
@@ -2449,6 +2614,15 @@ class SystemDatabase {
|
|
|
2449
2614
|
addFilter('name', input.workflowName);
|
|
2450
2615
|
addFilter('queue_name', input.queueName);
|
|
2451
2616
|
addFilter('schedule_name', input.scheduleName);
|
|
2617
|
+
// A workflow ID is a global address, so an ID-keyed read is an identity read: it takes an
|
|
2618
|
+
// explicit filter but is never defaulted to this one. Otherwise unset scopes to this
|
|
2619
|
+
// application, as on every other observability query.
|
|
2620
|
+
// Falsy, not just undefined: a Conductor request carries an omitted list as JSON null.
|
|
2621
|
+
const idKeyed = (input.workflowIDs?.length ?? 0) > 0;
|
|
2622
|
+
whereClauses.push(idKeyed
|
|
2623
|
+
? this.#appNameFilter('application_name', input.applicationName, params)
|
|
2624
|
+
: this.#observabilityFilter('application_name', input.applicationName, params));
|
|
2625
|
+
paramCounter = params.length + 1;
|
|
2452
2626
|
if (input.workflow_id_prefix) {
|
|
2453
2627
|
if (Array.isArray(input.workflow_id_prefix)) {
|
|
2454
2628
|
const likeClauses = input.workflow_id_prefix.map((_, i) => `workflow_uuid LIKE $${paramCounter + i}`);
|
|
@@ -2551,6 +2725,7 @@ class SystemDatabase {
|
|
|
2551
2725
|
['queue_name', input.groupByQueueName ?? false, 'queue_name'],
|
|
2552
2726
|
['executor_id', input.groupByExecutorId ?? false, 'executor_id'],
|
|
2553
2727
|
['application_version', input.groupByApplicationVersion ?? false, 'application_version'],
|
|
2728
|
+
['application_name', input.groupByApplicationName ?? false, 'application_name'],
|
|
2554
2729
|
];
|
|
2555
2730
|
const groupNames = [];
|
|
2556
2731
|
const groupColumns = [];
|
|
@@ -2621,6 +2796,9 @@ class SystemDatabase {
|
|
|
2621
2796
|
addFilter('forked_from', input.forkedFrom);
|
|
2622
2797
|
addFilter('parent_workflow_id', input.parentWorkflowID);
|
|
2623
2798
|
addFilter('schedule_name', input.scheduleName);
|
|
2799
|
+
// Unset scopes to this application, as on every other observability query.
|
|
2800
|
+
whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
|
|
2801
|
+
paramIdx = params.length + 1;
|
|
2624
2802
|
// Only workflows that are actively enqueued.
|
|
2625
2803
|
if (input.queuesOnly) {
|
|
2626
2804
|
whereClauses.push(`queue_name IS NOT NULL`);
|
|
@@ -2783,6 +2961,9 @@ class SystemDatabase {
|
|
|
2783
2961
|
params.push(new Date(input.completedBefore).getTime());
|
|
2784
2962
|
paramIdx++;
|
|
2785
2963
|
}
|
|
2964
|
+
// Unset scopes to this application, as on every other observability query.
|
|
2965
|
+
whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
|
|
2966
|
+
paramIdx = params.length + 1;
|
|
2786
2967
|
const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
|
|
2787
2968
|
const groupByClause = groupColumns.join(', ');
|
|
2788
2969
|
const selectClause = [...groupSelectColumns, ...selectColumns].join(', ');
|
|
@@ -2811,10 +2992,13 @@ class SystemDatabase {
|
|
|
2811
2992
|
async garbageCollect(cutoffEpochTimestampMs, rowsThreshold) {
|
|
2812
2993
|
if (rowsThreshold !== undefined && rowsThreshold !== null) {
|
|
2813
2994
|
// Get the created_at timestamp of the rows_threshold newest row
|
|
2995
|
+
const params = [rowsThreshold - 1];
|
|
2996
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
2814
2997
|
const result = await this.pool.query(`SELECT created_at
|
|
2815
2998
|
FROM "${this.schemaName}".workflow_status
|
|
2999
|
+
WHERE ${scope}
|
|
2816
3000
|
ORDER BY created_at DESC
|
|
2817
|
-
LIMIT 1 OFFSET $1`,
|
|
3001
|
+
LIMIT 1 OFFSET $1`, params);
|
|
2818
3002
|
if (result.rows.length > 0) {
|
|
2819
3003
|
const rowsBasedCutoff = result.rows[0].created_at;
|
|
2820
3004
|
// Use the more restrictive cutoff (higher timestamp = more recent = more deletion)
|
|
@@ -2828,21 +3012,51 @@ class SystemDatabase {
|
|
|
2828
3012
|
if (cutoffEpochTimestampMs === undefined || cutoffEpochTimestampMs === null) {
|
|
2829
3013
|
return;
|
|
2830
3014
|
}
|
|
3015
|
+
const deleteParams = [
|
|
3016
|
+
cutoffEpochTimestampMs,
|
|
3017
|
+
workflow_1.StatusString.PENDING,
|
|
3018
|
+
workflow_1.StatusString.ENQUEUED,
|
|
3019
|
+
workflow_1.StatusString.DELAYED,
|
|
3020
|
+
];
|
|
3021
|
+
// Unclaimed rows included: excluding them would leak pre-upgrade rows forever.
|
|
3022
|
+
const deleteScope = this.#appNameFilter('application_name', this.appName, deleteParams);
|
|
2831
3023
|
// Delete all workflows older than cutoff that are NOT PENDING, ENQUEUED, or DELAYED
|
|
2832
3024
|
await this.pool.query(`DELETE FROM "${this.schemaName}".workflow_status
|
|
2833
3025
|
WHERE created_at < $1
|
|
2834
|
-
AND status NOT IN ($2, $3, $4)
|
|
3026
|
+
AND status NOT IN ($2, $3, $4)
|
|
3027
|
+
AND ${deleteScope}`, deleteParams);
|
|
2835
3028
|
return;
|
|
2836
3029
|
}
|
|
2837
|
-
|
|
3030
|
+
/**
|
|
3031
|
+
* IDs of this application's in-flight workflows created at or before the cutoff.
|
|
3032
|
+
* Claiming-scoped, so an upgrade still times out its own unclaimed workflows.
|
|
3033
|
+
*/
|
|
3034
|
+
async listTimedOutWorkflowIds(cutoffEpochTimestampMs) {
|
|
3035
|
+
const params = [
|
|
3036
|
+
cutoffEpochTimestampMs,
|
|
3037
|
+
workflow_1.StatusString.PENDING,
|
|
3038
|
+
workflow_1.StatusString.ENQUEUED,
|
|
3039
|
+
workflow_1.StatusString.DELAYED,
|
|
3040
|
+
];
|
|
3041
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
3042
|
+
const { rows } = await this.pool.query(`SELECT workflow_uuid
|
|
3043
|
+
FROM "${this.schemaName}".workflow_status
|
|
3044
|
+
WHERE created_at <= $1
|
|
3045
|
+
AND status IN ($2, $3, $4)
|
|
3046
|
+
AND ${scope}`, params);
|
|
3047
|
+
return rows.map((row) => row.workflow_uuid);
|
|
3048
|
+
}
|
|
3049
|
+
async getMetrics(startTime, endTime, applicationName) {
|
|
2838
3050
|
const startEpochMs = new Date(startTime).getTime();
|
|
2839
3051
|
const endEpochMs = new Date(endTime).getTime();
|
|
2840
3052
|
const metrics = [];
|
|
2841
3053
|
// Query workflow metrics
|
|
3054
|
+
const workflowParams = [startEpochMs, endEpochMs];
|
|
3055
|
+
const workflowScope = this.#observabilityFilter('application_name', applicationName, workflowParams);
|
|
2842
3056
|
const workflowResult = await this.pool.query(`SELECT name, COUNT(workflow_uuid) as count
|
|
2843
3057
|
FROM "${this.schemaName}".workflow_status
|
|
2844
|
-
WHERE created_at >= $1 AND created_at < $2
|
|
2845
|
-
GROUP BY name`,
|
|
3058
|
+
WHERE created_at >= $1 AND created_at < $2 AND ${workflowScope}
|
|
3059
|
+
GROUP BY name`, workflowParams);
|
|
2846
3060
|
for (const row of workflowResult.rows) {
|
|
2847
3061
|
metrics.push({
|
|
2848
3062
|
metricType: 'workflow_count',
|
|
@@ -2851,10 +3065,12 @@ class SystemDatabase {
|
|
|
2851
3065
|
});
|
|
2852
3066
|
}
|
|
2853
3067
|
// Query step metrics
|
|
3068
|
+
const stepParams = [startEpochMs, endEpochMs];
|
|
3069
|
+
const stepScope = this.#observabilityFilter('application_name', applicationName, stepParams);
|
|
2854
3070
|
const stepResult = await this.pool.query(`SELECT function_name, COUNT(*) as count
|
|
2855
3071
|
FROM "${this.schemaName}".operation_outputs
|
|
2856
|
-
WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2
|
|
2857
|
-
GROUP BY function_name`,
|
|
3072
|
+
WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2 AND ${stepScope}
|
|
3073
|
+
GROUP BY function_name`, stepParams);
|
|
2858
3074
|
for (const row of stepResult.rows) {
|
|
2859
3075
|
metrics.push({
|
|
2860
3076
|
metricType: 'step_count',
|
|
@@ -2867,10 +3083,11 @@ class SystemDatabase {
|
|
|
2867
3083
|
// ==================== Scheduling ====================
|
|
2868
3084
|
async createSchedule(schedule, client) {
|
|
2869
3085
|
const q = client ?? this.pool;
|
|
3086
|
+
const owner = await this.#resolveRowOwner(q, 'workflow_schedules', 'schedule_name', schedule.scheduleName, schedule.applicationName, 'Schedule');
|
|
2870
3087
|
try {
|
|
2871
3088
|
await q.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
2872
|
-
(schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name)
|
|
2873
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, [
|
|
3089
|
+
(schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name)
|
|
3090
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, [
|
|
2874
3091
|
schedule.scheduleId,
|
|
2875
3092
|
schedule.scheduleName,
|
|
2876
3093
|
schedule.workflowName,
|
|
@@ -2882,6 +3099,7 @@ class SystemDatabase {
|
|
|
2882
3099
|
schedule.automaticBackfill,
|
|
2883
3100
|
schedule.cronTimezone,
|
|
2884
3101
|
schedule.queueName,
|
|
3102
|
+
owner ?? null,
|
|
2885
3103
|
]);
|
|
2886
3104
|
}
|
|
2887
3105
|
catch (e) {
|
|
@@ -2891,6 +3109,10 @@ class SystemDatabase {
|
|
|
2891
3109
|
throw e;
|
|
2892
3110
|
}
|
|
2893
3111
|
}
|
|
3112
|
+
/**
|
|
3113
|
+
* List only schedules owned by these applications, plus unclaimed ones.
|
|
3114
|
+
* By default, only list this application's schedules.
|
|
3115
|
+
*/
|
|
2894
3116
|
async listSchedules(filters, client) {
|
|
2895
3117
|
const q = client ?? this.pool;
|
|
2896
3118
|
const conditions = [];
|
|
@@ -2922,45 +3144,23 @@ class SystemDatabase {
|
|
|
2922
3144
|
});
|
|
2923
3145
|
conditions.push(`(${likeClauses.join(' OR ')})`);
|
|
2924
3146
|
}
|
|
3147
|
+
// Unset scopes to this application, as on every other observability query.
|
|
3148
|
+
conditions.push(this.#observabilityFilter('application_name', filters?.applicationName, params));
|
|
3149
|
+
paramIdx = params.length + 1;
|
|
2925
3150
|
const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
|
|
2926
|
-
const result = await q.query(`SELECT
|
|
3151
|
+
const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
|
|
2927
3152
|
FROM "${this.schemaName}".workflow_schedules${where}
|
|
2928
3153
|
ORDER BY schedule_name`, params);
|
|
2929
|
-
return result.rows.map((row) => (
|
|
2930
|
-
scheduleId: row.schedule_id,
|
|
2931
|
-
scheduleName: row.schedule_name,
|
|
2932
|
-
workflowName: row.workflow_name,
|
|
2933
|
-
workflowClassName: row.workflow_class_name,
|
|
2934
|
-
schedule: row.schedule,
|
|
2935
|
-
status: row.status,
|
|
2936
|
-
context: row.context,
|
|
2937
|
-
lastFiredAt: row.last_fired_at ?? null,
|
|
2938
|
-
automaticBackfill: !!row.automatic_backfill,
|
|
2939
|
-
cronTimezone: row.cron_timezone ?? null,
|
|
2940
|
-
queueName: row.queue_name ?? null,
|
|
2941
|
-
}));
|
|
3154
|
+
return result.rows.map((row) => mapWorkflowSchedule(row));
|
|
2942
3155
|
}
|
|
2943
3156
|
async getSchedule(name, client) {
|
|
2944
3157
|
const q = client ?? this.pool;
|
|
2945
|
-
const result = await q.query(`SELECT
|
|
3158
|
+
const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
|
|
2946
3159
|
FROM "${this.schemaName}".workflow_schedules
|
|
2947
3160
|
WHERE schedule_name = $1`, [name]);
|
|
2948
3161
|
if (result.rows.length === 0)
|
|
2949
3162
|
return null;
|
|
2950
|
-
|
|
2951
|
-
return {
|
|
2952
|
-
scheduleId: row.schedule_id,
|
|
2953
|
-
scheduleName: row.schedule_name,
|
|
2954
|
-
workflowName: row.workflow_name,
|
|
2955
|
-
workflowClassName: row.workflow_class_name,
|
|
2956
|
-
schedule: row.schedule,
|
|
2957
|
-
status: row.status,
|
|
2958
|
-
context: row.context,
|
|
2959
|
-
lastFiredAt: row.last_fired_at ?? null,
|
|
2960
|
-
automaticBackfill: !!row.automatic_backfill,
|
|
2961
|
-
cronTimezone: row.cron_timezone ?? null,
|
|
2962
|
-
queueName: row.queue_name ?? null,
|
|
2963
|
-
};
|
|
3163
|
+
return mapWorkflowSchedule(result.rows[0]);
|
|
2964
3164
|
}
|
|
2965
3165
|
async deleteSchedule(name, client) {
|
|
2966
3166
|
const q = client ?? this.pool;
|
|
@@ -3016,10 +3216,11 @@ class SystemDatabase {
|
|
|
3016
3216
|
try {
|
|
3017
3217
|
await client.query('BEGIN');
|
|
3018
3218
|
for (const sched of schedules) {
|
|
3219
|
+
const owner = await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
3019
3220
|
// Upsert on schedule_name; on conflict, preserve schedule_id and runtime state (status, last_fired_at) and update only the declared definition fields, so an unchanged re-apply is a no-op.
|
|
3020
3221
|
await client.query(`INSERT INTO "${this.schemaName}".workflow_schedules
|
|
3021
|
-
(schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name)
|
|
3022
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
|
3222
|
+
(schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name, application_name)
|
|
3223
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
|
3023
3224
|
ON CONFLICT (schedule_name) DO UPDATE SET
|
|
3024
3225
|
workflow_name = EXCLUDED.workflow_name,
|
|
3025
3226
|
workflow_class_name = EXCLUDED.workflow_class_name,
|
|
@@ -3027,7 +3228,9 @@ class SystemDatabase {
|
|
|
3027
3228
|
context = EXCLUDED.context,
|
|
3028
3229
|
automatic_backfill = EXCLUDED.automatic_backfill,
|
|
3029
3230
|
cron_timezone = EXCLUDED.cron_timezone,
|
|
3030
|
-
queue_name = EXCLUDED.queue_name
|
|
3231
|
+
queue_name = EXCLUDED.queue_name,
|
|
3232
|
+
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it took.
|
|
3233
|
+
application_name = COALESCE("${this.schemaName}".workflow_schedules.application_name, EXCLUDED.application_name)`, [
|
|
3031
3234
|
sched.scheduleId,
|
|
3032
3235
|
sched.scheduleName,
|
|
3033
3236
|
sched.workflowName,
|
|
@@ -3039,7 +3242,10 @@ class SystemDatabase {
|
|
|
3039
3242
|
sched.automaticBackfill,
|
|
3040
3243
|
sched.cronTimezone,
|
|
3041
3244
|
sched.queueName,
|
|
3245
|
+
owner ?? null,
|
|
3042
3246
|
]);
|
|
3247
|
+
// Read back, since the guard above is silent about why it declined to claim.
|
|
3248
|
+
await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
|
|
3043
3249
|
}
|
|
3044
3250
|
await client.query('COMMIT');
|
|
3045
3251
|
}
|
|
@@ -3052,56 +3258,109 @@ class SystemDatabase {
|
|
|
3052
3258
|
}
|
|
3053
3259
|
}
|
|
3054
3260
|
// ==================== Application Versions ====================
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
|
|
3261
|
+
/**
|
|
3262
|
+
* Register this version, claiming the row if nobody owns it yet so a pinned version
|
|
3263
|
+
* does not stay unclaimed. A peer's name is a collision, which is why this throws.
|
|
3264
|
+
*/
|
|
3265
|
+
async createApplicationVersion(versionName, applicationName) {
|
|
3266
|
+
const owner = applicationName ?? this.appName;
|
|
3267
|
+
const client = await this.pool.connect();
|
|
3268
|
+
try {
|
|
3269
|
+
await client.query('BEGIN');
|
|
3270
|
+
// Claim a pre-upgrade row in place, so the version is not recreated or retimed.
|
|
3271
|
+
const claimed = await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3272
|
+
SET application_name = $1
|
|
3273
|
+
WHERE version_name = $2 AND application_name IS NULL`, [owner ?? null, versionName]);
|
|
3274
|
+
if ((claimed.rowCount ?? 0) === 0) {
|
|
3275
|
+
// Targetless DO NOTHING: names no arbiter, so it survives version_name's uniqueness being dropped while still absorbing a concurrent registrar.
|
|
3276
|
+
await client.query(`INSERT INTO "${this.schemaName}".application_versions (version_id, version_name, application_name)
|
|
3277
|
+
VALUES ($1, $2, $3)
|
|
3278
|
+
ON CONFLICT DO NOTHING`, [(0, crypto_1.randomUUID)(), versionName, owner ?? null]);
|
|
3279
|
+
}
|
|
3280
|
+
// Read back, since the writes above are silent about why they declined to claim.
|
|
3281
|
+
await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
|
|
3282
|
+
await client.query('COMMIT');
|
|
3283
|
+
}
|
|
3284
|
+
catch (e) {
|
|
3285
|
+
await client.query('ROLLBACK');
|
|
3286
|
+
throw e;
|
|
3287
|
+
}
|
|
3288
|
+
finally {
|
|
3289
|
+
client.release();
|
|
3290
|
+
}
|
|
3060
3291
|
}
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3292
|
+
/**
|
|
3293
|
+
* Promote a version to latest. Promoting a peer's is a collision, not a retiming;
|
|
3294
|
+
* promotion claims an unclaimed row, which would otherwise be every peer's latest.
|
|
3295
|
+
*/
|
|
3296
|
+
async updateApplicationVersionTimestamp(versionName, newTimestamp, applicationName) {
|
|
3297
|
+
const owner = applicationName ?? this.appName;
|
|
3298
|
+
const client = await this.pool.connect();
|
|
3299
|
+
try {
|
|
3300
|
+
await client.query('BEGIN');
|
|
3301
|
+
const resolved = await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
|
|
3302
|
+
// Scoped to the row this writer resolved to: once version_name is no longer globally unique, a bare name match would retime every peer's version.
|
|
3303
|
+
const scope = resolved === undefined ? 'application_name IS NULL' : '(application_name = $3 OR application_name IS NULL)';
|
|
3304
|
+
const params = [newTimestamp, versionName];
|
|
3305
|
+
if (resolved !== undefined)
|
|
3306
|
+
params.push(resolved);
|
|
3307
|
+
await client.query(`UPDATE "${this.schemaName}".application_versions
|
|
3308
|
+
SET version_timestamp = $1, application_name = ${resolved === undefined ? 'application_name' : '$3'}
|
|
3309
|
+
WHERE version_name = $2 AND ${scope}`, params);
|
|
3310
|
+
await client.query('COMMIT');
|
|
3311
|
+
}
|
|
3312
|
+
catch (e) {
|
|
3313
|
+
await client.query('ROLLBACK');
|
|
3314
|
+
throw e;
|
|
3315
|
+
}
|
|
3316
|
+
finally {
|
|
3317
|
+
client.release();
|
|
3318
|
+
}
|
|
3065
3319
|
}
|
|
3066
3320
|
async listApplicationVersions() {
|
|
3067
|
-
const
|
|
3321
|
+
const params = [];
|
|
3322
|
+
const scope = this.#appNameFilter('application_name', this.appName, params);
|
|
3323
|
+
const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
|
|
3068
3324
|
FROM "${this.schemaName}".application_versions
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
versionName: r.version_name,
|
|
3073
|
-
versionTimestamp: Number(r.version_timestamp),
|
|
3074
|
-
createdAt: Number(r.created_at),
|
|
3075
|
-
}));
|
|
3325
|
+
WHERE ${scope}
|
|
3326
|
+
ORDER BY version_timestamp DESC`, params);
|
|
3327
|
+
return rows.map(mapVersionInfo);
|
|
3076
3328
|
}
|
|
3077
|
-
|
|
3078
|
-
|
|
3329
|
+
/**
|
|
3330
|
+
* The latest version registered by an application. Defaults to this handle's, so a
|
|
3331
|
+
* caller acting for another one — firing its schedule — must name it.
|
|
3332
|
+
*/
|
|
3333
|
+
async getLatestApplicationVersion(applicationName) {
|
|
3334
|
+
const owner = applicationName ?? this.appName;
|
|
3335
|
+
const params = [];
|
|
3336
|
+
const scope = this.#appNameFilter('application_name', owner, params);
|
|
3337
|
+
const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
|
|
3079
3338
|
FROM "${this.schemaName}".application_versions
|
|
3339
|
+
WHERE ${scope}
|
|
3080
3340
|
ORDER BY version_timestamp DESC
|
|
3081
|
-
LIMIT 1
|
|
3341
|
+
LIMIT 1`, params);
|
|
3082
3342
|
if (rows.length === 0) {
|
|
3083
3343
|
throw new error_1.DBOSInitializationError('No application versions found');
|
|
3084
3344
|
}
|
|
3085
|
-
|
|
3086
|
-
return {
|
|
3087
|
-
versionId: r.version_id,
|
|
3088
|
-
versionName: r.version_name,
|
|
3089
|
-
versionTimestamp: Number(r.version_timestamp),
|
|
3090
|
-
createdAt: Number(r.created_at),
|
|
3091
|
-
};
|
|
3345
|
+
return mapVersionInfo(rows[0]);
|
|
3092
3346
|
}
|
|
3093
3347
|
// ==================== Queues ====================
|
|
3094
3348
|
async getQueue(name) {
|
|
3095
|
-
const { rows } = await this.pool.query(`SELECT
|
|
3096
|
-
priority_enabled, partition_queue, polling_interval_sec
|
|
3349
|
+
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3097
3350
|
FROM "${this.schemaName}".queues
|
|
3098
3351
|
WHERE name = $1`, [name]);
|
|
3099
3352
|
return rows.length === 0 ? null : queueRecordFromRow(rows[0]);
|
|
3100
3353
|
}
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3354
|
+
/**
|
|
3355
|
+
* List only queues owned by these applications, plus unclaimed ones.
|
|
3356
|
+
* By default, only list this application's queues.
|
|
3357
|
+
*/
|
|
3358
|
+
async listQueues(applicationName) {
|
|
3359
|
+
const params = [];
|
|
3360
|
+
const scope = this.#observabilityFilter('application_name', applicationName, params);
|
|
3361
|
+
const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
|
|
3362
|
+
FROM "${this.schemaName}".queues
|
|
3363
|
+
WHERE ${scope}`, params);
|
|
3105
3364
|
return rows.map(queueRecordFromRow);
|
|
3106
3365
|
}
|
|
3107
3366
|
async deleteQueue(name) {
|
|
@@ -3137,16 +3396,21 @@ class SystemDatabase {
|
|
|
3137
3396
|
priority_enabled = EXCLUDED.priority_enabled,
|
|
3138
3397
|
partition_queue = EXCLUDED.partition_queue,
|
|
3139
3398
|
polling_interval_sec = EXCLUDED.polling_interval_sec,
|
|
3140
|
-
updated_at = EXCLUDED.updated_at
|
|
3399
|
+
updated_at = EXCLUDED.updated_at,
|
|
3400
|
+
-- Claim only an unclaimed row, so a registration landing between the check above and this write keeps the name it just took.
|
|
3401
|
+
application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
|
|
3141
3402
|
: `ON CONFLICT (name) DO NOTHING`;
|
|
3403
|
+
const owner = record.applicationName ?? this.appName;
|
|
3142
3404
|
const client = await this.pool.connect();
|
|
3143
3405
|
try {
|
|
3144
3406
|
await client.query('BEGIN');
|
|
3145
3407
|
const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE name = $1`, [record.name]);
|
|
3408
|
+
// A name collision is a conflict in every mode: the name is the queue's address.
|
|
3409
|
+
const resolvedOwner = await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
3146
3410
|
await client.query(`INSERT INTO "${this.schemaName}".queues
|
|
3147
3411
|
(name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
|
|
3148
|
-
priority_enabled, partition_queue, polling_interval_sec, updated_at)
|
|
3149
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
3412
|
+
priority_enabled, partition_queue, polling_interval_sec, updated_at, application_name)
|
|
3413
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
|
3150
3414
|
${onConflict}`, [
|
|
3151
3415
|
record.name,
|
|
3152
3416
|
record.concurrency,
|
|
@@ -3157,7 +3421,10 @@ class SystemDatabase {
|
|
|
3157
3421
|
record.partitionQueue,
|
|
3158
3422
|
record.pollingIntervalSec,
|
|
3159
3423
|
now,
|
|
3424
|
+
resolvedOwner ?? null,
|
|
3160
3425
|
]);
|
|
3426
|
+
// Read back, since the guard above is silent about why it declined to claim.
|
|
3427
|
+
await this.#resolveRowOwner(client, 'queues', 'name', record.name, owner, 'Queue');
|
|
3161
3428
|
await client.query('COMMIT');
|
|
3162
3429
|
return existed.rowCount === 0;
|
|
3163
3430
|
}
|
|
@@ -3169,6 +3436,123 @@ class SystemDatabase {
|
|
|
3169
3436
|
client.release();
|
|
3170
3437
|
}
|
|
3171
3438
|
}
|
|
3439
|
+
// ==================== Application Rename ====================
|
|
3440
|
+
/**
|
|
3441
|
+
* Rows a rename moves: an application's own, unclaimed ones, or both. Unlike
|
|
3442
|
+
* the claiming scope, unclaimed rows are not implied; they move only when asked.
|
|
3443
|
+
*/
|
|
3444
|
+
#renameSource(oldName, adoptUnclaimedRows, params) {
|
|
3445
|
+
const clauses = [];
|
|
3446
|
+
if (oldName !== undefined) {
|
|
3447
|
+
params.push(oldName);
|
|
3448
|
+
clauses.push(`application_name = $${params.length}`);
|
|
3449
|
+
}
|
|
3450
|
+
if (adoptUnclaimedRows) {
|
|
3451
|
+
clauses.push('application_name IS NULL');
|
|
3452
|
+
}
|
|
3453
|
+
// Callers validate that at least one source is named.
|
|
3454
|
+
return `(${clauses.join(' OR ')})`;
|
|
3455
|
+
}
|
|
3456
|
+
/**
|
|
3457
|
+
* Re-own a table's rows in half-open key ranges, so a long history neither moves in
|
|
3458
|
+
* one transaction nor rescans what it already moved; a re-run resumes.
|
|
3459
|
+
*/
|
|
3460
|
+
async #renameRowsInBatches(table, keyColumn, oldName, newName, batchSize, adoptUnclaimedRows) {
|
|
3461
|
+
if (batchSize === undefined) {
|
|
3462
|
+
const params = [newName];
|
|
3463
|
+
const predicate = this.#renameSource(oldName, adoptUnclaimedRows, params);
|
|
3464
|
+
const res = await this.pool.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${predicate}`, params);
|
|
3465
|
+
return res.rowCount ?? 0;
|
|
3466
|
+
}
|
|
3467
|
+
let total = 0;
|
|
3468
|
+
// Ranges, not LIMIT: a LIMIT repages every row already moved, and an IN list of keys plans as a whole-table hash join.
|
|
3469
|
+
let watermark = undefined;
|
|
3470
|
+
for (;;) {
|
|
3471
|
+
const boundParams = [];
|
|
3472
|
+
const predicate = this.#renameSource(oldName, adoptUnclaimedRows, boundParams);
|
|
3473
|
+
let scope = predicate;
|
|
3474
|
+
if (watermark !== undefined) {
|
|
3475
|
+
boundParams.push(watermark);
|
|
3476
|
+
scope = `${predicate} AND ${keyColumn} > $${boundParams.length}`;
|
|
3477
|
+
}
|
|
3478
|
+
// The batchSize-th matching key bounds this range; distinct, so a key's rows are never split across batches.
|
|
3479
|
+
const upperResult = await this.pool.query(`SELECT DISTINCT ${keyColumn} FROM "${this.schemaName}".${table}
|
|
3480
|
+
WHERE ${scope} ORDER BY ${keyColumn} LIMIT 1 OFFSET ${batchSize - 1}`, boundParams);
|
|
3481
|
+
const upper = upperResult.rows[0]?.[keyColumn];
|
|
3482
|
+
const updateParams = [newName];
|
|
3483
|
+
const updatePredicate = this.#renameSource(oldName, adoptUnclaimedRows, updateParams);
|
|
3484
|
+
let batch = updatePredicate;
|
|
3485
|
+
if (upper !== undefined) {
|
|
3486
|
+
if (watermark !== undefined) {
|
|
3487
|
+
updateParams.push(watermark);
|
|
3488
|
+
batch = `${batch} AND ${keyColumn} > $${updateParams.length}`;
|
|
3489
|
+
}
|
|
3490
|
+
updateParams.push(upper);
|
|
3491
|
+
batch = `${batch} AND ${keyColumn} <= $${updateParams.length}`;
|
|
3492
|
+
}
|
|
3493
|
+
// The final batch drops the watermark, so rows that appeared below it still move.
|
|
3494
|
+
const res = await this.pool.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${batch}`, updateParams);
|
|
3495
|
+
total += res.rowCount ?? 0;
|
|
3496
|
+
// Fewer than a full batch remained, so that update took the rest.
|
|
3497
|
+
if (upper === undefined)
|
|
3498
|
+
return total;
|
|
3499
|
+
watermark = upper;
|
|
3500
|
+
}
|
|
3501
|
+
}
|
|
3502
|
+
/**
|
|
3503
|
+
* Give `newName` ownership of rows `oldName` holds, of unclaimed rows, or of both.
|
|
3504
|
+
* The renamed application must be stopped, or its dequeues race this.
|
|
3505
|
+
*/
|
|
3506
|
+
async renameApplication(oldName, newName, options = {}) {
|
|
3507
|
+
const adoptUnclaimedRows = options.adoptUnclaimedRows ?? false;
|
|
3508
|
+
const batchSize = options.batchSize === null ? undefined : (options.batchSize ?? exports.DEFAULT_RENAME_BATCH_SIZE);
|
|
3509
|
+
if (oldName !== undefined && oldName === '') {
|
|
3510
|
+
throw new error_1.DBOSError("The application's previous name cannot be empty.");
|
|
3511
|
+
}
|
|
3512
|
+
if (oldName === undefined && !adoptUnclaimedRows) {
|
|
3513
|
+
throw new error_1.DBOSError('Nothing to re-own: name the application to rename, adopt unclaimed rows, or both.');
|
|
3514
|
+
}
|
|
3515
|
+
if (oldName === newName) {
|
|
3516
|
+
throw new error_1.DBOSError(`Application '${newName}' already holds that name; nothing to rename.`);
|
|
3517
|
+
}
|
|
3518
|
+
// A NaN survives a bare `< 1` test and would only fail once it reached SQL, leaving the rename half-applied.
|
|
3519
|
+
if (batchSize !== undefined && (!Number.isInteger(batchSize) || batchSize < 1)) {
|
|
3520
|
+
throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
|
|
3521
|
+
}
|
|
3522
|
+
// Never a merge: queue, schedule, and version names are globally unique whatever their owner, so this cannot collide.
|
|
3523
|
+
const client = await this.pool.connect();
|
|
3524
|
+
let queues, schedules, versions, inFlight;
|
|
3525
|
+
try {
|
|
3526
|
+
await client.query('BEGIN');
|
|
3527
|
+
const move = async (table, statuses) => {
|
|
3528
|
+
const params = [newName];
|
|
3529
|
+
let where = this.#renameSource(oldName, adoptUnclaimedRows, params);
|
|
3530
|
+
if (statuses !== undefined) {
|
|
3531
|
+
params.push(statuses);
|
|
3532
|
+
where = `${where} AND status = ANY($${params.length})`;
|
|
3533
|
+
}
|
|
3534
|
+
const res = await client.query(`UPDATE "${this.schemaName}".${table} SET application_name = $1 WHERE ${where}`, params);
|
|
3535
|
+
return res.rowCount ?? 0;
|
|
3536
|
+
};
|
|
3537
|
+
// A half-owned application dequeues work whose version row it can no longer see, so these move together.
|
|
3538
|
+
queues = await move('queues');
|
|
3539
|
+
schedules = await move('workflow_schedules');
|
|
3540
|
+
versions = await move('application_versions');
|
|
3541
|
+
inFlight = await move('workflow_status', [workflow_1.StatusString.PENDING, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED]);
|
|
3542
|
+
await client.query('COMMIT');
|
|
3543
|
+
}
|
|
3544
|
+
catch (e) {
|
|
3545
|
+
await client.query('ROLLBACK');
|
|
3546
|
+
throw e;
|
|
3547
|
+
}
|
|
3548
|
+
finally {
|
|
3549
|
+
client.release();
|
|
3550
|
+
}
|
|
3551
|
+
// Only terminal rows are left to match, and they scope observability and GC alone, so they may lag behind the commit above.
|
|
3552
|
+
const terminal = await this.#renameRowsInBatches('workflow_status', 'workflow_uuid', oldName, newName, batchSize, adoptUnclaimedRows);
|
|
3553
|
+
const steps = await this.#renameRowsInBatches('operation_outputs', 'workflow_uuid', oldName, newName, batchSize, adoptUnclaimedRows);
|
|
3554
|
+
return { queues, schedules, versions, workflows: inFlight + terminal, steps };
|
|
3555
|
+
}
|
|
3172
3556
|
// ==================== Internal ====================
|
|
3173
3557
|
async insertWorkflowStatus(client, initStatus, ownerXid, incrementAttempts = false) {
|
|
3174
3558
|
try {
|
|
@@ -3203,8 +3587,9 @@ class SystemDatabase {
|
|
|
3203
3587
|
attributes,
|
|
3204
3588
|
schedule_name,
|
|
3205
3589
|
debounce_deadline_epoch_ms,
|
|
3206
|
-
is_debounced
|
|
3207
|
-
|
|
3590
|
+
is_debounced,
|
|
3591
|
+
application_name
|
|
3592
|
+
) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $26, $27, $28, $29, $30, $31, $32, $33)
|
|
3208
3593
|
ON CONFLICT (workflow_uuid)
|
|
3209
3594
|
DO UPDATE SET
|
|
3210
3595
|
recovery_attempts = CASE
|
|
@@ -3252,6 +3637,8 @@ class SystemDatabase {
|
|
|
3252
3637
|
initStatus.scheduleName ?? null,
|
|
3253
3638
|
initStatus.debounceDeadlineEpochMS ?? null,
|
|
3254
3639
|
initStatus.isDebounced ?? false,
|
|
3640
|
+
// Absent from the conflict update: a re-enqueue must not re-own a claimed row.
|
|
3641
|
+
initStatus.applicationName ?? null,
|
|
3255
3642
|
]);
|
|
3256
3643
|
if (rows.length === 0) {
|
|
3257
3644
|
throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
|
|
@@ -3334,12 +3721,13 @@ class SystemDatabase {
|
|
|
3334
3721
|
if (throwOnFailure && result.rowCount !== 1) {
|
|
3335
3722
|
throw new error_1.DBOSWorkflowConflictError(`Attempt to record transition of nonexistent workflow ${workflowID}`);
|
|
3336
3723
|
}
|
|
3724
|
+
return result.rowCount ?? 0;
|
|
3337
3725
|
}
|
|
3338
3726
|
async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
3339
3727
|
try {
|
|
3340
3728
|
const out = await client.query(`INSERT INTO ${this.schemaName}.operation_outputs
|
|
3341
|
-
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization)
|
|
3342
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
3729
|
+
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization, application_name)
|
|
3730
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
|
3343
3731
|
ON CONFLICT (workflow_uuid, function_id) DO UPDATE
|
|
3344
3732
|
SET completed_at_epoch_ms = operation_outputs.completed_at_epoch_ms
|
|
3345
3733
|
RETURNING completed_at_epoch_ms;`, [
|
|
@@ -3352,6 +3740,8 @@ class SystemDatabase {
|
|
|
3352
3740
|
startTimeEpochMs,
|
|
3353
3741
|
endTimeEpochMs,
|
|
3354
3742
|
options.serialization ?? null,
|
|
3743
|
+
// Mirrors the parent: only the running application records its steps.
|
|
3744
|
+
this.appName ?? null,
|
|
3355
3745
|
]);
|
|
3356
3746
|
if (checkConflict &&
|
|
3357
3747
|
(out?.rowCount ?? 0) > 0 &&
|
|
@@ -3589,9 +3979,15 @@ __decorate([
|
|
|
3589
3979
|
__metadata("design:returntype", Promise)
|
|
3590
3980
|
], SystemDatabase.prototype, "debounceDelayedWorkflowStandalone", null);
|
|
3591
3981
|
__decorate([
|
|
3592
|
-
dbRetry()
|
|
3982
|
+
dbRetry()
|
|
3983
|
+
// A missing row normally means the workflow has not been inserted yet, so
|
|
3984
|
+
// polling for it is correct. Callers that know the row must already exist
|
|
3985
|
+
// (e.g. a run parking on an outcome it just failed to write) pass
|
|
3986
|
+
// `failIfMissing` to fail fast with DBOSNonExistentWorkflowError instead of
|
|
3987
|
+
// polling forever.
|
|
3988
|
+
,
|
|
3593
3989
|
__metadata("design:type", Function),
|
|
3594
|
-
__metadata("design:paramtypes", [String, Number, String, Number, Number]),
|
|
3990
|
+
__metadata("design:paramtypes", [String, Number, String, Number, Number, Boolean]),
|
|
3595
3991
|
__metadata("design:returntype", Promise)
|
|
3596
3992
|
], SystemDatabase.prototype, "awaitWorkflowResult", null);
|
|
3597
3993
|
__decorate([
|
|
@@ -3687,7 +4083,13 @@ __decorate([
|
|
|
3687
4083
|
__decorate([
|
|
3688
4084
|
dbRetry(),
|
|
3689
4085
|
__metadata("design:type", Function),
|
|
3690
|
-
__metadata("design:paramtypes", [
|
|
4086
|
+
__metadata("design:paramtypes", [Number]),
|
|
4087
|
+
__metadata("design:returntype", Promise)
|
|
4088
|
+
], SystemDatabase.prototype, "listTimedOutWorkflowIds", null);
|
|
4089
|
+
__decorate([
|
|
4090
|
+
dbRetry(),
|
|
4091
|
+
__metadata("design:type", Function),
|
|
4092
|
+
__metadata("design:paramtypes", [String, String, Array]),
|
|
3691
4093
|
__metadata("design:returntype", Promise)
|
|
3692
4094
|
], SystemDatabase.prototype, "getMetrics", null);
|
|
3693
4095
|
//# sourceMappingURL=system_database.js.map
|