@dbos-inc/dbos-sdk 4.28.8-preview → 4.28.12-preview
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -7
- package/dist/schemas/system_db_schema.d.ts +0 -8
- package/dist/schemas/system_db_schema.d.ts.map +1 -1
- package/dist/schemas/system_db_schema.js.map +1 -1
- package/dist/src/adminserver.d.ts.map +1 -1
- package/dist/src/adminserver.js +5 -19
- package/dist/src/adminserver.js.map +1 -1
- package/dist/src/client.js +4 -4
- package/dist/src/client.js.map +1 -1
- package/dist/src/conductor/conductor.js +2 -2
- package/dist/src/conductor/conductor.js.map +1 -1
- package/dist/src/conductor/protocol.d.ts +1 -0
- package/dist/src/conductor/protocol.d.ts.map +1 -1
- package/dist/src/conductor/protocol.js.map +1 -1
- package/dist/src/dbos-executor.d.ts +5 -36
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js +0 -24
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts +1 -40
- package/dist/src/dbos.d.ts.map +1 -1
- package/dist/src/dbos.js +11 -89
- package/dist/src/dbos.js.map +1 -1
- package/dist/src/eventreceiver.d.ts +5 -11
- package/dist/src/eventreceiver.d.ts.map +1 -1
- package/dist/src/eventreceiver.js +11 -18
- package/dist/src/eventreceiver.js.map +1 -1
- package/dist/src/index.d.ts +2 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -5
- package/dist/src/index.js.map +1 -1
- package/dist/src/sysdb_migrations/internal/migrations.js +11 -0
- package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
- package/dist/src/system_database.d.ts +1 -4
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +45 -89
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/utils.d.ts +1 -0
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +4 -2
- package/dist/src/utils.js.map +1 -1
- package/dist/src/wfqueue.d.ts +41 -47
- package/dist/src/wfqueue.d.ts.map +1 -1
- package/dist/src/wfqueue.js +116 -229
- package/dist/src/wfqueue.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/dist/src/scheduler/scheduler_decorator.d.ts +0 -41
- package/dist/src/scheduler/scheduler_decorator.d.ts.map +0 -1
- package/dist/src/scheduler/scheduler_decorator.js +0 -174
- package/dist/src/scheduler/scheduler_decorator.js.map +0 -1
package/dist/src/wfqueue.js
CHANGED
|
@@ -1,20 +1,19 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.wfQueueRunner = exports.WorkflowQueue = exports.
|
|
3
|
+
exports.wfQueueRunner = exports.registerInternalQueue = exports.WorkflowQueue = exports.isPartitionedQueue = exports.logQueue = void 0;
|
|
4
4
|
const dbos_executor_1 = require("./dbos-executor");
|
|
5
|
-
const dbos_1 = require("./dbos");
|
|
6
5
|
const debugpoint_1 = require("./debugpoint");
|
|
7
6
|
const utils_1 = require("./utils");
|
|
8
7
|
/**
|
|
9
8
|
* Log a single queue's name and its set parameters. Unset parameters are
|
|
10
9
|
* omitted, matching `Queue: <name> (concurrency=…, worker_concurrency=…,
|
|
11
|
-
* limit=N/Ts,
|
|
10
|
+
* limit=N/Ts, partition_concurrency=…)`.
|
|
12
11
|
*/
|
|
13
12
|
function logQueue(logger, q) {
|
|
14
13
|
const opts = [];
|
|
15
14
|
if (q.concurrency !== undefined) {
|
|
16
15
|
// On a partitioned queue the queue-wide scope is worth naming explicitly.
|
|
17
|
-
opts.push(`${
|
|
16
|
+
opts.push(`${isPartitionedQueue(q) ? 'global_concurrency' : 'concurrency'}=${q.concurrency}`);
|
|
18
17
|
}
|
|
19
18
|
if (q.workerConcurrency !== undefined)
|
|
20
19
|
opts.push(`worker_concurrency=${q.workerConcurrency}`);
|
|
@@ -28,60 +27,38 @@ function logQueue(logger, q) {
|
|
|
28
27
|
if (q.partitionRateLimit !== undefined) {
|
|
29
28
|
opts.push(`partition_limit=${q.partitionRateLimit.limitPerPeriod}/${q.partitionRateLimit.periodSec}s`);
|
|
30
29
|
}
|
|
31
|
-
if (q.priorityEnabled)
|
|
32
|
-
opts.push('priority');
|
|
33
|
-
if (q.partitionQueue)
|
|
34
|
-
opts.push('partitioned');
|
|
35
30
|
const optsStr = opts.length > 0 ? ` (${opts.join(', ')})` : '';
|
|
36
31
|
logger.info(`Queue: ${q.name}${optsStr}`);
|
|
37
32
|
}
|
|
38
33
|
exports.logQueue = logQueue;
|
|
34
|
+
/**
|
|
35
|
+
* Options removed in 5.0, and what replaces each.
|
|
36
|
+
*/
|
|
37
|
+
const REMOVED_QUEUE_PARAMS = {
|
|
38
|
+
priorityEnabled: 'every queue dispatches in priority order, so the option can be deleted',
|
|
39
|
+
partitionQueue: 'set partitionConcurrency, partitionWorkerConcurrency, or partitionRateLimit instead, any of which partitions the queue',
|
|
40
|
+
};
|
|
39
41
|
/** True when any per-partition limit is set, which is what partitions a queue. */
|
|
40
42
|
function hasPartitionLimits(limits) {
|
|
41
43
|
return (limits.partitionConcurrency !== undefined ||
|
|
42
44
|
limits.partitionWorkerConcurrency !== undefined ||
|
|
43
45
|
limits.partitionRateLimit !== undefined);
|
|
44
46
|
}
|
|
45
|
-
/**
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
*/
|
|
49
|
-
function isLegacyPartitioned(q) {
|
|
50
|
-
return q.partitionQueue && !hasPartitionLimits(q);
|
|
47
|
+
/** True when a queue is partitioned, which is to say any per-partition limit is set on it. */
|
|
48
|
+
function isPartitionedQueue(queue) {
|
|
49
|
+
return hasPartitionLimits(queue);
|
|
51
50
|
}
|
|
52
|
-
|
|
53
|
-
function resolveQueueLimits(q) {
|
|
54
|
-
if (isLegacyPartitioned(q)) {
|
|
55
|
-
return {
|
|
56
|
-
partitionConcurrency: q.concurrency,
|
|
57
|
-
partitionWorkerConcurrency: q.workerConcurrency,
|
|
58
|
-
partitionRateLimit: q.rateLimit,
|
|
59
|
-
};
|
|
60
|
-
}
|
|
61
|
-
return {
|
|
62
|
-
globalConcurrency: q.concurrency,
|
|
63
|
-
workerConcurrency: q.workerConcurrency,
|
|
64
|
-
rateLimit: q.rateLimit,
|
|
65
|
-
partitionConcurrency: q.partitionConcurrency,
|
|
66
|
-
partitionWorkerConcurrency: q.partitionWorkerConcurrency,
|
|
67
|
-
partitionRateLimit: q.partitionRateLimit,
|
|
68
|
-
};
|
|
69
|
-
}
|
|
70
|
-
exports.resolveQueueLimits = resolveQueueLimits;
|
|
51
|
+
exports.isPartitionedQueue = isPartitionedQueue;
|
|
71
52
|
/**
|
|
72
53
|
* Room left under this worker's queue-wide concurrency limit, given how many of
|
|
73
54
|
* its workflows are already running or claimed.
|
|
74
55
|
*/
|
|
75
|
-
function workerBudget(
|
|
76
|
-
if (
|
|
77
|
-
//
|
|
78
|
-
return 0;
|
|
79
|
-
}
|
|
80
|
-
if (limits.workerConcurrency === undefined) {
|
|
81
|
-
// A non-zero per-partition worker limit is enforced per partition instead.
|
|
56
|
+
function workerBudget(queue, running) {
|
|
57
|
+
if (queue.workerConcurrency === undefined) {
|
|
58
|
+
// A per-partition worker limit is enforced per partition instead.
|
|
82
59
|
return Infinity;
|
|
83
60
|
}
|
|
84
|
-
return Math.max(0,
|
|
61
|
+
return Math.max(0, queue.workerConcurrency - running);
|
|
85
62
|
}
|
|
86
63
|
/** 40001 serialization_failure or 55P03 lock_not_available: a peer is claiming the same rows. */
|
|
87
64
|
function isContentionError(e) {
|
|
@@ -120,14 +97,6 @@ function sysDBFor(q) {
|
|
|
120
97
|
}
|
|
121
98
|
return exec.systemDatabase;
|
|
122
99
|
}
|
|
123
|
-
/** Reject a write that would silently re-scope the other limits on a legacy queue. */
|
|
124
|
-
function requireNotLegacyPartitioned(q, field) {
|
|
125
|
-
if (isLegacyPartitioned(q)) {
|
|
126
|
-
throw new Error(`Cannot set ${field} on queue ${q.name}: it is registered with the deprecated partitionQueue option, ` +
|
|
127
|
-
`under which concurrency, workerConcurrency, and rateLimit apply per partition. ` +
|
|
128
|
-
`Re-register the queue with the partition limits instead.`);
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
100
|
/** Validate a new queue-wide concurrency against the queue's other cached limits. */
|
|
132
101
|
function checkConcurrencyBounds(q, value) {
|
|
133
102
|
if (value === undefined)
|
|
@@ -159,18 +128,15 @@ function applyRecord(q, record) {
|
|
|
159
128
|
q.concurrency = record.concurrency ?? undefined;
|
|
160
129
|
q.workerConcurrency = record.workerConcurrency ?? undefined;
|
|
161
130
|
q.rateLimit = rateLimitFromRecord(record.rateLimitMax, record.rateLimitPeriodSec);
|
|
162
|
-
q.priorityEnabled = record.priorityEnabled;
|
|
163
131
|
q.partitionConcurrency = record.partitionConcurrency ?? undefined;
|
|
164
132
|
q.partitionWorkerConcurrency = record.partitionWorkerConcurrency ?? undefined;
|
|
165
133
|
q.partitionRateLimit = rateLimitFromRecord(record.partitionRateLimitMax, record.partitionRateLimitPeriodSec);
|
|
166
|
-
// Partitioning is inferred from the limits, so a row whose flag disagrees with them heals on read.
|
|
167
|
-
q.partitionQueue = record.partitionQueue || hasPartitionLimits(q);
|
|
168
134
|
q.minPollingIntervalMs = record.pollingIntervalSec * 1000;
|
|
169
135
|
q.applicationName = record.applicationName;
|
|
170
136
|
}
|
|
171
137
|
/**
|
|
172
138
|
* Re-read the queue's row from the database and update the cached fields on
|
|
173
|
-
* `q` in place. No-op for
|
|
139
|
+
* `q` in place. No-op for internal queues. Throws if the row has been
|
|
174
140
|
* deleted.
|
|
175
141
|
*/
|
|
176
142
|
async function refreshFromDb(q) {
|
|
@@ -193,73 +159,69 @@ class WorkflowQueue {
|
|
|
193
159
|
/**
|
|
194
160
|
* Last-known cached values. May be stale for database-backed queues if
|
|
195
161
|
* another process has modified the row. Use getters instead.
|
|
162
|
+
* `concurrency` is the queue-wide limit, across every worker and partition.
|
|
196
163
|
*/
|
|
197
164
|
concurrency;
|
|
198
165
|
rateLimit;
|
|
199
166
|
workerConcurrency;
|
|
200
|
-
priorityEnabled = false;
|
|
201
|
-
partitionQueue = false;
|
|
202
167
|
partitionConcurrency;
|
|
203
168
|
partitionWorkerConcurrency;
|
|
204
169
|
partitionRateLimit;
|
|
205
170
|
minPollingIntervalMs;
|
|
206
|
-
/** Owner from the queues table; undefined for
|
|
171
|
+
/** Owner from the queues table; undefined for internal and unclaimed queues. */
|
|
207
172
|
applicationName;
|
|
208
173
|
/**
|
|
209
174
|
* When true, this queue's configuration is persisted in the `queues` system
|
|
210
|
-
* table and may be mutated at runtime via the `setX` methods.
|
|
211
|
-
* the
|
|
212
|
-
* process memory.
|
|
175
|
+
* table and may be mutated at runtime via the `setX` methods. False only for
|
|
176
|
+
* the process-local queues DBOS registers for its own use.
|
|
213
177
|
*/
|
|
214
|
-
databaseBacked
|
|
178
|
+
databaseBacked;
|
|
215
179
|
/**
|
|
216
180
|
* True when configuration reads/writes target a `DBOSClient`-supplied
|
|
217
181
|
* SystemDatabase rather than the global executor's. The actual handle is
|
|
218
182
|
* kept off this class's public type — see the module-level WeakMap below —
|
|
219
183
|
* so that `WorkflowQueue` does not transitively depend on `SystemDatabase`.
|
|
220
184
|
*/
|
|
221
|
-
clientBound
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
185
|
+
clientBound;
|
|
186
|
+
/**
|
|
187
|
+
* Not reachable from the published package, which exports `WorkflowQueue` as a
|
|
188
|
+
* type only: applications obtain queues from `DBOS.registerQueue`,
|
|
189
|
+
* `DBOS.retrieveQueue`, or `DBOS.listQueues`.
|
|
190
|
+
*
|
|
191
|
+
* @param record - The queue's configuration, persisted or not.
|
|
192
|
+
* @param databaseBacked - False only for the process-local queues DBOS registers for its own use.
|
|
193
|
+
* @param clientSystemDatabase - When set, config reads and writes target this client's database.
|
|
194
|
+
*/
|
|
195
|
+
constructor(record, databaseBacked = true, clientSystemDatabase) {
|
|
196
|
+
this.name = record.name;
|
|
197
|
+
this.databaseBacked = databaseBacked;
|
|
198
|
+
this.clientBound = clientSystemDatabase !== undefined;
|
|
199
|
+
applyRecord(this, record);
|
|
200
|
+
if (clientSystemDatabase !== undefined) {
|
|
201
|
+
clientSystemDatabases.set(this, clientSystemDatabase);
|
|
231
202
|
}
|
|
232
|
-
|
|
233
|
-
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Throws if a user-supplied queue registration is invalid. Internal queues bypass the name
|
|
206
|
+
* check: the reserved prefix is theirs, and they validate their parameters directly.
|
|
207
|
+
*/
|
|
208
|
+
static validateQueueRegistration(name, params) {
|
|
209
|
+
if (name.startsWith(utils_1.RESERVED_QUEUE_NAME_PREFIX)) {
|
|
210
|
+
throw new Error(`Queue name ${name} is reserved: names starting with '${utils_1.RESERVED_QUEUE_NAME_PREFIX}' belong to DBOS's internal queues.`);
|
|
234
211
|
}
|
|
235
212
|
WorkflowQueue.validateQueueParams(params);
|
|
236
|
-
this.concurrency = params.globalConcurrency ?? params.concurrency;
|
|
237
|
-
this.rateLimit = params.rateLimit;
|
|
238
|
-
this.workerConcurrency = params.workerConcurrency;
|
|
239
|
-
this.priorityEnabled = params.priorityEnabled ?? false;
|
|
240
|
-
this.partitionConcurrency = params.partitionConcurrency;
|
|
241
|
-
this.partitionWorkerConcurrency = params.partitionWorkerConcurrency;
|
|
242
|
-
this.partitionRateLimit = params.partitionRateLimit;
|
|
243
|
-
// Partitioning is inferred from any per-partition limit; the deprecated flag tracks it.
|
|
244
|
-
this.partitionQueue = (params.partitionQueue ?? false) || hasPartitionLimits(params);
|
|
245
|
-
this.minPollingIntervalMs = params.minPollingIntervalMs;
|
|
246
|
-
if (exports.wfQueueRunner.wfQueuesByName.has(name)) {
|
|
247
|
-
throw Error(`Workflow Queue '${name}' defined multiple times`);
|
|
248
|
-
}
|
|
249
|
-
exports.wfQueueRunner.wfQueuesByName.set(name, this);
|
|
250
213
|
}
|
|
251
214
|
/** Throws if any combination of queue parameters is invalid. */
|
|
252
215
|
static validateQueueParams(params) {
|
|
253
|
-
const
|
|
216
|
+
for (const [option, replacement] of Object.entries(REMOVED_QUEUE_PARAMS)) {
|
|
217
|
+
if (Object.hasOwn(params, option)) {
|
|
218
|
+
throw new Error(`${option} was removed: ${replacement}.`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
const { concurrency, globalConcurrency, workerConcurrency, rateLimit, partitionConcurrency, partitionWorkerConcurrency, partitionRateLimit, minPollingIntervalMs, } = params;
|
|
254
222
|
if (concurrency !== undefined && globalConcurrency !== undefined) {
|
|
255
223
|
throw new Error('concurrency is deprecated in favor of globalConcurrency; set only one of them');
|
|
256
224
|
}
|
|
257
|
-
if (partitionQueue && hasPartitionLimits(params)) {
|
|
258
|
-
throw new Error('partitionQueue is deprecated in favor of the partition limits; set only one of them');
|
|
259
|
-
}
|
|
260
|
-
if (partitionQueue && globalConcurrency !== undefined) {
|
|
261
|
-
throw new Error('partitionQueue applies every limit per partition, so it cannot be combined with globalConcurrency; use partitionConcurrency instead');
|
|
262
|
-
}
|
|
263
225
|
if (partitionConcurrency !== undefined && partitionConcurrency < 1) {
|
|
264
226
|
throw new Error('partitionConcurrency must be at least 1');
|
|
265
227
|
}
|
|
@@ -280,7 +242,6 @@ class WorkflowQueue {
|
|
|
280
242
|
partitionWorkerConcurrency > workerConcurrency) {
|
|
281
243
|
throw new Error('workerConcurrency must be greater than or equal to partitionWorkerConcurrency');
|
|
282
244
|
}
|
|
283
|
-
// Under the deprecated partitionQueue mode concurrency is itself a per-partition limit, so these compare like with like.
|
|
284
245
|
const queueConcurrency = globalConcurrency ?? concurrency;
|
|
285
246
|
if (workerConcurrency !== undefined && queueConcurrency !== undefined && workerConcurrency > queueConcurrency) {
|
|
286
247
|
throw new Error('concurrency must be greater than or equal to workerConcurrency');
|
|
@@ -310,9 +271,7 @@ class WorkflowQueue {
|
|
|
310
271
|
workerConcurrency: params.workerConcurrency ?? null,
|
|
311
272
|
rateLimitMax: params.rateLimit ? params.rateLimit.limitPerPeriod : null,
|
|
312
273
|
rateLimitPeriodSec: params.rateLimit ? params.rateLimit.periodSec : null,
|
|
313
|
-
|
|
314
|
-
// Any per-partition limit implies partitioning, whichever mode was used.
|
|
315
|
-
partitionQueue: (params.partitionQueue ?? false) || hasPartitionLimits(params),
|
|
274
|
+
partitionQueue: hasPartitionLimits(params),
|
|
316
275
|
partitionConcurrency: params.partitionConcurrency ?? null,
|
|
317
276
|
partitionWorkerConcurrency: params.partitionWorkerConcurrency ?? null,
|
|
318
277
|
partitionRateLimitMax: params.partitionRateLimit ? params.partitionRateLimit.limitPerPeriod : null,
|
|
@@ -320,38 +279,14 @@ class WorkflowQueue {
|
|
|
320
279
|
pollingIntervalSec: (params.minPollingIntervalMs ?? 1000) / 1000,
|
|
321
280
|
};
|
|
322
281
|
}
|
|
323
|
-
/**
|
|
324
|
-
* Construct a database-backed queue from a persisted record. Bypasses the
|
|
325
|
-
* legacy constructor so the instance is not added to the global registry —
|
|
326
|
-
* the queues table is the source of truth.
|
|
327
|
-
* @internal
|
|
328
|
-
*/
|
|
329
|
-
static _fromRecord(record, clientSystemDatabase) {
|
|
330
|
-
// Allocate without invoking the constructor (which would auto-register
|
|
331
|
-
// in `wfQueuesByName`) and strip `readonly` so we can set the fields here.
|
|
332
|
-
const q = Object.create(WorkflowQueue.prototype);
|
|
333
|
-
q.name = record.name;
|
|
334
|
-
q.databaseBacked = true;
|
|
335
|
-
q.clientBound = clientSystemDatabase !== undefined;
|
|
336
|
-
applyRecord(q, record);
|
|
337
|
-
if (clientSystemDatabase !== undefined) {
|
|
338
|
-
clientSystemDatabases.set(q, clientSystemDatabase);
|
|
339
|
-
}
|
|
340
|
-
return q;
|
|
341
|
-
}
|
|
342
282
|
/** @deprecated Use `setGlobalConcurrency`. */
|
|
343
283
|
async setConcurrency(value) {
|
|
344
|
-
|
|
345
|
-
// Refresh so the cross-field checks see the limits currently stored in the database.
|
|
346
|
-
await refreshFromDb(this);
|
|
347
|
-
checkConcurrencyBounds(this, value);
|
|
348
|
-
await sysDBFor(this).updateQueue(this.name, { concurrency: value ?? null });
|
|
349
|
-
this.concurrency = value;
|
|
284
|
+
return this.setGlobalConcurrency(value);
|
|
350
285
|
}
|
|
351
286
|
async setGlobalConcurrency(value) {
|
|
352
287
|
requireDatabaseBacked(this);
|
|
288
|
+
// Refresh so the cross-field checks see the limits currently stored in the database.
|
|
353
289
|
await refreshFromDb(this);
|
|
354
|
-
requireNotLegacyPartitioned(this, 'globalConcurrency');
|
|
355
290
|
checkConcurrencyBounds(this, value);
|
|
356
291
|
await sysDBFor(this).updateQueue(this.name, { concurrency: value ?? null });
|
|
357
292
|
this.concurrency = value;
|
|
@@ -359,7 +294,6 @@ class WorkflowQueue {
|
|
|
359
294
|
async setWorkerConcurrency(value) {
|
|
360
295
|
requireDatabaseBacked(this);
|
|
361
296
|
await refreshFromDb(this);
|
|
362
|
-
requireNotLegacyPartitioned(this, 'workerConcurrency');
|
|
363
297
|
if (value !== undefined) {
|
|
364
298
|
if (this.concurrency !== undefined && value > this.concurrency) {
|
|
365
299
|
throw new Error('workerConcurrency must be less than or equal to concurrency');
|
|
@@ -376,9 +310,6 @@ class WorkflowQueue {
|
|
|
376
310
|
if (value !== undefined && (value.limitPerPeriod === undefined || value.periodSec === undefined)) {
|
|
377
311
|
throw new Error('rateLimit must specify both limitPerPeriod and periodSec');
|
|
378
312
|
}
|
|
379
|
-
// Refresh so the check below sees the partition limits currently stored in the database.
|
|
380
|
-
await refreshFromDb(this);
|
|
381
|
-
requireNotLegacyPartitioned(this, 'rateLimit');
|
|
382
313
|
await sysDBFor(this).updateQueue(this.name, {
|
|
383
314
|
rateLimitMax: value ? value.limitPerPeriod : null,
|
|
384
315
|
rateLimitPeriodSec: value ? value.periodSec : null,
|
|
@@ -391,7 +322,6 @@ class WorkflowQueue {
|
|
|
391
322
|
throw new Error('partitionConcurrency must be at least 1');
|
|
392
323
|
}
|
|
393
324
|
await refreshFromDb(this);
|
|
394
|
-
requireNotLegacyPartitioned(this, 'partitionConcurrency');
|
|
395
325
|
if (value !== undefined) {
|
|
396
326
|
if (this.concurrency !== undefined && value > this.concurrency) {
|
|
397
327
|
throw new Error('partitionConcurrency must be less than or equal to globalConcurrency');
|
|
@@ -400,14 +330,13 @@ class WorkflowQueue {
|
|
|
400
330
|
throw new Error('partitionConcurrency must be greater than or equal to partitionWorkerConcurrency');
|
|
401
331
|
}
|
|
402
332
|
}
|
|
403
|
-
// Partitioning is inferred from the limits, so the
|
|
333
|
+
// Partitioning is inferred from the limits, so the stored flag follows them.
|
|
404
334
|
const partitioned = partitionedAfter(this, { partitionConcurrency: value });
|
|
405
335
|
await sysDBFor(this).updateQueue(this.name, {
|
|
406
336
|
partitionConcurrency: value ?? null,
|
|
407
337
|
partitionQueue: partitioned,
|
|
408
338
|
});
|
|
409
339
|
this.partitionConcurrency = value;
|
|
410
|
-
this.partitionQueue = partitioned;
|
|
411
340
|
}
|
|
412
341
|
async setPartitionWorkerConcurrency(value) {
|
|
413
342
|
requireDatabaseBacked(this);
|
|
@@ -415,7 +344,6 @@ class WorkflowQueue {
|
|
|
415
344
|
throw new Error('partitionWorkerConcurrency must be at least 1');
|
|
416
345
|
}
|
|
417
346
|
await refreshFromDb(this);
|
|
418
|
-
requireNotLegacyPartitioned(this, 'partitionWorkerConcurrency');
|
|
419
347
|
if (value !== undefined) {
|
|
420
348
|
if (this.partitionConcurrency !== undefined && value > this.partitionConcurrency) {
|
|
421
349
|
throw new Error('partitionWorkerConcurrency must be less than or equal to partitionConcurrency');
|
|
@@ -433,7 +361,6 @@ class WorkflowQueue {
|
|
|
433
361
|
partitionQueue: partitioned,
|
|
434
362
|
});
|
|
435
363
|
this.partitionWorkerConcurrency = value;
|
|
436
|
-
this.partitionQueue = partitioned;
|
|
437
364
|
}
|
|
438
365
|
async setPartitionRateLimit(value) {
|
|
439
366
|
requireDatabaseBacked(this);
|
|
@@ -441,7 +368,6 @@ class WorkflowQueue {
|
|
|
441
368
|
throw new Error('partitionRateLimit must specify both limitPerPeriod and periodSec');
|
|
442
369
|
}
|
|
443
370
|
await refreshFromDb(this);
|
|
444
|
-
requireNotLegacyPartitioned(this, 'partitionRateLimit');
|
|
445
371
|
const partitioned = partitionedAfter(this, { partitionRateLimit: value });
|
|
446
372
|
await sysDBFor(this).updateQueue(this.name, {
|
|
447
373
|
partitionRateLimitMax: value ? value.limitPerPeriod : null,
|
|
@@ -449,24 +375,6 @@ class WorkflowQueue {
|
|
|
449
375
|
partitionQueue: partitioned,
|
|
450
376
|
});
|
|
451
377
|
this.partitionRateLimit = value;
|
|
452
|
-
this.partitionQueue = partitioned;
|
|
453
|
-
}
|
|
454
|
-
/** @deprecated Priority is always enabled. */
|
|
455
|
-
async setPriorityEnabled(value) {
|
|
456
|
-
requireDatabaseBacked(this);
|
|
457
|
-
await sysDBFor(this).updateQueue(this.name, { priorityEnabled: value });
|
|
458
|
-
this.priorityEnabled = value;
|
|
459
|
-
}
|
|
460
|
-
/** @deprecated Use the partition limit setters. */
|
|
461
|
-
async setPartitionQueue(value) {
|
|
462
|
-
requireDatabaseBacked(this);
|
|
463
|
-
// Refresh so the check below sees the partition limits currently stored in the database.
|
|
464
|
-
await refreshFromDb(this);
|
|
465
|
-
if (hasPartitionLimits(this)) {
|
|
466
|
-
throw new Error(`Cannot set partitionQueue on queue ${this.name}: it is partitioned by its partition limits. Clear those instead.`);
|
|
467
|
-
}
|
|
468
|
-
await sysDBFor(this).updateQueue(this.name, { partitionQueue: value });
|
|
469
|
-
this.partitionQueue = value;
|
|
470
378
|
}
|
|
471
379
|
async setMinPollingIntervalMs(value) {
|
|
472
380
|
requireDatabaseBacked(this);
|
|
@@ -478,42 +386,31 @@ class WorkflowQueue {
|
|
|
478
386
|
}
|
|
479
387
|
/** @deprecated Use `getGlobalConcurrency`. */
|
|
480
388
|
async getConcurrency() {
|
|
481
|
-
|
|
482
|
-
return this.concurrency;
|
|
389
|
+
return this.getGlobalConcurrency();
|
|
483
390
|
}
|
|
484
391
|
async getGlobalConcurrency() {
|
|
485
392
|
await refreshFromDb(this);
|
|
486
|
-
return
|
|
393
|
+
return this.concurrency;
|
|
487
394
|
}
|
|
488
395
|
async getWorkerConcurrency() {
|
|
489
396
|
await refreshFromDb(this);
|
|
490
|
-
return
|
|
397
|
+
return this.workerConcurrency;
|
|
491
398
|
}
|
|
492
399
|
async getRateLimit() {
|
|
493
400
|
await refreshFromDb(this);
|
|
494
|
-
return
|
|
401
|
+
return this.rateLimit;
|
|
495
402
|
}
|
|
496
403
|
async getPartitionConcurrency() {
|
|
497
404
|
await refreshFromDb(this);
|
|
498
|
-
return
|
|
405
|
+
return this.partitionConcurrency;
|
|
499
406
|
}
|
|
500
407
|
async getPartitionWorkerConcurrency() {
|
|
501
408
|
await refreshFromDb(this);
|
|
502
|
-
return
|
|
409
|
+
return this.partitionWorkerConcurrency;
|
|
503
410
|
}
|
|
504
411
|
async getPartitionRateLimit() {
|
|
505
412
|
await refreshFromDb(this);
|
|
506
|
-
return
|
|
507
|
-
}
|
|
508
|
-
/** @deprecated Priority is always enabled. */
|
|
509
|
-
async getPriorityEnabled() {
|
|
510
|
-
await refreshFromDb(this);
|
|
511
|
-
return this.priorityEnabled;
|
|
512
|
-
}
|
|
513
|
-
/** @deprecated Use the partition limit getters. */
|
|
514
|
-
async getPartitionQueue() {
|
|
515
|
-
await refreshFromDb(this);
|
|
516
|
-
return this.partitionQueue;
|
|
413
|
+
return this.partitionRateLimit;
|
|
517
414
|
}
|
|
518
415
|
async getMinPollingIntervalMs() {
|
|
519
416
|
await refreshFromDb(this);
|
|
@@ -521,8 +418,35 @@ class WorkflowQueue {
|
|
|
521
418
|
}
|
|
522
419
|
}
|
|
523
420
|
exports.WorkflowQueue = WorkflowQueue;
|
|
421
|
+
/**
|
|
422
|
+
* Register a queue that DBOS itself needs — the internal queue, a Kafka receiver's
|
|
423
|
+
* queue — or return the one already registered under `name`.
|
|
424
|
+
*
|
|
425
|
+
* Internal queues are not persisted in the `queues` table: their configuration is
|
|
426
|
+
* fixed here and lives only in this process's memory. They are always dispatched,
|
|
427
|
+
* bypassing any `listenQueues` filter, since this process is the only one that
|
|
428
|
+
* enqueues onto them.
|
|
429
|
+
*
|
|
430
|
+
* Callers must resolve their queues through this rather than caching them: a registry
|
|
431
|
+
* clear (`DBOS.shutdown({ deregister: true })`) drops the registration, and a cached
|
|
432
|
+
* queue would silently stop being dispatched, leaving its workflows ENQUEUED forever.
|
|
433
|
+
*
|
|
434
|
+
* Not a user-facing API: applications register queues with `DBOS.registerQueue`.
|
|
435
|
+
* @internal
|
|
436
|
+
*/
|
|
437
|
+
function registerInternalQueue(name, params = {}) {
|
|
438
|
+
const existing = exports.wfQueueRunner.getInternalQueue(name);
|
|
439
|
+
if (existing)
|
|
440
|
+
return existing;
|
|
441
|
+
WorkflowQueue.validateQueueParams(params);
|
|
442
|
+
const queue = new WorkflowQueue(WorkflowQueue.recordFromParams(name, params), false);
|
|
443
|
+
exports.wfQueueRunner.addInternalQueue(queue);
|
|
444
|
+
return queue;
|
|
445
|
+
}
|
|
446
|
+
exports.registerInternalQueue = registerInternalQueue;
|
|
524
447
|
class WFQueueRunner {
|
|
525
|
-
|
|
448
|
+
/** DBOS's own process-local queues, registered via `registerInternalQueue`. */
|
|
449
|
+
internalQueues = new Map();
|
|
526
450
|
/**
|
|
527
451
|
* Queues fed by this process's own pollers (e.g. a Kafka consumer). Always dispatched,
|
|
528
452
|
* regardless of any listenQueues filter, so this process executes what it enqueues.
|
|
@@ -533,8 +457,6 @@ class WFQueueRunner {
|
|
|
533
457
|
listenQueueNames = null;
|
|
534
458
|
/** Per-queue scheduling state, keyed by queue name. */
|
|
535
459
|
states = new Map();
|
|
536
|
-
/** Names already warned about colliding with an in-memory queue (warn-once). */
|
|
537
|
-
conflictWarned = new Set();
|
|
538
460
|
static defaultMinPollingIntervalMs = 1000;
|
|
539
461
|
static defaultMaxPollingIntervalMs = 120000;
|
|
540
462
|
static reconcileIntervalMs = 1000;
|
|
@@ -543,6 +465,12 @@ class WFQueueRunner {
|
|
|
543
465
|
scalebackFactor = 0.9;
|
|
544
466
|
jitterMin = 0.95;
|
|
545
467
|
jitterMax = 1.05;
|
|
468
|
+
addInternalQueue(queue) {
|
|
469
|
+
this.internalQueues.set(queue.name, queue);
|
|
470
|
+
}
|
|
471
|
+
getInternalQueue(name) {
|
|
472
|
+
return this.internalQueues.get(name);
|
|
473
|
+
}
|
|
546
474
|
stop() {
|
|
547
475
|
if (!this.isRunning)
|
|
548
476
|
return;
|
|
@@ -550,24 +478,17 @@ class WFQueueRunner {
|
|
|
550
478
|
this.abortController?.abort();
|
|
551
479
|
}
|
|
552
480
|
clearRegistrations() {
|
|
553
|
-
this.
|
|
481
|
+
this.internalQueues.clear();
|
|
554
482
|
this.pollerQueueNames.clear();
|
|
555
483
|
}
|
|
556
484
|
async dispatchLoop(exec, listenQueuesArg, maxConcurrentQueueDispatches = 3) {
|
|
557
485
|
this.isRunning = true;
|
|
558
486
|
this.states.clear();
|
|
559
|
-
this.
|
|
560
|
-
this.listenQueueNames = listenQueuesArg
|
|
561
|
-
? new Set(listenQueuesArg.map((entry) => (typeof entry === 'string' ? entry : entry.name)))
|
|
562
|
-
: null;
|
|
487
|
+
this.listenQueueNames = listenQueuesArg ? new Set(listenQueuesArg) : null;
|
|
563
488
|
this.abortController = new AbortController();
|
|
564
489
|
const startNow = Date.now();
|
|
565
|
-
//
|
|
566
|
-
const
|
|
567
|
-
if (internal)
|
|
568
|
-
this.ensureState(internal, startNow);
|
|
569
|
-
// Unmatched string entries are deferred to refreshDbQueues as DB-backed queues.
|
|
570
|
-
for (const q of this.resolveInMemoryQueues(listenQueuesArg)) {
|
|
490
|
+
// Internal queues are process-private and bypass the listenQueues filter.
|
|
491
|
+
for (const q of this.internalQueues.values()) {
|
|
571
492
|
this.ensureState(q, startNow);
|
|
572
493
|
}
|
|
573
494
|
// Add pre-launch DB-backed queues now so an immediate enqueue can't race the first reconcile.
|
|
@@ -577,31 +498,6 @@ class WFQueueRunner {
|
|
|
577
498
|
// One loop drives global maintenance; queue polls run in a bounded set of independent lanes.
|
|
578
499
|
await this.schedulerLoop(exec, startNow, maxConcurrentQueueDispatches);
|
|
579
500
|
}
|
|
580
|
-
/** Resolve the listenQueues argument to the set of in-memory queues to dispatch for. */
|
|
581
|
-
resolveInMemoryQueues(listenQueuesArg) {
|
|
582
|
-
if (listenQueuesArg === null) {
|
|
583
|
-
return Array.from(this.wfQueuesByName.values()).filter((q) => q.name !== utils_1.INTERNAL_QUEUE_NAME);
|
|
584
|
-
}
|
|
585
|
-
const result = [];
|
|
586
|
-
for (const entry of listenQueuesArg) {
|
|
587
|
-
if (typeof entry === 'string') {
|
|
588
|
-
const q = this.wfQueuesByName.get(entry);
|
|
589
|
-
if (q)
|
|
590
|
-
result.push(q);
|
|
591
|
-
}
|
|
592
|
-
else {
|
|
593
|
-
result.push(entry);
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
// Poller-fed queues are always dispatched: this process enqueues onto them, so under a
|
|
597
|
-
// listenQueues filter their workflows would otherwise sit ENQUEUED forever.
|
|
598
|
-
for (const name of this.pollerQueueNames) {
|
|
599
|
-
const q = this.wfQueuesByName.get(name);
|
|
600
|
-
if (q && !result.some((r) => r.name === name))
|
|
601
|
-
result.push(q);
|
|
602
|
-
}
|
|
603
|
-
return result;
|
|
604
|
-
}
|
|
605
501
|
/** Begin tracking a queue if it isn't already, scheduling its first poll one interval out. */
|
|
606
502
|
ensureState(queue, now) {
|
|
607
503
|
if (this.states.has(queue.name))
|
|
@@ -621,17 +517,9 @@ class WFQueueRunner {
|
|
|
621
517
|
}
|
|
622
518
|
const present = new Set();
|
|
623
519
|
for (const record of records) {
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
if (this.wfQueuesByName.has(record.name)) {
|
|
627
|
-
if (!this.conflictWarned.has(record.name)) {
|
|
628
|
-
this.conflictWarned.add(record.name);
|
|
629
|
-
exec.logger.warn(`Database-backed queue '${record.name}' has the same name as an in-memory queue. ` +
|
|
630
|
-
`The in-memory queue's configuration is being used; the database-backed queue is ignored. ` +
|
|
631
|
-
`Rename one of them to resolve the conflict.`);
|
|
632
|
-
}
|
|
520
|
+
// An internal queue owns its name outright; its process-local configuration wins.
|
|
521
|
+
if (this.internalQueues.has(record.name))
|
|
633
522
|
continue;
|
|
634
|
-
}
|
|
635
523
|
if (this.listenQueueNames !== null &&
|
|
636
524
|
!this.listenQueueNames.has(record.name) &&
|
|
637
525
|
!this.pollerQueueNames.has(record.name)) {
|
|
@@ -641,10 +529,10 @@ class WFQueueRunner {
|
|
|
641
529
|
const existing = this.states.get(record.name);
|
|
642
530
|
if (existing) {
|
|
643
531
|
// Refresh config in place, preserving this queue's polling/backoff state.
|
|
644
|
-
existing.queue = WorkflowQueue
|
|
532
|
+
existing.queue = new WorkflowQueue(record);
|
|
645
533
|
}
|
|
646
534
|
else {
|
|
647
|
-
this.ensureState(WorkflowQueue
|
|
535
|
+
this.ensureState(new WorkflowQueue(record), now);
|
|
648
536
|
}
|
|
649
537
|
}
|
|
650
538
|
// A database-backed queue whose row is gone stops being dispatched.
|
|
@@ -659,7 +547,7 @@ class WFQueueRunner {
|
|
|
659
547
|
}
|
|
660
548
|
/** Log every queue this process will dispatch for, once at startup after discovery. */
|
|
661
549
|
logRunningQueues(exec) {
|
|
662
|
-
const names = Array.from(this.states.keys()).filter((n) => n
|
|
550
|
+
const names = Array.from(this.states.keys()).filter((n) => !this.internalQueues.has(n));
|
|
663
551
|
exec.logger.info(`Listening to ${names.length} queues:`);
|
|
664
552
|
for (const name of names) {
|
|
665
553
|
logQueue(exec.logger, this.states.get(name).queue);
|
|
@@ -780,20 +668,19 @@ class WFQueueRunner {
|
|
|
780
668
|
}
|
|
781
669
|
await exec.dispatchDequeuedWorkflows(wfids);
|
|
782
670
|
};
|
|
783
|
-
const limits = resolveQueueLimits(queue);
|
|
784
671
|
const sysdb = exec.systemDatabase;
|
|
785
672
|
// Dequeue workflows for this queue, either in one batched sweep across partitions or one partition at a time.
|
|
786
673
|
try {
|
|
787
|
-
if (!queue
|
|
674
|
+
if (!isPartitionedQueue(queue)) {
|
|
788
675
|
const wfids = await sysdb.findAndMarkStartableWorkflows(queue, exec.executorID, utils_1.globalParams.appVersion, undefined, sysdb.countRunningWorkflowsForQueue(queue.name));
|
|
789
676
|
await dispatch(wfids);
|
|
790
677
|
}
|
|
791
|
-
else if (
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
678
|
+
else if (queue.partitionConcurrency === 1 &&
|
|
679
|
+
queue.concurrency === undefined &&
|
|
680
|
+
queue.rateLimit === undefined &&
|
|
681
|
+
queue.partitionRateLimit === undefined) {
|
|
795
682
|
// Batched path: one transaction claims every partition's head (see findAndMarkStartablePartitionedWorkflows).
|
|
796
|
-
const maxTasks = workerBudget(
|
|
683
|
+
const maxTasks = workerBudget(queue, sysdb.countRunningWorkflowsForQueue(queue.name));
|
|
797
684
|
if (maxTasks > 0) {
|
|
798
685
|
const wfids = await sysdb.findAndMarkStartablePartitionedWorkflows(queue, exec.executorID, utils_1.globalParams.appVersion, maxTasks);
|
|
799
686
|
await dispatch(wfids);
|
|
@@ -806,7 +693,7 @@ class WFQueueRunner {
|
|
|
806
693
|
const running = sysdb.countRunningWorkflowsForQueue(queue.name);
|
|
807
694
|
let claimed = 0;
|
|
808
695
|
for (const partitionKey of partitionKeys) {
|
|
809
|
-
if (workerBudget(
|
|
696
|
+
if (workerBudget(queue, running + claimed) <= 0)
|
|
810
697
|
break;
|
|
811
698
|
let partitionWfids;
|
|
812
699
|
try {
|