@dbos-inc/dbos-sdk 4.28.11-preview → 4.28.12-preview
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/conductor/conductor.js +2 -2
- package/dist/src/conductor/conductor.js.map +1 -1
- package/dist/src/conductor/protocol.d.ts +1 -0
- package/dist/src/conductor/protocol.d.ts.map +1 -1
- package/dist/src/conductor/protocol.js.map +1 -1
- package/dist/src/eventreceiver.d.ts +1 -1
- package/dist/src/eventreceiver.d.ts.map +1 -1
- package/dist/src/eventreceiver.js +2 -1
- package/dist/src/eventreceiver.js.map +1 -1
- package/dist/src/system_database.d.ts +1 -1
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +33 -36
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/wfqueue.d.ts +4 -26
- package/dist/src/wfqueue.d.ts.map +1 -1
- package/dist/src/wfqueue.js +43 -125
- package/dist/src/wfqueue.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
|
@@ -15,7 +15,6 @@ const pg_1 = require("pg");
|
|
|
15
15
|
const error_1 = require("./error");
|
|
16
16
|
const workflow_1 = require("./workflow");
|
|
17
17
|
const utils_1 = require("./utils");
|
|
18
|
-
const wfqueue_1 = require("./wfqueue");
|
|
19
18
|
const crypto_1 = require("crypto");
|
|
20
19
|
const utils_2 = require("./utils");
|
|
21
20
|
const database_utils_1 = require("./database_utils");
|
|
@@ -76,7 +75,6 @@ const QUEUE_COLUMN_BY_FIELD = {
|
|
|
76
75
|
workerConcurrency: 'worker_concurrency',
|
|
77
76
|
rateLimitMax: 'rate_limit_max',
|
|
78
77
|
rateLimitPeriodSec: 'rate_limit_period_sec',
|
|
79
|
-
priorityEnabled: 'priority_enabled',
|
|
80
78
|
partitionQueue: 'partition_queue',
|
|
81
79
|
partitionConcurrency: 'partition_concurrency',
|
|
82
80
|
partitionWorkerConcurrency: 'partition_worker_concurrency',
|
|
@@ -84,7 +82,7 @@ const QUEUE_COLUMN_BY_FIELD = {
|
|
|
84
82
|
partitionRateLimitPeriodSec: 'partition_rate_limit_period_sec',
|
|
85
83
|
pollingIntervalSec: 'polling_interval_sec',
|
|
86
84
|
};
|
|
87
|
-
const QUEUE_COLUMNS = 'name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
|
|
85
|
+
const QUEUE_COLUMNS = 'name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec, ' +
|
|
88
86
|
'partition_concurrency, partition_worker_concurrency, partition_rate_limit_max, partition_rate_limit_period_sec, ' +
|
|
89
87
|
'polling_interval_sec, application_name';
|
|
90
88
|
function queueRecordFromRow(row) {
|
|
@@ -94,8 +92,9 @@ function queueRecordFromRow(row) {
|
|
|
94
92
|
workerConcurrency: row.worker_concurrency,
|
|
95
93
|
rateLimitMax: row.rate_limit_max,
|
|
96
94
|
rateLimitPeriodSec: row.rate_limit_period_sec,
|
|
97
|
-
|
|
98
|
-
|
|
95
|
+
partitionQueue: row.partition_concurrency !== null ||
|
|
96
|
+
row.partition_worker_concurrency !== null ||
|
|
97
|
+
(row.partition_rate_limit_max !== null && row.partition_rate_limit_period_sec !== null),
|
|
99
98
|
partitionConcurrency: row.partition_concurrency,
|
|
100
99
|
partitionWorkerConcurrency: row.partition_worker_concurrency,
|
|
101
100
|
partitionRateLimitMax: row.partition_rate_limit_max,
|
|
@@ -2591,15 +2590,14 @@ class SystemDatabase {
|
|
|
2591
2590
|
}
|
|
2592
2591
|
async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey, localRunningCount = 0, partitionLocalRunningCount = 0) {
|
|
2593
2592
|
const claimedIDs = [];
|
|
2594
|
-
const limits = (0, wfqueue_1.resolveQueueLimits)(queue);
|
|
2595
2593
|
const partitionParams = queuePartitionKey !== undefined ? [queuePartitionKey] : [];
|
|
2596
2594
|
// Shares a concurrency or rate limit budget with other executors.
|
|
2597
|
-
const hasSharedBudget =
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2595
|
+
const hasSharedBudget = queue.concurrency !== undefined ||
|
|
2596
|
+
queue.partitionConcurrency !== undefined ||
|
|
2597
|
+
queue.rateLimit !== undefined ||
|
|
2598
|
+
queue.partitionRateLimit !== undefined;
|
|
2601
2599
|
// Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
|
|
2602
|
-
const hasWriteSkew = queuePartitionKey !== undefined && (
|
|
2600
|
+
const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
|
|
2603
2601
|
const client = await this.#connect();
|
|
2604
2602
|
try {
|
|
2605
2603
|
// Default to READ COMMITTED except with a budget shared across executors
|
|
@@ -2640,42 +2638,42 @@ class SystemDatabase {
|
|
|
2640
2638
|
};
|
|
2641
2639
|
// Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
|
|
2642
2640
|
let maxTasks = Infinity;
|
|
2643
|
-
if (
|
|
2641
|
+
if (queue.workerConcurrency !== undefined) {
|
|
2644
2642
|
// Use the in-memory registry for this worker's running count — avoids a DB round trip.
|
|
2645
|
-
maxTasks = Math.min(maxTasks, Math.max(0,
|
|
2643
|
+
maxTasks = Math.min(maxTasks, Math.max(0, queue.workerConcurrency - localRunningCount));
|
|
2646
2644
|
}
|
|
2647
|
-
if (
|
|
2648
|
-
maxTasks = Math.min(maxTasks, Math.max(0,
|
|
2645
|
+
if (queue.partitionWorkerConcurrency !== undefined) {
|
|
2646
|
+
maxTasks = Math.min(maxTasks, Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount));
|
|
2649
2647
|
}
|
|
2650
2648
|
if (maxTasks <= 0) {
|
|
2651
2649
|
await client.query('COMMIT');
|
|
2652
2650
|
return claimedIDs;
|
|
2653
2651
|
}
|
|
2654
|
-
if (
|
|
2652
|
+
if (queue.rateLimit !== undefined) {
|
|
2655
2653
|
// Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
|
|
2656
|
-
maxTasks = Math.min(maxTasks, await rateLimitRemaining(
|
|
2654
|
+
maxTasks = Math.min(maxTasks, await rateLimitRemaining(queue.rateLimit, false));
|
|
2657
2655
|
}
|
|
2658
|
-
if (
|
|
2659
|
-
maxTasks = Math.min(maxTasks, await rateLimitRemaining(
|
|
2656
|
+
if (queue.partitionRateLimit !== undefined) {
|
|
2657
|
+
maxTasks = Math.min(maxTasks, await rateLimitRemaining(queue.partitionRateLimit, true));
|
|
2660
2658
|
}
|
|
2661
2659
|
if (maxTasks <= 0) {
|
|
2662
2660
|
await client.query('COMMIT');
|
|
2663
2661
|
return claimedIDs;
|
|
2664
2662
|
}
|
|
2665
|
-
if (
|
|
2663
|
+
if (queue.concurrency !== undefined) {
|
|
2666
2664
|
// Global concurrency still requires a DB query since other workers may be running workflows too.
|
|
2667
2665
|
const totalRunningTasks = await pendingCount(false);
|
|
2668
|
-
if (totalRunningTasks >
|
|
2669
|
-
this.logger.warn(`Total running tasks (${totalRunningTasks}) exceeds the global concurrency limit (${
|
|
2666
|
+
if (totalRunningTasks > queue.concurrency) {
|
|
2667
|
+
this.logger.warn(`Total running tasks (${totalRunningTasks}) exceeds the global concurrency limit (${queue.concurrency})`);
|
|
2670
2668
|
}
|
|
2671
|
-
maxTasks = Math.min(maxTasks, Math.max(0,
|
|
2669
|
+
maxTasks = Math.min(maxTasks, Math.max(0, queue.concurrency - totalRunningTasks));
|
|
2672
2670
|
}
|
|
2673
|
-
if (
|
|
2671
|
+
if (queue.partitionConcurrency !== undefined) {
|
|
2674
2672
|
const partitionRunningTasks = await pendingCount(true);
|
|
2675
|
-
if (partitionRunningTasks >
|
|
2676
|
-
this.logger.warn(`Total running tasks (${partitionRunningTasks}) on partition ${queuePartitionKey} of queue ${queue.name} exceeds the partition concurrency limit (${
|
|
2673
|
+
if (partitionRunningTasks > queue.partitionConcurrency) {
|
|
2674
|
+
this.logger.warn(`Total running tasks (${partitionRunningTasks}) on partition ${queuePartitionKey} of queue ${queue.name} exceeds the partition concurrency limit (${queue.partitionConcurrency})`);
|
|
2677
2675
|
}
|
|
2678
|
-
maxTasks = Math.min(maxTasks, Math.max(0,
|
|
2676
|
+
maxTasks = Math.min(maxTasks, Math.max(0, queue.partitionConcurrency - partitionRunningTasks));
|
|
2679
2677
|
}
|
|
2680
2678
|
// Return immediately if there are no available tasks due to flow control limits
|
|
2681
2679
|
if (maxTasks <= 0) {
|
|
@@ -2718,7 +2716,7 @@ class SystemDatabase {
|
|
|
2718
2716
|
workflow_1.StatusString.PENDING,
|
|
2719
2717
|
executorID,
|
|
2720
2718
|
appVersion,
|
|
2721
|
-
|
|
2719
|
+
queue.rateLimit !== undefined || queue.partitionRateLimit !== undefined,
|
|
2722
2720
|
workflowIDs,
|
|
2723
2721
|
workflow_1.StatusString.ENQUEUED,
|
|
2724
2722
|
// Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
|
|
@@ -2765,14 +2763,13 @@ class SystemDatabase {
|
|
|
2765
2763
|
partitionedDequeueSweepCap = 8192;
|
|
2766
2764
|
/** Dequeue each partition's head-of-line workflow in one transaction, at most {@link partitionedDequeueSweepCap} per sweep; only valid for partition-concurrency-1 queues with no queue-wide limit. */
|
|
2767
2765
|
async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity) {
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
limits.partitionRateLimit !== undefined) {
|
|
2766
|
+
if (queue.partitionConcurrency !== 1 ||
|
|
2767
|
+
queue.concurrency !== undefined ||
|
|
2768
|
+
queue.rateLimit !== undefined ||
|
|
2769
|
+
queue.partitionRateLimit !== undefined) {
|
|
2773
2770
|
throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
|
|
2774
2771
|
}
|
|
2775
|
-
// partitionWorkerConcurrency needs no handling here:
|
|
2772
|
+
// partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
|
|
2776
2773
|
const client = await this.#connect();
|
|
2777
2774
|
try {
|
|
2778
2775
|
await client.query('BEGIN');
|
|
@@ -4061,14 +4058,14 @@ class SystemDatabase {
|
|
|
4061
4058
|
priority_enabled, partition_queue, partition_concurrency, partition_worker_concurrency,
|
|
4062
4059
|
partition_rate_limit_max, partition_rate_limit_period_sec,
|
|
4063
4060
|
polling_interval_sec, updated_at, application_name)
|
|
4064
|
-
|
|
4061
|
+
-- priority_enabled is vestigial: every queue dispatches in priority order.
|
|
4062
|
+
VALUES ($1, $2, $3, $4, $5, TRUE, $6, $7, $8, $9, $10, $11, $12, $13)
|
|
4065
4063
|
${onConflict}`, [
|
|
4066
4064
|
record.name,
|
|
4067
4065
|
record.concurrency,
|
|
4068
4066
|
record.workerConcurrency,
|
|
4069
4067
|
record.rateLimitMax,
|
|
4070
4068
|
record.rateLimitPeriodSec,
|
|
4071
|
-
record.priorityEnabled,
|
|
4072
4069
|
record.partitionQueue,
|
|
4073
4070
|
record.partitionConcurrency,
|
|
4074
4071
|
record.partitionWorkerConcurrency,
|