bunqueue 2.8.53 → 2.8.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/backgroundTasks.js +5 -1
- package/dist/application/cleanupTasks.js +1 -1
- package/dist/application/clientTracking.js +1 -1
- package/dist/application/contextFactory.js +1 -0
- package/dist/application/dlqManager.d.ts +7 -1
- package/dist/application/dlqManager.js +16 -5
- package/dist/application/lockManager.js +3 -3
- package/dist/application/operations/ack.js +2 -2
- package/dist/application/operations/ackHelpers.js +1 -1
- package/dist/application/operations/jobManagement.js +6 -4
- package/dist/application/operations/jobMoveOperations.js +3 -3
- package/dist/application/operations/jobStateTransitions.d.ts +1 -1
- package/dist/application/operations/jobStateTransitions.js +16 -2
- package/dist/application/queueManager.d.ts +22 -2
- package/dist/application/queueManager.js +48 -6
- package/dist/application/stallDetection.js +3 -3
- package/dist/client/flowJobDependencies.js +3 -1
- package/dist/client/flowJobMoveMethods.js +7 -2
- package/dist/client/jobConversion.d.ts +3 -3
- package/dist/client/jobConversion.js +10 -6
- package/dist/client/jobConversionHelpers.d.ts +1 -1
- package/dist/client/jobConversionHelpers.js +3 -6
- package/dist/client/jobConversionTypes.d.ts +4 -0
- package/dist/client/jobDeduplication.d.ts +6 -0
- package/dist/client/jobDeduplication.js +14 -0
- package/dist/client/queue/bullmqCompat.d.ts +18 -8
- package/dist/client/queue/bullmqCompat.js +39 -71
- package/dist/client/queue/deduplication.d.ts +2 -0
- package/dist/client/queue/deduplication.js +20 -11
- package/dist/client/queue/dlq.d.ts +11 -6
- package/dist/client/queue/dlq.js +47 -4
- package/dist/client/queue/dlqJobMethods.d.ts +9 -0
- package/dist/client/queue/dlqJobMethods.js +176 -0
- package/dist/client/queue/dlqOps.d.ts +2 -1
- package/dist/client/queue/dlqOps.js +2 -2
- package/dist/client/queue/jobMove.js +6 -1
- package/dist/client/queue/jobProxy.js +10 -5
- package/dist/client/queue/operations/add.js +17 -0
- package/dist/client/queue/operations/index.d.ts +1 -0
- package/dist/client/queue/operations/index.js +1 -0
- package/dist/client/queue/operations/management.d.ts +0 -4
- package/dist/client/queue/operations/management.js +33 -5
- package/dist/client/queue/operations/query.d.ts +1 -20
- package/dist/client/queue/operations/query.js +18 -53
- package/dist/client/queue/operations/queryStates.d.ts +13 -0
- package/dist/client/queue/operations/queryStates.js +32 -0
- package/dist/client/queue/operations/queryTcpPages.d.ts +14 -0
- package/dist/client/queue/operations/queryTcpPages.js +40 -0
- package/dist/client/queue/queue.d.ts +3 -0
- package/dist/client/queue/queue.js +23 -11
- package/dist/client/queue/rateLimit.d.ts +4 -4
- package/dist/client/queue/rateLimit.js +17 -8
- package/dist/client/queue/workers.d.ts +12 -0
- package/dist/client/queue/workers.js +20 -2
- package/dist/client/queueGroup.d.ts +11 -0
- package/dist/client/queueGroup.js +28 -1
- package/dist/client/sandboxed/worker.js +1 -1
- package/dist/client/worker/processor.js +1 -1
- package/dist/client/worker/processorHandlers.d.ts +1 -5
- package/dist/client/worker/processorHandlers.js +5 -7
- package/dist/client/worker/worker.d.ts +1 -1
- package/dist/client/worker/worker.js +37 -13
- package/dist/domain/queue/limiterManager.d.ts +11 -0
- package/dist/domain/queue/limiterManager.js +28 -0
- package/dist/domain/queue/shard.d.ts +10 -2
- package/dist/domain/queue/shard.js +23 -4
- package/dist/domain/queue/uniqueKeyManager.d.ts +3 -1
- package/dist/domain/queue/uniqueKeyManager.js +9 -1
- package/dist/domain/types/command.d.ts +34 -1
- package/dist/domain/types/queue.d.ts +5 -0
- package/dist/domain/types/queue.js +13 -0
- package/dist/infrastructure/persistence/sqlite.d.ts +4 -0
- package/dist/infrastructure/persistence/sqlite.js +16 -0
- package/dist/infrastructure/server/handlerRoutes.js +14 -1
- package/dist/infrastructure/server/handlers/dlq.d.ts +3 -0
- package/dist/infrastructure/server/handlers/dlq.js +18 -4
- package/dist/infrastructure/server/handlers/introspection.d.ts +19 -0
- package/dist/infrastructure/server/handlers/introspection.js +21 -0
- package/dist/infrastructure/server/handlers/query.js +1 -1
- package/package.json +1 -1
|
@@ -7,6 +7,7 @@ import { getSharedManager } from '../../manager';
|
|
|
7
7
|
import { toPublicJob } from '../../types';
|
|
8
8
|
import { jobId } from '../../../domain/types/job';
|
|
9
9
|
import { createJobProxy, createSimpleJob } from '../jobProxy';
|
|
10
|
+
import { removeJobDeduplicationKey } from '../../jobDeduplication';
|
|
10
11
|
/** Add a single job */
|
|
11
12
|
export async function add(ctx, jobName, data, opts = {}) {
|
|
12
13
|
const merged = { ...ctx.opts.defaultJobOptions, ...opts };
|
|
@@ -82,6 +83,12 @@ export async function add(ctx, jobName, data, opts = {}) {
|
|
|
82
83
|
return toPublicJob({
|
|
83
84
|
job,
|
|
84
85
|
name: jobName,
|
|
86
|
+
updateProgress: async (jid, progress, message) => {
|
|
87
|
+
await manager.updateProgress(jobId(jid), progress, message);
|
|
88
|
+
},
|
|
89
|
+
log: async (jid, message) => {
|
|
90
|
+
manager.addLog(jobId(jid), message);
|
|
91
|
+
},
|
|
85
92
|
getState: (jid) => ctx.getJobState(jid),
|
|
86
93
|
remove: (jid) => ctx.removeAsync(jid),
|
|
87
94
|
retry: (jid) => ctx.retryJob(jid),
|
|
@@ -100,6 +107,16 @@ export async function add(ctx, jobName, data, opts = {}) {
|
|
|
100
107
|
moveToDelayed: (jid, timestamp, token) => ctx.moveJobToDelayed(jid, timestamp, token),
|
|
101
108
|
moveToWaitingChildren: (jid, token, o) => ctx.moveJobToWaitingChildren(jid, token, o),
|
|
102
109
|
waitUntilFinished: (jid, queueEvents, ttl) => ctx.waitJobUntilFinished(jid, queueEvents, ttl),
|
|
110
|
+
removeDeduplicationKey: (jid) => removeJobDeduplicationKey(jid, true, null),
|
|
111
|
+
discard: (jid) => {
|
|
112
|
+
void manager.discard(jobId(jid));
|
|
113
|
+
},
|
|
114
|
+
getFailedChildrenValues: (jid) => manager.getFailedChildrenValues(jobId(jid)),
|
|
115
|
+
getIgnoredChildrenFailures: (jid) => manager.getIgnoredChildrenFailures(jobId(jid)),
|
|
116
|
+
removeChildDependency: (jid) => manager.removeChildDependency(jobId(jid)),
|
|
117
|
+
removeUnprocessedChildren: async (jid) => {
|
|
118
|
+
await manager.removeUnprocessedChildren(jobId(jid));
|
|
119
|
+
},
|
|
103
120
|
});
|
|
104
121
|
}
|
|
105
122
|
// TCP mode — forward the full option set so nothing is silently dropped (#88)
|
|
@@ -54,15 +54,43 @@ export async function retryJob(ctx, id) {
|
|
|
54
54
|
}
|
|
55
55
|
/** Retry jobs matching criteria */
|
|
56
56
|
export async function retryJobs(ctx, opts) {
|
|
57
|
+
const state = opts?.state ?? 'failed';
|
|
57
58
|
if (ctx.embedded) {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
const manager = getSharedManager();
|
|
60
|
+
if (state === 'completed') {
|
|
61
|
+
manager.retryCompleted(ctx.name, undefined, {
|
|
62
|
+
limit: opts?.count,
|
|
63
|
+
timestamp: opts?.timestamp,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
else if (opts?.timestamp !== undefined) {
|
|
67
|
+
manager.retryDlqByFilter(ctx.name, {
|
|
68
|
+
olderThan: opts.timestamp,
|
|
69
|
+
limit: opts.count,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
manager.retryDlq(ctx.name, undefined, opts?.count);
|
|
61
74
|
}
|
|
62
75
|
return;
|
|
63
76
|
}
|
|
64
|
-
if (
|
|
65
|
-
await ctx.tcp.send({
|
|
77
|
+
if (state === 'completed') {
|
|
78
|
+
await ctx.tcp.send({
|
|
79
|
+
cmd: 'RetryCompleted',
|
|
80
|
+
queue: ctx.name,
|
|
81
|
+
count: opts?.count,
|
|
82
|
+
timestamp: opts?.timestamp,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
await ctx.tcp.send({
|
|
87
|
+
cmd: 'RetryDlq',
|
|
88
|
+
queue: ctx.name,
|
|
89
|
+
count: opts?.count,
|
|
90
|
+
filter: opts?.timestamp === undefined
|
|
91
|
+
? undefined
|
|
92
|
+
: { olderThan: opts.timestamp, limit: opts.count },
|
|
93
|
+
});
|
|
66
94
|
}
|
|
67
95
|
}
|
|
68
96
|
// ============ Clean Operations ============
|
|
@@ -1,10 +1,6 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Queue Query Operations
|
|
3
|
-
* getJob, getJobs, getJobState, getJobsAsync
|
|
4
|
-
*/
|
|
5
1
|
import type { TcpConnectionPool } from '../../tcpPool';
|
|
6
2
|
import type { Job, JobStateType, GetDependenciesOpts, JobDependencies, JobDependenciesCount } from '../../types';
|
|
7
|
-
interface QueryContext {
|
|
3
|
+
export interface QueryContext {
|
|
8
4
|
name: string;
|
|
9
5
|
embedded: boolean;
|
|
10
6
|
tcp: TcpConnectionPool | null;
|
|
@@ -51,19 +47,4 @@ interface GetJobsOptions {
|
|
|
51
47
|
export declare function getJobs<T>(ctx: QueryContext, options?: GetJobsOptions): Job<T>[];
|
|
52
48
|
/** Get jobs with filtering (async, works with TCP) */
|
|
53
49
|
export declare function getJobsAsync<T>(ctx: QueryContext, options?: GetJobsOptions): Promise<Job<T>[]>;
|
|
54
|
-
/** Get waiting jobs */
|
|
55
|
-
export declare function getWaiting<T>(ctx: QueryContext, start?: number, end?: number): Job<T>[];
|
|
56
|
-
export declare function getWaitingAsync<T>(ctx: QueryContext, start?: number, end?: number): Promise<Job<T>[]>;
|
|
57
|
-
/** Get delayed jobs */
|
|
58
|
-
export declare function getDelayed<T>(ctx: QueryContext, start?: number, end?: number): Job<T>[];
|
|
59
|
-
export declare function getDelayedAsync<T>(ctx: QueryContext, start?: number, end?: number): Promise<Job<T>[]>;
|
|
60
|
-
/** Get active jobs */
|
|
61
|
-
export declare function getActive<T>(ctx: QueryContext, start?: number, end?: number): Job<T>[];
|
|
62
|
-
export declare function getActiveAsync<T>(ctx: QueryContext, start?: number, end?: number): Promise<Job<T>[]>;
|
|
63
|
-
/** Get completed jobs */
|
|
64
|
-
export declare function getCompleted<T>(ctx: QueryContext, start?: number, end?: number): Job<T>[];
|
|
65
|
-
export declare function getCompletedAsync<T>(ctx: QueryContext, start?: number, end?: number): Promise<Job<T>[]>;
|
|
66
|
-
/** Get failed jobs */
|
|
67
|
-
export declare function getFailed<T>(ctx: QueryContext, start?: number, end?: number): Job<T>[];
|
|
68
|
-
export declare function getFailedAsync<T>(ctx: QueryContext, start?: number, end?: number): Promise<Job<T>[]>;
|
|
69
50
|
export {};
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
/* eslint-disable @typescript-eslint/no-non-null-assertion, @typescript-eslint/no-unnecessary-condition, @typescript-eslint/switch-exhaustiveness-check */
|
|
2
|
-
/**
|
|
3
|
-
* Queue Query Operations
|
|
4
|
-
* getJob, getJobs, getJobState, getJobsAsync
|
|
5
|
-
*/
|
|
6
2
|
import { getSharedManager } from '../../manager';
|
|
7
3
|
import { toPublicJob } from '../../types';
|
|
8
4
|
import { jobId } from '../../../domain/types/job';
|
|
9
5
|
import { createSimpleJob } from '../jobProxy';
|
|
10
6
|
import { buildJobOpts } from '../../jobHelpers';
|
|
7
|
+
import { removeJobDeduplicationKey } from '../../jobDeduplication';
|
|
8
|
+
import { fetchTcpJobRows } from './queryTcpPages';
|
|
11
9
|
/** Last failure's error message from the job timeline (#74) */
|
|
12
10
|
function lastFailedError(timeline) {
|
|
13
11
|
if (!timeline)
|
|
@@ -44,6 +42,10 @@ export async function getJob(ctx, id) {
|
|
|
44
42
|
return toPublicJob({
|
|
45
43
|
job,
|
|
46
44
|
name,
|
|
45
|
+
updateProgress: async (jid, progress, message) => {
|
|
46
|
+
await mgr.updateProgress(jobId(jid), progress, message);
|
|
47
|
+
},
|
|
48
|
+
log: (jid, message) => Promise.resolve(void mgr.addLog(jobId(jid), message)),
|
|
47
49
|
getState: (jid) => ctx.getJobState(jid),
|
|
48
50
|
remove: (jid) => ctx.removeAsync(jid),
|
|
49
51
|
retry: (jid) => ctx.retryJob(jid),
|
|
@@ -72,6 +74,7 @@ export async function getJob(ctx, id) {
|
|
|
72
74
|
getFailedChildrenValues: (jid) => mgr.getFailedChildrenValues(jobId(jid)),
|
|
73
75
|
getIgnoredChildrenFailures: (jid) => mgr.getIgnoredChildrenFailures(jobId(jid)),
|
|
74
76
|
removeChildDependency: (jid) => mgr.removeChildDependency(jobId(jid)),
|
|
77
|
+
removeDeduplicationKey: (jid) => removeJobDeduplicationKey(jid, true, null),
|
|
75
78
|
removeUnprocessedChildren: async (jid) => {
|
|
76
79
|
await mgr.removeUnprocessedChildren(jobId(jid));
|
|
77
80
|
},
|
|
@@ -161,7 +164,12 @@ export function getJobs(ctx, options = {}) {
|
|
|
161
164
|
const jobs = manager.getJobs(ctx.name, {
|
|
162
165
|
state: options.state,
|
|
163
166
|
start: options.start ?? 0,
|
|
164
|
-
|
|
167
|
+
asc: options.asc,
|
|
168
|
+
end: options.end === -1
|
|
169
|
+
? Number.MAX_SAFE_INTEGER
|
|
170
|
+
: options.end !== undefined && options.end >= 0
|
|
171
|
+
? options.end
|
|
172
|
+
: 100,
|
|
165
173
|
});
|
|
166
174
|
return jobs.map((j) => {
|
|
167
175
|
const name = j.data?.name ?? 'unknown';
|
|
@@ -186,22 +194,14 @@ export function getJobs(ctx, options = {}) {
|
|
|
186
194
|
export async function getJobsAsync(ctx, options = {}) {
|
|
187
195
|
if (ctx.embedded)
|
|
188
196
|
return getJobs(ctx, options);
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
const
|
|
192
|
-
|
|
193
|
-
cmd: 'GetJobs',
|
|
194
|
-
queue: ctx.name,
|
|
195
|
-
state: options.state,
|
|
196
|
-
offset: start,
|
|
197
|
-
limit: end - start,
|
|
198
|
-
});
|
|
199
|
-
if (!response.ok || !Array.isArray(response.jobs)) {
|
|
197
|
+
if (!ctx.tcp)
|
|
198
|
+
return [];
|
|
199
|
+
const rows = await fetchTcpJobRows({ name: ctx.name, tcp: ctx.tcp }, options);
|
|
200
|
+
if (!rows)
|
|
200
201
|
return [];
|
|
201
|
-
}
|
|
202
202
|
// handleGetJobs returns the full internal job per element ({ ...job, state }),
|
|
203
203
|
// so reflect the complete opts via metaFromJob instead of a slim subset (#90).
|
|
204
|
-
const jobs =
|
|
204
|
+
const jobs = rows;
|
|
205
205
|
const now = Date.now();
|
|
206
206
|
return jobs.map((j) => {
|
|
207
207
|
const name = j.data?.name ?? 'unknown';
|
|
@@ -230,38 +230,3 @@ export async function getJobsAsync(ctx, options = {}) {
|
|
|
230
230
|
return result;
|
|
231
231
|
});
|
|
232
232
|
}
|
|
233
|
-
/** Get waiting jobs */
|
|
234
|
-
export function getWaiting(ctx, start = 0, end = 100) {
|
|
235
|
-
return getJobs(ctx, { state: 'waiting', start, end });
|
|
236
|
-
}
|
|
237
|
-
export async function getWaitingAsync(ctx, start = 0, end = 100) {
|
|
238
|
-
return getJobsAsync(ctx, { state: 'waiting', start, end });
|
|
239
|
-
}
|
|
240
|
-
/** Get delayed jobs */
|
|
241
|
-
export function getDelayed(ctx, start = 0, end = 100) {
|
|
242
|
-
return getJobs(ctx, { state: 'delayed', start, end });
|
|
243
|
-
}
|
|
244
|
-
export async function getDelayedAsync(ctx, start = 0, end = 100) {
|
|
245
|
-
return getJobsAsync(ctx, { state: 'delayed', start, end });
|
|
246
|
-
}
|
|
247
|
-
/** Get active jobs */
|
|
248
|
-
export function getActive(ctx, start = 0, end = 100) {
|
|
249
|
-
return getJobs(ctx, { state: 'active', start, end });
|
|
250
|
-
}
|
|
251
|
-
export async function getActiveAsync(ctx, start = 0, end = 100) {
|
|
252
|
-
return getJobsAsync(ctx, { state: 'active', start, end });
|
|
253
|
-
}
|
|
254
|
-
/** Get completed jobs */
|
|
255
|
-
export function getCompleted(ctx, start = 0, end = 100) {
|
|
256
|
-
return getJobs(ctx, { state: 'completed', start, end });
|
|
257
|
-
}
|
|
258
|
-
export async function getCompletedAsync(ctx, start = 0, end = 100) {
|
|
259
|
-
return getJobsAsync(ctx, { state: 'completed', start, end });
|
|
260
|
-
}
|
|
261
|
-
/** Get failed jobs */
|
|
262
|
-
export function getFailed(ctx, start = 0, end = 100) {
|
|
263
|
-
return getJobs(ctx, { state: 'failed', start, end });
|
|
264
|
-
}
|
|
265
|
-
export async function getFailedAsync(ctx, start = 0, end = 100) {
|
|
266
|
-
return getJobsAsync(ctx, { state: 'failed', start, end });
|
|
267
|
-
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** State-specific aliases over the generic queue query operations. */
|
|
2
|
+
import type { Job } from '../../types';
|
|
3
|
+
import { type QueryContext } from './query';
|
|
4
|
+
export declare function getWaiting<T>(ctx: QueryContext, start?: number, end?: number): Job<T>[];
|
|
5
|
+
export declare function getWaitingAsync<T>(ctx: QueryContext, start?: number, end?: number): Promise<Job<T>[]>;
|
|
6
|
+
export declare function getDelayed<T>(ctx: QueryContext, start?: number, end?: number): Job<T>[];
|
|
7
|
+
export declare function getDelayedAsync<T>(ctx: QueryContext, start?: number, end?: number): Promise<Job<T>[]>;
|
|
8
|
+
export declare function getActive<T>(ctx: QueryContext, start?: number, end?: number): Job<T>[];
|
|
9
|
+
export declare function getActiveAsync<T>(ctx: QueryContext, start?: number, end?: number): Promise<Job<T>[]>;
|
|
10
|
+
export declare function getCompleted<T>(ctx: QueryContext, start?: number, end?: number): Job<T>[];
|
|
11
|
+
export declare function getCompletedAsync<T>(ctx: QueryContext, start?: number, end?: number): Promise<Job<T>[]>;
|
|
12
|
+
export declare function getFailed<T>(ctx: QueryContext, start?: number, end?: number): Job<T>[];
|
|
13
|
+
export declare function getFailedAsync<T>(ctx: QueryContext, start?: number, end?: number): Promise<Job<T>[]>;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/** State-specific aliases over the generic queue query operations. */
|
|
2
|
+
import { getJobs, getJobsAsync } from './query';
|
|
3
|
+
export function getWaiting(ctx, start = 0, end = 100) {
|
|
4
|
+
return getJobs(ctx, { state: 'waiting', start, end });
|
|
5
|
+
}
|
|
6
|
+
export function getWaitingAsync(ctx, start = 0, end = 100) {
|
|
7
|
+
return getJobsAsync(ctx, { state: 'waiting', start, end });
|
|
8
|
+
}
|
|
9
|
+
export function getDelayed(ctx, start = 0, end = 100) {
|
|
10
|
+
return getJobs(ctx, { state: 'delayed', start, end });
|
|
11
|
+
}
|
|
12
|
+
export function getDelayedAsync(ctx, start = 0, end = 100) {
|
|
13
|
+
return getJobsAsync(ctx, { state: 'delayed', start, end });
|
|
14
|
+
}
|
|
15
|
+
export function getActive(ctx, start = 0, end = 100) {
|
|
16
|
+
return getJobs(ctx, { state: 'active', start, end });
|
|
17
|
+
}
|
|
18
|
+
export function getActiveAsync(ctx, start = 0, end = 100) {
|
|
19
|
+
return getJobsAsync(ctx, { state: 'active', start, end });
|
|
20
|
+
}
|
|
21
|
+
export function getCompleted(ctx, start = 0, end = 100) {
|
|
22
|
+
return getJobs(ctx, { state: 'completed', start, end });
|
|
23
|
+
}
|
|
24
|
+
export function getCompletedAsync(ctx, start = 0, end = 100) {
|
|
25
|
+
return getJobsAsync(ctx, { state: 'completed', start, end });
|
|
26
|
+
}
|
|
27
|
+
export function getFailed(ctx, start = 0, end = 100) {
|
|
28
|
+
return getJobs(ctx, { state: 'failed', start, end });
|
|
29
|
+
}
|
|
30
|
+
export function getFailedAsync(ctx, start = 0, end = 100) {
|
|
31
|
+
return getJobsAsync(ctx, { state: 'failed', start, end });
|
|
32
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { TcpConnectionPool } from '../../tcpPool';
|
|
2
|
+
interface TcpQueryContext {
|
|
3
|
+
name: string;
|
|
4
|
+
tcp: TcpConnectionPool;
|
|
5
|
+
}
|
|
6
|
+
export interface TcpJobQueryOptions {
|
|
7
|
+
state?: string | string[];
|
|
8
|
+
start?: number;
|
|
9
|
+
end?: number;
|
|
10
|
+
asc?: boolean;
|
|
11
|
+
}
|
|
12
|
+
/** Read a finite range once, or page until exhaustion when `end` is -1. */
|
|
13
|
+
export declare function fetchTcpJobRows(ctx: TcpQueryContext, options: TcpJobQueryOptions): Promise<Record<string, unknown>[] | null>;
|
|
14
|
+
export {};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
const QUERY_PAGE_SIZE = 1000;
|
|
2
|
+
async function fetchPage(ctx, options, offset, limit) {
|
|
3
|
+
const response = await ctx.tcp.send({
|
|
4
|
+
cmd: 'GetJobs',
|
|
5
|
+
queue: ctx.name,
|
|
6
|
+
state: options.state,
|
|
7
|
+
offset,
|
|
8
|
+
limit,
|
|
9
|
+
asc: options.asc,
|
|
10
|
+
});
|
|
11
|
+
if (!response.ok || !Array.isArray(response.jobs))
|
|
12
|
+
return null;
|
|
13
|
+
return response.jobs;
|
|
14
|
+
}
|
|
15
|
+
/** Read a finite range once, or page until exhaustion when `end` is -1. */
|
|
16
|
+
export async function fetchTcpJobRows(ctx, options) {
|
|
17
|
+
const start = options.start ?? 0;
|
|
18
|
+
if (options.end !== -1) {
|
|
19
|
+
const end = options.end !== undefined && options.end >= 0 ? options.end : 1000;
|
|
20
|
+
return fetchPage(ctx, options, start, end - start);
|
|
21
|
+
}
|
|
22
|
+
const rows = [];
|
|
23
|
+
const seen = new Set();
|
|
24
|
+
let offset = start;
|
|
25
|
+
while (true) {
|
|
26
|
+
const page = await fetchPage(ctx, options, offset, QUERY_PAGE_SIZE);
|
|
27
|
+
if (!page)
|
|
28
|
+
return null;
|
|
29
|
+
for (const row of page) {
|
|
30
|
+
const id = String(row.id);
|
|
31
|
+
if (!seen.has(id)) {
|
|
32
|
+
seen.add(id);
|
|
33
|
+
rows.push(row);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
if (page.length < QUERY_PAGE_SIZE)
|
|
37
|
+
return rows;
|
|
38
|
+
offset += page.length;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -118,11 +118,14 @@ export declare class Queue<T = unknown> {
|
|
|
118
118
|
getDlqConfig(): DlqConfig;
|
|
119
119
|
getDlqConfigAsync(): Promise<DlqConfig>;
|
|
120
120
|
getDlq(filter?: DlqFilter): DlqEntry<T>[];
|
|
121
|
+
getDlqAsync(filter?: DlqFilter): Promise<DlqEntry<T>[]>;
|
|
121
122
|
getDlqJobsAsync(count?: number): Promise<Job<T>[]>;
|
|
122
123
|
getDlqStats(): DlqStats;
|
|
124
|
+
getDlqStatsAsync(): Promise<DlqStats>;
|
|
123
125
|
retryDlq(id?: string): number;
|
|
124
126
|
retryDlqAsync(id?: string): Promise<number>;
|
|
125
127
|
retryDlqByFilter(filter: DlqFilter): number;
|
|
128
|
+
retryDlqByFilterAsync(filter: DlqFilter): Promise<number>;
|
|
126
129
|
purgeDlq(): number;
|
|
127
130
|
purgeDlqAsync(): Promise<number>;
|
|
128
131
|
retryCompleted(id?: string): number;
|
|
@@ -11,6 +11,7 @@ import { getSharedManager } from '../manager';
|
|
|
11
11
|
// Import operation modules
|
|
12
12
|
import * as addOps from './operations/add';
|
|
13
13
|
import * as queryOps from './operations/query';
|
|
14
|
+
import * as queryStateOps from './operations/queryStates';
|
|
14
15
|
import * as countsOps from './operations/counts';
|
|
15
16
|
import * as controlOps from './operations/control';
|
|
16
17
|
import * as managementOps from './operations/management';
|
|
@@ -127,6 +128,8 @@ export class Queue {
|
|
|
127
128
|
clearJobLogs: (id, keep) => this.clearJobLogs(id, keep),
|
|
128
129
|
getJobDependencies: (id, o) => this.getJobDependencies(id, o),
|
|
129
130
|
getJobDependenciesCount: (id, o) => this.getJobDependenciesCount(id, o),
|
|
131
|
+
getJobCountsAsync: () => this.getJobCountsAsync(),
|
|
132
|
+
getJobsAsync: (options) => this.getJobsAsync(options),
|
|
130
133
|
moveJobToCompleted: (id, r, t) => this.moveJobToCompleted(id, r, t),
|
|
131
134
|
moveJobToFailed: (id, e, t) => this.moveJobToFailed(id, e, t),
|
|
132
135
|
moveJobToWait: (id, t) => this.moveJobToWait(id, t),
|
|
@@ -195,34 +198,34 @@ export class Queue {
|
|
|
195
198
|
return queryOps.getJobsAsync(this.queryCtx, opts);
|
|
196
199
|
}
|
|
197
200
|
getWaiting(start, end) {
|
|
198
|
-
return
|
|
201
|
+
return queryStateOps.getWaiting(this.queryCtx, start, end);
|
|
199
202
|
}
|
|
200
203
|
getWaitingAsync(start, end) {
|
|
201
|
-
return
|
|
204
|
+
return queryStateOps.getWaitingAsync(this.queryCtx, start, end);
|
|
202
205
|
}
|
|
203
206
|
getDelayed(start, end) {
|
|
204
|
-
return
|
|
207
|
+
return queryStateOps.getDelayed(this.queryCtx, start, end);
|
|
205
208
|
}
|
|
206
209
|
getDelayedAsync(start, end) {
|
|
207
|
-
return
|
|
210
|
+
return queryStateOps.getDelayedAsync(this.queryCtx, start, end);
|
|
208
211
|
}
|
|
209
212
|
getActive(start, end) {
|
|
210
|
-
return
|
|
213
|
+
return queryStateOps.getActive(this.queryCtx, start, end);
|
|
211
214
|
}
|
|
212
215
|
getActiveAsync(start, end) {
|
|
213
|
-
return
|
|
216
|
+
return queryStateOps.getActiveAsync(this.queryCtx, start, end);
|
|
214
217
|
}
|
|
215
218
|
getCompleted(start, end) {
|
|
216
|
-
return
|
|
219
|
+
return queryStateOps.getCompleted(this.queryCtx, start, end);
|
|
217
220
|
}
|
|
218
221
|
getCompletedAsync(start, end) {
|
|
219
|
-
return
|
|
222
|
+
return queryStateOps.getCompletedAsync(this.queryCtx, start, end);
|
|
220
223
|
}
|
|
221
224
|
getFailed(start, end) {
|
|
222
|
-
return
|
|
225
|
+
return queryStateOps.getFailed(this.queryCtx, start, end);
|
|
223
226
|
}
|
|
224
227
|
getFailedAsync(start, end) {
|
|
225
|
-
return
|
|
228
|
+
return queryStateOps.getFailedAsync(this.queryCtx, start, end);
|
|
226
229
|
}
|
|
227
230
|
// ============ Count Operations ============
|
|
228
231
|
getJobCounts() {
|
|
@@ -368,7 +371,10 @@ export class Queue {
|
|
|
368
371
|
return dlqOps.getDlqConfigAsync(this.ctx);
|
|
369
372
|
}
|
|
370
373
|
getDlq(filter) {
|
|
371
|
-
return dlqOps.getDlq(this.
|
|
374
|
+
return dlqOps.getDlq(this.queryCtx, filter);
|
|
375
|
+
}
|
|
376
|
+
getDlqAsync(filter) {
|
|
377
|
+
return dlqOps.getDlqAsync(this.queryCtx, filter);
|
|
372
378
|
}
|
|
373
379
|
getDlqJobsAsync(count) {
|
|
374
380
|
return dlqOps.getDlqJobsAsync(this.queryCtx, count);
|
|
@@ -376,6 +382,9 @@ export class Queue {
|
|
|
376
382
|
getDlqStats() {
|
|
377
383
|
return dlqOps.getDlqStats(this.ctx);
|
|
378
384
|
}
|
|
385
|
+
getDlqStatsAsync() {
|
|
386
|
+
return dlqOps.getDlqStatsAsync(this.ctx);
|
|
387
|
+
}
|
|
379
388
|
retryDlq(id) {
|
|
380
389
|
return dlqOps.retryDlq(this.ctx, id);
|
|
381
390
|
}
|
|
@@ -385,6 +394,9 @@ export class Queue {
|
|
|
385
394
|
retryDlqByFilter(filter) {
|
|
386
395
|
return dlqOps.retryDlqByFilter(this.ctx, filter);
|
|
387
396
|
}
|
|
397
|
+
retryDlqByFilterAsync(filter) {
|
|
398
|
+
return dlqOps.retryDlqByFilterAsync(this.ctx, filter);
|
|
399
|
+
}
|
|
388
400
|
purgeDlq() {
|
|
389
401
|
return dlqOps.purgeDlq(this.ctx);
|
|
390
402
|
}
|
|
@@ -16,7 +16,7 @@ export declare function removeGlobalConcurrency(ctx: RateLimitContext): void;
|
|
|
16
16
|
/** Remove global concurrency limit and resolve once the server has applied it. */
|
|
17
17
|
export declare function removeGlobalConcurrencyAsync(ctx: RateLimitContext): Promise<void>;
|
|
18
18
|
/** Get global concurrency limit */
|
|
19
|
-
export declare function getGlobalConcurrency(
|
|
19
|
+
export declare function getGlobalConcurrency(ctx: RateLimitContext): Promise<number | null>;
|
|
20
20
|
/**
|
|
21
21
|
* Set global rate limit: `max` jobs per `duration` ms (default 1000ms).
|
|
22
22
|
* The window is honored in both embedded and TCP modes.
|
|
@@ -29,7 +29,7 @@ export declare function removeGlobalRateLimit(ctx: RateLimitContext): void;
|
|
|
29
29
|
/** Remove global rate limit and resolve once the server has applied it. */
|
|
30
30
|
export declare function removeGlobalRateLimitAsync(ctx: RateLimitContext): Promise<void>;
|
|
31
31
|
/** Get global rate limit */
|
|
32
|
-
export declare function getGlobalRateLimit(
|
|
32
|
+
export declare function getGlobalRateLimit(ctx: RateLimitContext): Promise<{
|
|
33
33
|
max: number;
|
|
34
34
|
duration: number;
|
|
35
35
|
} | null>;
|
|
@@ -40,7 +40,7 @@ export declare function getGlobalRateLimit(_ctx: RateLimitContext): Promise<{
|
|
|
40
40
|
*/
|
|
41
41
|
export declare function rateLimit(ctx: RateLimitContext, expireTimeMs: number): Promise<void>;
|
|
42
42
|
/** Get rate limit TTL */
|
|
43
|
-
export declare function getRateLimitTtl(
|
|
43
|
+
export declare function getRateLimitTtl(ctx: RateLimitContext, maxJobs?: number): Promise<number>;
|
|
44
44
|
/** Check if queue is at max capacity */
|
|
45
|
-
export declare function isMaxed(
|
|
45
|
+
export declare function isMaxed(ctx: RateLimitContext): Promise<boolean>;
|
|
46
46
|
export {};
|
|
@@ -2,6 +2,15 @@
|
|
|
2
2
|
* Rate Limit and Concurrency Operations
|
|
3
3
|
*/
|
|
4
4
|
import { getSharedManager } from '../manager';
|
|
5
|
+
async function getQueueLimits(ctx, maxJobs) {
|
|
6
|
+
if (ctx.embedded)
|
|
7
|
+
return getSharedManager().getQueueLimitStatus(ctx.name, maxJobs);
|
|
8
|
+
if (!ctx.tcp) {
|
|
9
|
+
return { rateLimit: null, rateLimitTtl: -2, concurrencyLimit: null, maxed: false };
|
|
10
|
+
}
|
|
11
|
+
const response = await ctx.tcp.send({ cmd: 'GetQueueLimits', queue: ctx.name, maxJobs });
|
|
12
|
+
return response.data.limits;
|
|
13
|
+
}
|
|
5
14
|
/** Set global concurrency limit */
|
|
6
15
|
export function setGlobalConcurrency(ctx, concurrency) {
|
|
7
16
|
if (ctx.embedded) {
|
|
@@ -35,8 +44,8 @@ export async function removeGlobalConcurrencyAsync(ctx) {
|
|
|
35
44
|
await ctx.tcp.send({ cmd: 'ClearConcurrency', queue: ctx.name });
|
|
36
45
|
}
|
|
37
46
|
/** Get global concurrency limit */
|
|
38
|
-
export function getGlobalConcurrency(
|
|
39
|
-
return
|
|
47
|
+
export async function getGlobalConcurrency(ctx) {
|
|
48
|
+
return (await getQueueLimits(ctx)).concurrencyLimit;
|
|
40
49
|
}
|
|
41
50
|
/**
|
|
42
51
|
* Set global rate limit: `max` jobs per `duration` ms (default 1000ms).
|
|
@@ -74,8 +83,8 @@ export async function removeGlobalRateLimitAsync(ctx) {
|
|
|
74
83
|
await ctx.tcp.send({ cmd: 'RateLimitClear', queue: ctx.name });
|
|
75
84
|
}
|
|
76
85
|
/** Get global rate limit */
|
|
77
|
-
export function getGlobalRateLimit(
|
|
78
|
-
return
|
|
86
|
+
export function getGlobalRateLimit(ctx) {
|
|
87
|
+
return getQueueLimits(ctx).then((limits) => limits.rateLimit);
|
|
79
88
|
}
|
|
80
89
|
/**
|
|
81
90
|
* Set temporary rate limit with expiration. The expiry lives broker-side
|
|
@@ -94,10 +103,10 @@ export async function rateLimit(ctx, expireTimeMs) {
|
|
|
94
103
|
}
|
|
95
104
|
}
|
|
96
105
|
/** Get rate limit TTL */
|
|
97
|
-
export function getRateLimitTtl(
|
|
98
|
-
return
|
|
106
|
+
export async function getRateLimitTtl(ctx, maxJobs) {
|
|
107
|
+
return (await getQueueLimits(ctx, maxJobs)).rateLimitTtl;
|
|
99
108
|
}
|
|
100
109
|
/** Check if queue is at max capacity */
|
|
101
|
-
export function isMaxed(
|
|
102
|
-
return
|
|
110
|
+
export async function isMaxed(ctx) {
|
|
111
|
+
return (await getQueueLimits(ctx)).maxed;
|
|
103
112
|
}
|
|
@@ -11,6 +11,18 @@ export interface WorkerInfo {
|
|
|
11
11
|
id: string;
|
|
12
12
|
name: string;
|
|
13
13
|
addr?: string;
|
|
14
|
+
queues?: string[];
|
|
15
|
+
concurrency?: number;
|
|
16
|
+
hostname?: string;
|
|
17
|
+
pid?: number;
|
|
18
|
+
status?: 'active' | 'stale';
|
|
19
|
+
registeredAt?: number;
|
|
20
|
+
lastSeen?: number;
|
|
21
|
+
activeJobs?: number;
|
|
22
|
+
processedJobs?: number;
|
|
23
|
+
failedJobs?: number;
|
|
24
|
+
currentJob?: string | null;
|
|
25
|
+
uptime?: number;
|
|
14
26
|
}
|
|
15
27
|
/** Get workers processing this queue */
|
|
16
28
|
export declare function getWorkers(ctx: WorkersContext): Promise<WorkerInfo[]>;
|
|
@@ -6,12 +6,30 @@ import { getSharedManager } from '../manager';
|
|
|
6
6
|
/** Get workers processing this queue */
|
|
7
7
|
export async function getWorkers(ctx) {
|
|
8
8
|
if (ctx.embedded) {
|
|
9
|
-
return
|
|
9
|
+
return getSharedManager()
|
|
10
|
+
.workerManager.getForQueue(ctx.name)
|
|
11
|
+
.map((worker) => ({
|
|
12
|
+
id: worker.id,
|
|
13
|
+
name: worker.name,
|
|
14
|
+
queues: worker.queues,
|
|
15
|
+
concurrency: worker.concurrency,
|
|
16
|
+
hostname: worker.hostname,
|
|
17
|
+
pid: worker.pid,
|
|
18
|
+
status: 'active',
|
|
19
|
+
registeredAt: worker.registeredAt,
|
|
20
|
+
lastSeen: worker.lastSeen,
|
|
21
|
+
activeJobs: worker.activeJobs,
|
|
22
|
+
processedJobs: worker.processedJobs,
|
|
23
|
+
failedJobs: worker.failedJobs,
|
|
24
|
+
currentJob: worker.currentJob,
|
|
25
|
+
uptime: Date.now() - worker.registeredAt,
|
|
26
|
+
}));
|
|
10
27
|
}
|
|
11
28
|
const response = await ctx.tcp.send({ cmd: 'ListWorkers' });
|
|
12
29
|
if (!response.ok)
|
|
13
30
|
return [];
|
|
14
|
-
|
|
31
|
+
const workers = response.data?.workers ?? [];
|
|
32
|
+
return workers.filter((worker) => worker.queues?.includes(ctx.name));
|
|
15
33
|
}
|
|
16
34
|
/** Get worker count */
|
|
17
35
|
export async function getWorkersCount(ctx) {
|
|
@@ -22,6 +22,7 @@ import type { QueueOptions, WorkerOptions, Processor } from './types';
|
|
|
22
22
|
*/
|
|
23
23
|
export declare class QueueGroup {
|
|
24
24
|
readonly prefix: string;
|
|
25
|
+
private readonly queues;
|
|
25
26
|
constructor(namespace: string);
|
|
26
27
|
/** Get a queue within this group */
|
|
27
28
|
getQueue<T = unknown>(name: string, opts?: QueueOptions): Queue<T>;
|
|
@@ -37,4 +38,14 @@ export declare class QueueGroup {
|
|
|
37
38
|
drainAll(): void;
|
|
38
39
|
/** Obliterate all queues in this group (remove all data) */
|
|
39
40
|
obliterateAll(): void;
|
|
41
|
+
/** List queues created through this group, including remote queues. */
|
|
42
|
+
listQueuesAsync(): Promise<string[]>;
|
|
43
|
+
/** Pause every queue created through this group. */
|
|
44
|
+
pauseAllAsync(): Promise<void>;
|
|
45
|
+
/** Resume every queue created through this group. */
|
|
46
|
+
resumeAllAsync(): Promise<void>;
|
|
47
|
+
/** Drain waiting jobs from every queue and return the aggregate count. */
|
|
48
|
+
drainAllAsync(): Promise<number>;
|
|
49
|
+
/** Remove all data from every queue created through this group. */
|
|
50
|
+
obliterateAllAsync(): Promise<void>;
|
|
40
51
|
}
|