bunqueue 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/dlqManager.d.ts +21 -4
- package/dist/application/dlqManager.d.ts.map +1 -1
- package/dist/application/dlqManager.js +167 -25
- package/dist/application/dlqManager.js.map +1 -1
- package/dist/application/queueManager.d.ts +18 -0
- package/dist/application/queueManager.d.ts.map +1 -1
- package/dist/application/queueManager.js +134 -0
- package/dist/application/queueManager.js.map +1 -1
- package/dist/domain/queue/shard.d.ts +32 -7
- package/dist/domain/queue/shard.d.ts.map +1 -1
- package/dist/domain/queue/shard.js +102 -8
- package/dist/domain/queue/shard.js.map +1 -1
- package/dist/domain/types/dlq.d.ts +120 -0
- package/dist/domain/types/dlq.d.ts.map +1 -0
- package/dist/domain/types/dlq.js +77 -0
- package/dist/domain/types/dlq.js.map +1 -0
- package/dist/domain/types/stall.d.ts +49 -0
- package/dist/domain/types/stall.d.ts.map +1 -0
- package/dist/domain/types/stall.js +77 -0
- package/dist/domain/types/stall.js.map +1 -0
- package/dist/main.js +1 -1
- package/dist/main.js.map +1 -1
- package/dist/shared/version.d.ts +2 -2
- package/dist/shared/version.d.ts.map +1 -1
- package/dist/shared/version.js +27 -2
- package/dist/shared/version.js.map +1 -1
- package/package.json +6 -6
|
@@ -1,19 +1,36 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* DLQ Manager
|
|
3
|
-
* Dead Letter Queue operations
|
|
2
|
+
* Advanced DLQ Manager
|
|
3
|
+
* Dead Letter Queue operations with auto-retry, metadata, and lifecycle management
|
|
4
4
|
*/
|
|
5
5
|
import type { Job, JobId } from '../domain/types/job';
|
|
6
6
|
import type { JobLocation } from '../domain/types/queue';
|
|
7
7
|
import type { Shard } from '../domain/queue/shard';
|
|
8
|
+
import type { DlqEntry, DlqConfig, DlqFilter, DlqStats } from '../domain/types/dlq';
|
|
8
9
|
/** Context for DLQ operations */
|
|
9
10
|
export interface DlqContext {
|
|
10
11
|
shards: Shard[];
|
|
11
12
|
jobIndex: Map<JobId, JobLocation>;
|
|
12
13
|
}
|
|
13
|
-
/** Get jobs from DLQ */
|
|
14
|
+
/** Get jobs from DLQ (backward compatible) */
|
|
14
15
|
export declare function getDlqJobs(queue: string, ctx: DlqContext, count?: number): Job[];
|
|
15
|
-
/**
|
|
16
|
+
/** Get DLQ entries with full metadata */
|
|
17
|
+
export declare function getDlqEntries(queue: string, ctx: DlqContext, filter?: DlqFilter): DlqEntry[];
|
|
18
|
+
/** Get DLQ statistics */
|
|
19
|
+
export declare function getDlqStats(queue: string, ctx: DlqContext): DlqStats;
|
|
20
|
+
/** Retry a single job from DLQ */
|
|
21
|
+
export declare function retryDlqJob(queue: string, jobId: JobId, ctx: DlqContext): Job | null;
|
|
22
|
+
/** Retry jobs from DLQ (backward compatible) */
|
|
16
23
|
export declare function retryDlqJobs(queue: string, ctx: DlqContext, jobId?: JobId): number;
|
|
24
|
+
/** Retry jobs by filter */
|
|
25
|
+
export declare function retryDlqByFilter(queue: string, ctx: DlqContext, filter: DlqFilter): number;
|
|
26
|
+
/** Process auto-retry for a queue */
|
|
27
|
+
export declare function processAutoRetry(queue: string, ctx: DlqContext): number;
|
|
28
|
+
/** Purge expired entries from DLQ */
|
|
29
|
+
export declare function purgeExpiredDlq(queue: string, ctx: DlqContext): number;
|
|
17
30
|
/** Purge all jobs from DLQ */
|
|
18
31
|
export declare function purgeDlqJobs(queue: string, ctx: DlqContext): number;
|
|
32
|
+
/** Configure DLQ for a queue */
|
|
33
|
+
export declare function configureDlq(queue: string, ctx: DlqContext, config: Partial<DlqConfig>): void;
|
|
34
|
+
/** Get DLQ configuration */
|
|
35
|
+
export declare function getDlqConfig(queue: string, ctx: DlqContext): DlqConfig;
|
|
19
36
|
//# sourceMappingURL=dlqManager.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dlqManager.d.ts","sourceRoot":"","sources":["../../src/application/dlqManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"dlqManager.d.ts","sourceRoot":"","sources":["../../src/application/dlqManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAKpF,iCAAiC;AACjC,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;CACnC;AAED,8CAA8C;AAC9C,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,GAAG,EAAE,CAGhF;AAED,yCAAyC;AACzC,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,SAAS,GAAG,QAAQ,EAAE,CAQ5F;AAED,yBAAyB;AACzB,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,GAAG,QAAQ,CA4CpE;AAED,kCAAkC;AAClC,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,GAAG,GAAG,GAAG,IAAI,CAqBpF;AAED,gDAAgD;AAChD,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CA6BlF;AAED,2BAA2B;AAC3B,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,GAAG,MAAM,CA2B1F;AAED,qCAAqC;AACrC,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,GAAG,MAAM,CAwCvE;AAED,qCAAqC;AACrC,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,GAAG,MAAM,CAQtE;AAED,8BAA8B;AAC9B,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,GAAG,MAAM,CAGnE;AAED,gCAAgC;AAChC,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,IAAI,CAI7F;AAED,4BAA4B;AAC5B,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,GAAG,SAAS,CAGtE"}
|
|
@@ -1,47 +1,178 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* DLQ Manager
|
|
3
|
-
* Dead Letter Queue operations
|
|
2
|
+
* Advanced DLQ Manager
|
|
3
|
+
* Dead Letter Queue operations with auto-retry, metadata, and lifecycle management
|
|
4
4
|
*/
|
|
5
|
+
import { scheduleNextRetry } from '../domain/types/dlq';
|
|
5
6
|
import { shardIndex } from '../shared/hash';
|
|
6
|
-
|
|
7
|
+
import { queueLog } from '../shared/logger';
|
|
8
|
+
/** Get jobs from DLQ (backward compatible) */
|
|
7
9
|
export function getDlqJobs(queue, ctx, count) {
|
|
8
10
|
const idx = shardIndex(queue);
|
|
9
|
-
|
|
10
|
-
return count ? jobs.slice(0, count) : jobs;
|
|
11
|
+
return ctx.shards[idx].getDlq(queue, count);
|
|
11
12
|
}
|
|
12
|
-
/**
|
|
13
|
-
export function
|
|
13
|
+
/** Get DLQ entries with full metadata */
|
|
14
|
+
export function getDlqEntries(queue, ctx, filter) {
|
|
14
15
|
const idx = shardIndex(queue);
|
|
15
16
|
const shard = ctx.shards[idx];
|
|
17
|
+
if (filter) {
|
|
18
|
+
return shard.getDlqFiltered(queue, filter);
|
|
19
|
+
}
|
|
20
|
+
return shard.getDlqEntries(queue);
|
|
21
|
+
}
|
|
22
|
+
/** Get DLQ statistics */
|
|
23
|
+
export function getDlqStats(queue, ctx) {
|
|
24
|
+
const idx = shardIndex(queue);
|
|
25
|
+
const entries = ctx.shards[idx].getDlqEntries(queue);
|
|
26
|
+
const config = ctx.shards[idx].getDlqConfig(queue);
|
|
16
27
|
const now = Date.now();
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
const stats = {
|
|
29
|
+
total: entries.length,
|
|
30
|
+
byReason: {
|
|
31
|
+
["explicit_fail" /* FailureReason.ExplicitFail */]: 0,
|
|
32
|
+
["max_attempts_exceeded" /* FailureReason.MaxAttemptsExceeded */]: 0,
|
|
33
|
+
["timeout" /* FailureReason.Timeout */]: 0,
|
|
34
|
+
["stalled" /* FailureReason.Stalled */]: 0,
|
|
35
|
+
["ttl_expired" /* FailureReason.TtlExpired */]: 0,
|
|
36
|
+
["worker_lost" /* FailureReason.WorkerLost */]: 0,
|
|
37
|
+
["unknown" /* FailureReason.Unknown */]: 0,
|
|
38
|
+
},
|
|
39
|
+
byQueue: { [queue]: entries.length },
|
|
40
|
+
pendingRetry: 0,
|
|
41
|
+
expired: 0,
|
|
42
|
+
oldestEntry: null,
|
|
43
|
+
newestEntry: null,
|
|
44
|
+
};
|
|
45
|
+
for (const entry of entries) {
|
|
46
|
+
stats.byReason[entry.reason]++;
|
|
47
|
+
if (entry.nextRetryAt && entry.nextRetryAt <= now && entry.retryCount < config.maxAutoRetries) {
|
|
48
|
+
stats.pendingRetry++;
|
|
49
|
+
}
|
|
50
|
+
if (entry.expiresAt && entry.expiresAt <= now) {
|
|
51
|
+
stats.expired++;
|
|
52
|
+
}
|
|
53
|
+
if (stats.oldestEntry === null || entry.enteredAt < stats.oldestEntry) {
|
|
54
|
+
stats.oldestEntry = entry.enteredAt;
|
|
55
|
+
}
|
|
56
|
+
if (stats.newestEntry === null || entry.enteredAt > stats.newestEntry) {
|
|
57
|
+
stats.newestEntry = entry.enteredAt;
|
|
29
58
|
}
|
|
30
|
-
return 0;
|
|
31
59
|
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
60
|
+
return stats;
|
|
61
|
+
}
|
|
62
|
+
/** Retry a single job from DLQ */
|
|
63
|
+
export function retryDlqJob(queue, jobId, ctx) {
|
|
64
|
+
const idx = shardIndex(queue);
|
|
65
|
+
const shard = ctx.shards[idx];
|
|
66
|
+
const now = Date.now();
|
|
67
|
+
const entry = shard.removeFromDlq(queue, jobId);
|
|
68
|
+
if (!entry)
|
|
69
|
+
return null;
|
|
70
|
+
const job = entry.job;
|
|
71
|
+
job.attempts = 0;
|
|
72
|
+
job.runAt = now;
|
|
73
|
+
job.stallCount = 0;
|
|
74
|
+
job.lastHeartbeat = now;
|
|
75
|
+
shard.getQueue(queue).push(job);
|
|
76
|
+
const isDelayed = job.runAt > now;
|
|
77
|
+
shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
|
|
78
|
+
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx: idx, queueName: queue });
|
|
79
|
+
queueLog.info('Retried DLQ job', { jobId: String(jobId), queue });
|
|
80
|
+
return job;
|
|
81
|
+
}
|
|
82
|
+
/** Retry jobs from DLQ (backward compatible) */
|
|
83
|
+
export function retryDlqJobs(queue, ctx, jobId) {
|
|
84
|
+
if (jobId) {
|
|
85
|
+
return retryDlqJob(queue, jobId, ctx) ? 1 : 0;
|
|
86
|
+
}
|
|
87
|
+
const idx = shardIndex(queue);
|
|
88
|
+
const shard = ctx.shards[idx];
|
|
89
|
+
const entries = shard.getDlqEntries(queue);
|
|
90
|
+
const count = entries.length;
|
|
91
|
+
// Clear all entries
|
|
35
92
|
shard.clearDlq(queue);
|
|
36
|
-
|
|
93
|
+
const now = Date.now();
|
|
94
|
+
for (const entry of entries) {
|
|
95
|
+
const job = entry.job;
|
|
37
96
|
job.attempts = 0;
|
|
38
97
|
job.runAt = now;
|
|
98
|
+
job.stallCount = 0;
|
|
99
|
+
job.lastHeartbeat = now;
|
|
39
100
|
shard.getQueue(queue).push(job);
|
|
40
|
-
// Update running counters for O(1) stats and temporal index
|
|
41
101
|
const isDelayed = job.runAt > now;
|
|
42
102
|
shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
|
|
43
103
|
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx: idx, queueName: queue });
|
|
44
104
|
}
|
|
105
|
+
queueLog.info('Retried all DLQ jobs', { queue, count });
|
|
106
|
+
return count;
|
|
107
|
+
}
|
|
108
|
+
/** Retry jobs by filter */
|
|
109
|
+
export function retryDlqByFilter(queue, ctx, filter) {
|
|
110
|
+
const idx = shardIndex(queue);
|
|
111
|
+
const shard = ctx.shards[idx];
|
|
112
|
+
const entries = shard.getDlqFiltered(queue, filter);
|
|
113
|
+
let count = 0;
|
|
114
|
+
const now = Date.now();
|
|
115
|
+
for (const entry of entries) {
|
|
116
|
+
const removed = shard.removeFromDlq(queue, entry.job.id);
|
|
117
|
+
if (!removed)
|
|
118
|
+
continue;
|
|
119
|
+
const job = entry.job;
|
|
120
|
+
job.attempts = 0;
|
|
121
|
+
job.runAt = now;
|
|
122
|
+
job.stallCount = 0;
|
|
123
|
+
job.lastHeartbeat = now;
|
|
124
|
+
shard.getQueue(queue).push(job);
|
|
125
|
+
const isDelayed = job.runAt > now;
|
|
126
|
+
shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
|
|
127
|
+
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx: idx, queueName: queue });
|
|
128
|
+
count++;
|
|
129
|
+
}
|
|
130
|
+
queueLog.info('Retried filtered DLQ jobs', { queue, count, filter: JSON.stringify(filter) });
|
|
131
|
+
return count;
|
|
132
|
+
}
|
|
133
|
+
/** Process auto-retry for a queue */
|
|
134
|
+
export function processAutoRetry(queue, ctx) {
|
|
135
|
+
const idx = shardIndex(queue);
|
|
136
|
+
const shard = ctx.shards[idx];
|
|
137
|
+
const config = shard.getDlqConfig(queue);
|
|
138
|
+
if (!config.autoRetry)
|
|
139
|
+
return 0;
|
|
140
|
+
const now = Date.now();
|
|
141
|
+
const entries = shard.getAutoRetryEntries(queue, now);
|
|
142
|
+
let count = 0;
|
|
143
|
+
for (const entry of entries) {
|
|
144
|
+
// Update retry tracking
|
|
145
|
+
scheduleNextRetry(entry, config);
|
|
146
|
+
// Remove from DLQ
|
|
147
|
+
const removed = shard.removeFromDlq(queue, entry.job.id);
|
|
148
|
+
if (!removed)
|
|
149
|
+
continue;
|
|
150
|
+
// Re-queue job
|
|
151
|
+
const job = entry.job;
|
|
152
|
+
job.attempts = 0;
|
|
153
|
+
job.runAt = now;
|
|
154
|
+
job.stallCount = 0;
|
|
155
|
+
job.lastHeartbeat = now;
|
|
156
|
+
shard.getQueue(queue).push(job);
|
|
157
|
+
const isDelayed = job.runAt > now;
|
|
158
|
+
shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
|
|
159
|
+
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx: idx, queueName: queue });
|
|
160
|
+
count++;
|
|
161
|
+
queueLog.info('Auto-retried DLQ job', {
|
|
162
|
+
jobId: String(job.id),
|
|
163
|
+
queue,
|
|
164
|
+
retryCount: entry.retryCount,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
return count;
|
|
168
|
+
}
|
|
169
|
+
/** Purge expired entries from DLQ */
|
|
170
|
+
export function purgeExpiredDlq(queue, ctx) {
|
|
171
|
+
const idx = shardIndex(queue);
|
|
172
|
+
const count = ctx.shards[idx].purgeExpired(queue);
|
|
173
|
+
if (count > 0) {
|
|
174
|
+
queueLog.info('Purged expired DLQ entries', { queue, count });
|
|
175
|
+
}
|
|
45
176
|
return count;
|
|
46
177
|
}
|
|
47
178
|
/** Purge all jobs from DLQ */
|
|
@@ -49,4 +180,15 @@ export function purgeDlqJobs(queue, ctx) {
|
|
|
49
180
|
const idx = shardIndex(queue);
|
|
50
181
|
return ctx.shards[idx].clearDlq(queue);
|
|
51
182
|
}
|
|
183
|
+
/** Configure DLQ for a queue */
|
|
184
|
+
export function configureDlq(queue, ctx, config) {
|
|
185
|
+
const idx = shardIndex(queue);
|
|
186
|
+
ctx.shards[idx].setDlqConfig(queue, config);
|
|
187
|
+
queueLog.info('Updated DLQ config', { queue, config });
|
|
188
|
+
}
|
|
189
|
+
/** Get DLQ configuration */
|
|
190
|
+
export function getDlqConfig(queue, ctx) {
|
|
191
|
+
const idx = shardIndex(queue);
|
|
192
|
+
return ctx.shards[idx].getDlqConfig(queue);
|
|
193
|
+
}
|
|
52
194
|
//# sourceMappingURL=dlqManager.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dlqManager.js","sourceRoot":"","sources":["../../src/application/dlqManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"dlqManager.js","sourceRoot":"","sources":["../../src/application/dlqManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,OAAO,EAAiB,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAQ5C,8CAA8C;AAC9C,MAAM,UAAU,UAAU,CAAC,KAAa,EAAE,GAAe,EAAE,KAAc;IACvE,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAC9C,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,GAAe,EAAE,MAAkB;IAC9E,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAE9B,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;AACpC,CAAC;AAED,yBAAyB;AACzB,MAAM,UAAU,WAAW,CAAC,KAAa,EAAE,GAAe;IACxD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,MAAM,KAAK,GAAa;QACtB,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,QAAQ,EAAE;YACR,kDAA4B,EAAE,CAAC;YAC/B,iEAAmC,EAAE,CAAC;YACtC,uCAAuB,EAAE,CAAC;YAC1B,uCAAuB,EAAE,CAAC;YAC1B,8CAA0B,EAAE,CAAC;YAC7B,8CAA0B,EAAE,CAAC;YAC7B,uCAAuB,EAAE,CAAC;SAC3B;QACD,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE;QACpC,YAAY,EAAE,CAAC;QACf,OAAO,EAAE,CAAC;QACV,WAAW,EAAE,IAAI;QACjB,WAAW,EAAE,IAAI;KAClB,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAE/B,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,WAAW,IAAI,GAAG,IAAI,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;YAC9F,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,CAAC;QAED,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,IAAI,GAAG,EAAE,CAAC;YAC9C,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,CAAC;QAED,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;YACtE,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,SAAS,CAAC;QACtC,CAAC;QACD,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;YACtE,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,SAAS,CAAC;QACtC,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,kCAAkC;AAClC,MAAM,UAAU,WAAW,CAAC,KAAa,EAAE,KAAY,EAAE,GAAe;IACtE,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAChD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;IACtB,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC;IACjB,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;IAChB,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC;IACnB,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC;IAExB,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;IAClC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1E,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,QAAQ,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAClE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,gDAAgD;AAChD,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,GAAe,EAAE,KAAa;IACxE,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,WAAW,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAE7B,oBAAoB;IACpB,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QACtB,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC;QACjB,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;QAChB,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC;QACnB,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC;QAExB,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;QAClC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1E,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACxD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,GAAe,EAAE,MAAiB;IAChF,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAEpD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QACtB,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC;QACjB,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;QAChB,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC;QACnB,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC;QAExB,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;QAClC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1E,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7E,KAAK,EAAE,CAAC;IACV,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7F,OAAO,KAAK,CAAC;AACf,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,GAAe;IAC7D,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;IAEzC,IAAI,CAAC,MAAM,CAAC,SAAS;QAAE,OAAO,CAAC,CAAC;IAEhC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,OAAO,GAAG,KAAK,CAAC,mBAAmB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAEtD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,wBAAwB;QACxB,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAEjC,kBAAkB;QAClB,MAAM,OAAO,GAAG,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,eAAe;QACf,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QACtB,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC;QACjB,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;QAChB,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC;QACnB,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC;QAExB,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;QAClC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1E,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7E,KAAK,EAAE,CAAC;QAER,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE;YACpC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrB,KAAK;YACL,UAAU,EAAE,KAAK,CAAC,UAAU;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,eAAe,CAAC,KAAa,EAAE,GAAe;IAC5D,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;IAElD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,QAAQ,CAAC,IAAI,CAAC,4BAA4B,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8BAA8B;AAC9B,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,GAAe;IACzD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AACzC,CAAC;AAED,gCAAgC;AAChC,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,GAAe,EAAE,MAA0B;IACrF,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;AACzD,CAAC;AAED,4BAA4B;AAC5B,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,GAAe;IACzD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;AAC7C,CAAC"}
|
|
@@ -20,6 +20,8 @@ export interface QueueManagerConfig {
|
|
|
20
20
|
cleanupIntervalMs?: number;
|
|
21
21
|
jobTimeoutCheckMs?: number;
|
|
22
22
|
dependencyCheckMs?: number;
|
|
23
|
+
stallCheckMs?: number;
|
|
24
|
+
dlqMaintenanceMs?: number;
|
|
23
25
|
}
|
|
24
26
|
/**
|
|
25
27
|
* QueueManager - Central coordinator
|
|
@@ -47,6 +49,8 @@ export declare class QueueManager {
|
|
|
47
49
|
private readonly startTime;
|
|
48
50
|
private cleanupInterval;
|
|
49
51
|
private timeoutInterval;
|
|
52
|
+
private stallCheckInterval;
|
|
53
|
+
private dlqMaintenanceInterval;
|
|
50
54
|
private readonly queueNamesCache;
|
|
51
55
|
constructor(config?: QueueManagerConfig);
|
|
52
56
|
private getPushContext;
|
|
@@ -56,6 +60,7 @@ export declare class QueueManager {
|
|
|
56
60
|
private handleRepeat;
|
|
57
61
|
private getJobMgmtContext;
|
|
58
62
|
private getQueryContext;
|
|
63
|
+
private getDlqContext;
|
|
59
64
|
push(queue: string, input: JobInput): Promise<Job>;
|
|
60
65
|
pushBatch(queue: string, inputs: JobInput[]): Promise<JobId[]>;
|
|
61
66
|
pull(queue: string, timeoutMs?: number): Promise<Job | null>;
|
|
@@ -140,6 +145,19 @@ export declare class QueueManager {
|
|
|
140
145
|
private processPendingDependencies;
|
|
141
146
|
private startBackgroundTasks;
|
|
142
147
|
private checkJobTimeouts;
|
|
148
|
+
/**
|
|
149
|
+
* Check for stalled jobs and handle them
|
|
150
|
+
* Stalled = active job with no heartbeat for too long
|
|
151
|
+
*/
|
|
152
|
+
private checkStalledJobs;
|
|
153
|
+
/**
|
|
154
|
+
* Handle a stalled job based on the action
|
|
155
|
+
*/
|
|
156
|
+
private handleStalledJob;
|
|
157
|
+
/**
|
|
158
|
+
* Perform DLQ maintenance: auto-retry and purge expired
|
|
159
|
+
*/
|
|
160
|
+
private performDlqMaintenance;
|
|
143
161
|
private recover;
|
|
144
162
|
private cleanup;
|
|
145
163
|
shutdown(): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queueManager.d.ts","sourceRoot":"","sources":["../../src/application/queueManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"queueManager.d.ts","sourceRoot":"","sources":["../../src/application/queueManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAGhE,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAO1D,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAmBhD,OAAO,EAAkC,KAAK,OAAO,EAAE,MAAM,eAAe,CAAC;AAE7E,kCAAkC;AAClC,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAeD;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgD;IACvE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;IAG/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgB;IAC3C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAyB;IAC1D,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgB;IAGhD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiC;IAC1D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA8B;IACzD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgC;IAGxD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoB;IACrD,OAAO,CAAC,gBAAgB,CAA+C;IAGvE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAG9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAG9C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAO;IAGrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAKtB;IACF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IAGxC,OAAO,CAAC,eAAe,CAA+C;IACtE,OAAO,CAAC,eAAe,CAA+C;IACtE,OAAO,CAAC,kBAAkB,CAA+C;IACzE,OAAO,CAAC,sBAAsB,CAA+C;IAG7E,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAqB;gBAEzC,MAAM,GAAE,kBAAuB;IAsC3C,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,aAAa;IAqBrB,8DAA8D;IAC9D,OAAO,CAAC,YAAY;IA0BpB,OAAO,CAAC,iBAAiB;IAYzB,OAAO,CAAC,eAAe;IAcvB,OAAO,CAAC,aAAa;IASf,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IAMlD,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAM9D,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAU,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAIrE,iFAAiF;IAC3E,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAU,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAI9E,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxD,6DAA6D;IACvD,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9C,0EAA0E;IACpE,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhF,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMjD,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAI/C,SAAS,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;IAIhC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI;IAI9C,WAAW,CAAC,KAAK,EAAE,KAAK;;;;IAIxB,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAM5B,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAI1B,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAI3B,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAIhC,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI5B,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAM/B,UAAU,IAAI,MAAM,EAAE;IAKtB,qEAAqE;IACrE,OAAO,CAAC,iBAAiB;IAIzB,8DAA8D;IAC9D,OAAO,CAAC,mBAAmB;IAI3B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM;IAY7E,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,GAAG,EAAE;IAI5C,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM;IAI9C,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAM/B,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAIhD,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAInC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAIlD,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAM/B,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC;IAItC,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIlF,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAI5D,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIhE,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC;IAIvC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAI5D,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC;IAM7C,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,GAAE,MAAM,GAAG,MAAM,GAAG,OAAgB,GAAG,OAAO;IAazF,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,WAAW,EAAE;IAQpC,SAAS,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAU7B,oBAAoB,IAAI,MAAM;IAM9B,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO;IAMrC,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAMjC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS;IAI1C,SAAS,IAAI,OAAO,EAAE;IAMtB,SAAS,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,IAAI;IAI1D,yDAAyD;IACzD,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAMvE,8CAA8C;IAC9C,WAAW,IAAI,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC;IAItC,uDAAuD;IACvD,gBAAgB,IAAI,OAAO,CAAC,KAAK,CAAC;IAIlC;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,eAAe;IAMvB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAQtB;;;;OAIG;YACW,0BAA0B;IAoExC,OAAO,CAAC,oBAAoB;IAqB5B,OAAO,CAAC,gBAAgB;IAgBxB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IA6BxB;;OAEG;YACW,gBAAgB;IAiE9B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAuB7B,OAAO,CAAC,OAAO;IAaf,OAAO,CAAC,OAAO;IAgGf,QAAQ,IAAI,IAAI;IA+BhB,QAAQ;;;;;;;;;;;;;;CAmCT"}
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
* Queue Manager
|
|
3
3
|
* Core orchestrator for all queue operations
|
|
4
4
|
*/
|
|
5
|
+
import { calculateBackoff } from '../domain/types/job';
|
|
5
6
|
import { queueLog } from '../shared/logger';
|
|
7
|
+
import { getStallAction, incrementStallCount } from '../domain/types/stall';
|
|
6
8
|
import { Shard } from '../domain/queue/shard';
|
|
7
9
|
import { SqliteStorage } from '../infrastructure/persistence/sqlite';
|
|
8
10
|
import { CronScheduler } from '../infrastructure/scheduler/cronScheduler';
|
|
@@ -30,6 +32,8 @@ const DEFAULT_CONFIG = {
|
|
|
30
32
|
cleanupIntervalMs: 10_000,
|
|
31
33
|
jobTimeoutCheckMs: 5_000,
|
|
32
34
|
dependencyCheckMs: 1_000,
|
|
35
|
+
stallCheckMs: 5_000,
|
|
36
|
+
dlqMaintenanceMs: 60_000,
|
|
33
37
|
};
|
|
34
38
|
/**
|
|
35
39
|
* QueueManager - Central coordinator
|
|
@@ -70,6 +74,8 @@ export class QueueManager {
|
|
|
70
74
|
// Background intervals
|
|
71
75
|
cleanupInterval = null;
|
|
72
76
|
timeoutInterval = null;
|
|
77
|
+
stallCheckInterval = null;
|
|
78
|
+
dlqMaintenanceInterval = null;
|
|
73
79
|
// Queue names cache for O(1) listQueues instead of O(32 * queues)
|
|
74
80
|
queueNamesCache = new Set();
|
|
75
81
|
constructor(config = {}) {
|
|
@@ -197,6 +203,12 @@ export class QueueManager {
|
|
|
197
203
|
customIdMap: this.customIdMap,
|
|
198
204
|
};
|
|
199
205
|
}
|
|
206
|
+
getDlqContext() {
|
|
207
|
+
return {
|
|
208
|
+
shards: this.shards,
|
|
209
|
+
jobIndex: this.jobIndex,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
200
212
|
// ============ Core Operations ============
|
|
201
213
|
async push(queue, input) {
|
|
202
214
|
// Register queue name in cache for O(1) listQueues
|
|
@@ -487,6 +499,12 @@ export class QueueManager {
|
|
|
487
499
|
queueLog.error('Dependency check failed', { error: String(err) });
|
|
488
500
|
});
|
|
489
501
|
}, this.config.dependencyCheckMs);
|
|
502
|
+
this.stallCheckInterval = setInterval(() => {
|
|
503
|
+
this.checkStalledJobs();
|
|
504
|
+
}, this.config.stallCheckMs);
|
|
505
|
+
this.dlqMaintenanceInterval = setInterval(() => {
|
|
506
|
+
this.performDlqMaintenance();
|
|
507
|
+
}, this.config.dlqMaintenanceMs);
|
|
490
508
|
this.cronScheduler.start();
|
|
491
509
|
}
|
|
492
510
|
checkJobTimeouts() {
|
|
@@ -504,6 +522,118 @@ export class QueueManager {
|
|
|
504
522
|
}
|
|
505
523
|
}
|
|
506
524
|
}
|
|
525
|
+
/**
|
|
526
|
+
* Check for stalled jobs and handle them
|
|
527
|
+
* Stalled = active job with no heartbeat for too long
|
|
528
|
+
*/
|
|
529
|
+
checkStalledJobs() {
|
|
530
|
+
const now = Date.now();
|
|
531
|
+
for (let i = 0; i < SHARD_COUNT; i++) {
|
|
532
|
+
const procShard = this.processingShards[i];
|
|
533
|
+
const stalledJobs = [];
|
|
534
|
+
for (const [_jobId, job] of procShard) {
|
|
535
|
+
const stallConfig = this.shards[shardIndex(job.queue)].getStallConfig(job.queue);
|
|
536
|
+
if (!stallConfig.enabled)
|
|
537
|
+
continue;
|
|
538
|
+
const action = getStallAction(job, stallConfig, now);
|
|
539
|
+
if (action !== "keep" /* StallAction.Keep */) {
|
|
540
|
+
stalledJobs.push({ job, action });
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
// Process stalled jobs
|
|
544
|
+
for (const { job, action } of stalledJobs) {
|
|
545
|
+
this.handleStalledJob(job, action).catch((err) => {
|
|
546
|
+
queueLog.error('Failed to handle stalled job', {
|
|
547
|
+
jobId: String(job.id),
|
|
548
|
+
error: String(err),
|
|
549
|
+
});
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
/**
|
|
555
|
+
* Handle a stalled job based on the action
|
|
556
|
+
*/
|
|
557
|
+
async handleStalledJob(job, action) {
|
|
558
|
+
const idx = shardIndex(job.queue);
|
|
559
|
+
const shard = this.shards[idx];
|
|
560
|
+
const procIdx = Number(BigInt(job.id.replace(/-/g, '').slice(0, 8)) & BigInt(SHARD_COUNT - 1));
|
|
561
|
+
// Emit stalled event
|
|
562
|
+
this.eventsManager.broadcast({
|
|
563
|
+
eventType: "stalled" /* EventType.Stalled */,
|
|
564
|
+
queue: job.queue,
|
|
565
|
+
jobId: job.id,
|
|
566
|
+
timestamp: Date.now(),
|
|
567
|
+
data: { stallCount: job.stallCount + 1, action },
|
|
568
|
+
});
|
|
569
|
+
void this.webhookManager.trigger('stalled', String(job.id), job.queue, {
|
|
570
|
+
data: { stallCount: job.stallCount + 1, action },
|
|
571
|
+
});
|
|
572
|
+
if (action === "move_to_dlq" /* StallAction.MoveToDlq */) {
|
|
573
|
+
// Max stalls reached - move to DLQ
|
|
574
|
+
queueLog.warn('Job exceeded max stalls, moving to DLQ', {
|
|
575
|
+
jobId: String(job.id),
|
|
576
|
+
queue: job.queue,
|
|
577
|
+
stallCount: job.stallCount,
|
|
578
|
+
});
|
|
579
|
+
// Remove from processing
|
|
580
|
+
this.processingShards[procIdx].delete(job.id);
|
|
581
|
+
shard.releaseConcurrency(job.queue);
|
|
582
|
+
// Add to DLQ with stalled reason
|
|
583
|
+
shard.addToDlq(job, "stalled" /* FailureReason.Stalled */, `Job stalled ${job.stallCount + 1} times`);
|
|
584
|
+
this.jobIndex.set(job.id, { type: 'dlq', queueName: job.queue });
|
|
585
|
+
// Persist
|
|
586
|
+
this.storage?.markFailed(job, `Job stalled ${job.stallCount + 1} times`);
|
|
587
|
+
}
|
|
588
|
+
else {
|
|
589
|
+
// Retry - increment stall count and re-queue
|
|
590
|
+
incrementStallCount(job);
|
|
591
|
+
job.attempts++;
|
|
592
|
+
job.startedAt = null;
|
|
593
|
+
job.runAt = Date.now() + calculateBackoff(job);
|
|
594
|
+
job.lastHeartbeat = Date.now();
|
|
595
|
+
queueLog.warn('Job stalled, retrying', {
|
|
596
|
+
jobId: String(job.id),
|
|
597
|
+
queue: job.queue,
|
|
598
|
+
stallCount: job.stallCount,
|
|
599
|
+
attempt: job.attempts,
|
|
600
|
+
});
|
|
601
|
+
// Remove from processing
|
|
602
|
+
this.processingShards[procIdx].delete(job.id);
|
|
603
|
+
shard.releaseConcurrency(job.queue);
|
|
604
|
+
// Re-queue
|
|
605
|
+
shard.getQueue(job.queue).push(job);
|
|
606
|
+
const isDelayed = job.runAt > Date.now();
|
|
607
|
+
shard.incrementQueued(job.id, isDelayed, job.createdAt, job.queue, job.runAt);
|
|
608
|
+
this.jobIndex.set(job.id, { type: 'queue', shardIdx: idx, queueName: job.queue });
|
|
609
|
+
// Persist
|
|
610
|
+
this.storage?.updateForRetry(job);
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
/**
|
|
614
|
+
* Perform DLQ maintenance: auto-retry and purge expired
|
|
615
|
+
*/
|
|
616
|
+
performDlqMaintenance() {
|
|
617
|
+
const ctx = this.getDlqContext();
|
|
618
|
+
// Process each queue
|
|
619
|
+
for (const queueName of this.queueNamesCache) {
|
|
620
|
+
try {
|
|
621
|
+
// Auto-retry eligible entries
|
|
622
|
+
const retried = dlqOps.processAutoRetry(queueName, ctx);
|
|
623
|
+
if (retried > 0) {
|
|
624
|
+
queueLog.info('DLQ auto-retry completed', { queue: queueName, retried });
|
|
625
|
+
}
|
|
626
|
+
// Purge expired entries
|
|
627
|
+
const purged = dlqOps.purgeExpiredDlq(queueName, ctx);
|
|
628
|
+
if (purged > 0) {
|
|
629
|
+
queueLog.info('DLQ purge completed', { queue: queueName, purged });
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
catch (err) {
|
|
633
|
+
queueLog.error('DLQ maintenance failed', { queue: queueName, error: String(err) });
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
}
|
|
507
637
|
recover() {
|
|
508
638
|
if (!this.storage)
|
|
509
639
|
return;
|
|
@@ -616,6 +746,10 @@ export class QueueManager {
|
|
|
616
746
|
clearInterval(this.timeoutInterval);
|
|
617
747
|
if (this.depCheckInterval)
|
|
618
748
|
clearInterval(this.depCheckInterval);
|
|
749
|
+
if (this.stallCheckInterval)
|
|
750
|
+
clearInterval(this.stallCheckInterval);
|
|
751
|
+
if (this.dlqMaintenanceInterval)
|
|
752
|
+
clearInterval(this.dlqMaintenanceInterval);
|
|
619
753
|
this.storage?.close();
|
|
620
754
|
// Clear in-memory collections
|
|
621
755
|
this.jobIndex.clear();
|