bunqueue 1.9.7 → 1.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/backgroundTasks.d.ts +3 -6
- package/dist/application/backgroundTasks.d.ts.map +1 -1
- package/dist/application/backgroundTasks.js +10 -179
- package/dist/application/backgroundTasks.js.map +1 -1
- package/dist/application/cleanupTasks.d.ts +1 -1
- package/dist/application/cleanupTasks.d.ts.map +1 -1
- package/dist/application/cleanupTasks.js +61 -22
- package/dist/application/cleanupTasks.js.map +1 -1
- package/dist/application/clientTracking.d.ts +22 -0
- package/dist/application/clientTracking.d.ts.map +1 -0
- package/dist/application/clientTracking.js +118 -0
- package/dist/application/clientTracking.js.map +1 -0
- package/dist/application/contextFactory.d.ts +97 -0
- package/dist/application/contextFactory.d.ts.map +1 -0
- package/dist/application/contextFactory.js +170 -0
- package/dist/application/contextFactory.js.map +1 -0
- package/dist/application/dependencyProcessor.d.ts +11 -0
- package/dist/application/dependencyProcessor.d.ts.map +1 -0
- package/dist/application/dependencyProcessor.js +69 -0
- package/dist/application/dependencyProcessor.js.map +1 -0
- package/dist/application/dlqManager.d.ts +12 -0
- package/dist/application/dlqManager.d.ts.map +1 -1
- package/dist/application/dlqManager.js +36 -0
- package/dist/application/dlqManager.js.map +1 -1
- package/dist/application/lockManager.d.ts +3 -49
- package/dist/application/lockManager.d.ts.map +1 -1
- package/dist/application/lockManager.js +101 -272
- package/dist/application/lockManager.js.map +1 -1
- package/dist/application/lockOperations.d.ts +39 -0
- package/dist/application/lockOperations.d.ts.map +1 -0
- package/dist/application/lockOperations.js +101 -0
- package/dist/application/lockOperations.js.map +1 -0
- package/dist/application/operations/ack.d.ts +1 -5
- package/dist/application/operations/ack.d.ts.map +1 -1
- package/dist/application/operations/ack.js +43 -259
- package/dist/application/operations/ack.js.map +1 -1
- package/dist/application/operations/ackHelpers.d.ts +79 -0
- package/dist/application/operations/ackHelpers.d.ts.map +1 -0
- package/dist/application/operations/ackHelpers.js +173 -0
- package/dist/application/operations/ackHelpers.js.map +1 -0
- package/dist/application/operations/jobManagement.d.ts +2 -0
- package/dist/application/operations/jobManagement.d.ts.map +1 -1
- package/dist/application/operations/jobManagement.js +8 -0
- package/dist/application/operations/jobManagement.js.map +1 -1
- package/dist/application/operations/push.d.ts.map +1 -1
- package/dist/application/operations/push.js +20 -6
- package/dist/application/operations/push.js.map +1 -1
- package/dist/application/operations/queryOperations.d.ts +11 -0
- package/dist/application/operations/queryOperations.d.ts.map +1 -1
- package/dist/application/operations/queryOperations.js +32 -0
- package/dist/application/operations/queryOperations.js.map +1 -1
- package/dist/application/queueManager.d.ts +3 -11
- package/dist/application/queueManager.d.ts.map +1 -1
- package/dist/application/queueManager.js +98 -244
- package/dist/application/queueManager.js.map +1 -1
- package/dist/application/stallDetection.d.ts +11 -0
- package/dist/application/stallDetection.d.ts.map +1 -0
- package/dist/application/stallDetection.js +128 -0
- package/dist/application/stallDetection.js.map +1 -0
- package/dist/application/types.js +1 -1
- package/dist/application/types.js.map +1 -1
- package/dist/cli/client.d.ts +3 -5
- package/dist/cli/client.d.ts.map +1 -1
- package/dist/cli/client.js +31 -27
- package/dist/cli/client.js.map +1 -1
- package/dist/cli/commands/core.js +3 -3
- package/dist/cli/commands/core.js.map +1 -1
- package/dist/cli/commands/job.js +14 -14
- package/dist/cli/commands/job.js.map +1 -1
- package/dist/cli/commands/server.d.ts.map +1 -1
- package/dist/cli/commands/server.js +5 -29
- package/dist/cli/commands/server.js.map +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +1 -9
- package/dist/cli/index.js.map +1 -1
- package/dist/client/events.d.ts +0 -1
- package/dist/client/events.d.ts.map +1 -1
- package/dist/client/events.js +4 -7
- package/dist/client/events.js.map +1 -1
- package/dist/client/flow.d.ts +23 -1
- package/dist/client/flow.d.ts.map +1 -1
- package/dist/client/flow.js +166 -68
- package/dist/client/flow.js.map +1 -1
- package/dist/client/queue/queue.d.ts.map +1 -1
- package/dist/client/queue/queue.js +3 -1
- package/dist/client/queue/queue.js.map +1 -1
- package/dist/client/sandboxed/types.d.ts +1 -0
- package/dist/client/sandboxed/types.d.ts.map +1 -1
- package/dist/client/sandboxed/worker.d.ts +1 -0
- package/dist/client/sandboxed/worker.d.ts.map +1 -1
- package/dist/client/sandboxed/worker.js +31 -8
- package/dist/client/sandboxed/worker.js.map +1 -1
- package/dist/client/sandboxed/wrapper.d.ts.map +1 -1
- package/dist/client/sandboxed/wrapper.js +10 -1
- package/dist/client/sandboxed/wrapper.js.map +1 -1
- package/dist/client/tcp/client.d.ts +4 -1
- package/dist/client/tcp/client.d.ts.map +1 -1
- package/dist/client/tcp/client.js +26 -8
- package/dist/client/tcp/client.js.map +1 -1
- package/dist/client/tcp/connection.d.ts +6 -8
- package/dist/client/tcp/connection.d.ts.map +1 -1
- package/dist/client/tcp/connection.js +24 -22
- package/dist/client/tcp/connection.js.map +1 -1
- package/dist/client/tcp/index.d.ts +0 -1
- package/dist/client/tcp/index.d.ts.map +1 -1
- package/dist/client/tcp/index.js +0 -1
- package/dist/client/tcp/index.js.map +1 -1
- package/dist/client/tcp/types.d.ts +8 -13
- package/dist/client/tcp/types.d.ts.map +1 -1
- package/dist/client/tcp/types.js +0 -1
- package/dist/client/tcp/types.js.map +1 -1
- package/dist/client/tcpPool.d.ts.map +1 -1
- package/dist/client/tcpPool.js +0 -6
- package/dist/client/tcpPool.js.map +1 -1
- package/dist/client/worker/ackBatcher.d.ts +1 -1
- package/dist/client/worker/ackBatcher.d.ts.map +1 -1
- package/dist/client/worker/ackBatcher.js +20 -18
- package/dist/client/worker/ackBatcher.js.map +1 -1
- package/dist/client/worker/jobParser.d.ts.map +1 -1
- package/dist/client/worker/jobParser.js +8 -7
- package/dist/client/worker/jobParser.js.map +1 -1
- package/dist/client/worker/processor.d.ts.map +1 -1
- package/dist/client/worker/processor.js +10 -6
- package/dist/client/worker/processor.js.map +1 -1
- package/dist/domain/queue/dependencyTracker.d.ts +74 -0
- package/dist/domain/queue/dependencyTracker.d.ts.map +1 -0
- package/dist/domain/queue/dependencyTracker.js +126 -0
- package/dist/domain/queue/dependencyTracker.js.map +1 -0
- package/dist/domain/queue/dlqShard.d.ts +61 -0
- package/dist/domain/queue/dlqShard.d.ts.map +1 -0
- package/dist/domain/queue/dlqShard.js +175 -0
- package/dist/domain/queue/dlqShard.js.map +1 -0
- package/dist/domain/queue/limiterManager.d.ts +44 -0
- package/dist/domain/queue/limiterManager.d.ts.map +1 -0
- package/dist/domain/queue/limiterManager.js +99 -0
- package/dist/domain/queue/limiterManager.js.map +1 -0
- package/dist/domain/queue/shard.d.ts +33 -124
- package/dist/domain/queue/shard.d.ts.map +1 -1
- package/dist/domain/queue/shard.js +157 -427
- package/dist/domain/queue/shard.js.map +1 -1
- package/dist/domain/queue/temporalManager.d.ts +82 -0
- package/dist/domain/queue/temporalManager.d.ts.map +1 -0
- package/dist/domain/queue/temporalManager.js +150 -0
- package/dist/domain/queue/temporalManager.js.map +1 -0
- package/dist/domain/queue/uniqueKeyManager.d.ts +32 -0
- package/dist/domain/queue/uniqueKeyManager.d.ts.map +1 -0
- package/dist/domain/queue/uniqueKeyManager.js +87 -0
- package/dist/domain/queue/uniqueKeyManager.js.map +1 -0
- package/dist/domain/types/command.d.ts +6 -0
- package/dist/domain/types/command.d.ts.map +1 -1
- package/dist/infrastructure/backup/s3Backup.d.ts +3 -40
- package/dist/infrastructure/backup/s3Backup.d.ts.map +1 -1
- package/dist/infrastructure/backup/s3Backup.js +10 -182
- package/dist/infrastructure/backup/s3Backup.js.map +1 -1
- package/dist/infrastructure/backup/s3BackupConfig.d.ts +67 -0
- package/dist/infrastructure/backup/s3BackupConfig.d.ts.map +1 -0
- package/dist/infrastructure/backup/s3BackupConfig.js +48 -0
- package/dist/infrastructure/backup/s3BackupConfig.js.map +1 -0
- package/dist/infrastructure/backup/s3BackupOperations.d.ts +23 -0
- package/dist/infrastructure/backup/s3BackupOperations.d.ts.map +1 -0
- package/dist/infrastructure/backup/s3BackupOperations.js +170 -0
- package/dist/infrastructure/backup/s3BackupOperations.js.map +1 -0
- package/dist/infrastructure/persistence/sqlite.d.ts +6 -13
- package/dist/infrastructure/persistence/sqlite.d.ts.map +1 -1
- package/dist/infrastructure/persistence/sqlite.js +28 -179
- package/dist/infrastructure/persistence/sqlite.js.map +1 -1
- package/dist/infrastructure/persistence/sqliteBatch.d.ts +38 -0
- package/dist/infrastructure/persistence/sqliteBatch.d.ts.map +1 -0
- package/dist/infrastructure/persistence/sqliteBatch.js +124 -0
- package/dist/infrastructure/persistence/sqliteBatch.js.map +1 -0
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts +17 -0
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts.map +1 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +81 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js.map +1 -0
- package/dist/infrastructure/persistence/statements.d.ts +1 -1
- package/dist/infrastructure/persistence/statements.d.ts.map +1 -1
- package/dist/infrastructure/persistence/statements.js +3 -2
- package/dist/infrastructure/persistence/statements.js.map +1 -1
- package/dist/infrastructure/scheduler/cronScheduler.d.ts +7 -0
- package/dist/infrastructure/scheduler/cronScheduler.d.ts.map +1 -1
- package/dist/infrastructure/scheduler/cronScheduler.js +23 -3
- package/dist/infrastructure/scheduler/cronScheduler.js.map +1 -1
- package/dist/infrastructure/server/handler.d.ts.map +1 -1
- package/dist/infrastructure/server/handler.js +1 -186
- package/dist/infrastructure/server/handler.js.map +1 -1
- package/dist/infrastructure/server/handlerRoutes.d.ts +23 -0
- package/dist/infrastructure/server/handlerRoutes.d.ts.map +1 -0
- package/dist/infrastructure/server/handlerRoutes.js +190 -0
- package/dist/infrastructure/server/handlerRoutes.js.map +1 -0
- package/dist/infrastructure/server/handlers/core.d.ts.map +1 -1
- package/dist/infrastructure/server/handlers/core.js +26 -19
- package/dist/infrastructure/server/handlers/core.js.map +1 -1
- package/dist/infrastructure/server/http.d.ts +4 -25
- package/dist/infrastructure/server/http.d.ts.map +1 -1
- package/dist/infrastructure/server/http.js +68 -285
- package/dist/infrastructure/server/http.js.map +1 -1
- package/dist/infrastructure/server/httpEndpoints.d.ts +19 -0
- package/dist/infrastructure/server/httpEndpoints.d.ts.map +1 -0
- package/dist/infrastructure/server/httpEndpoints.js +151 -0
- package/dist/infrastructure/server/httpEndpoints.js.map +1 -0
- package/dist/infrastructure/server/protocol.d.ts +15 -1
- package/dist/infrastructure/server/protocol.d.ts.map +1 -1
- package/dist/infrastructure/server/protocol.js +37 -3
- package/dist/infrastructure/server/protocol.js.map +1 -1
- package/dist/infrastructure/server/sseHandler.d.ts +27 -0
- package/dist/infrastructure/server/sseHandler.d.ts.map +1 -0
- package/dist/infrastructure/server/sseHandler.js +77 -0
- package/dist/infrastructure/server/sseHandler.js.map +1 -0
- package/dist/infrastructure/server/tcp.d.ts +8 -10
- package/dist/infrastructure/server/tcp.d.ts.map +1 -1
- package/dist/infrastructure/server/tcp.js +51 -42
- package/dist/infrastructure/server/tcp.js.map +1 -1
- package/dist/infrastructure/server/wsHandler.d.ts +31 -0
- package/dist/infrastructure/server/wsHandler.d.ts.map +1 -0
- package/dist/infrastructure/server/wsHandler.js +63 -0
- package/dist/infrastructure/server/wsHandler.js.map +1 -0
- package/dist/main.js +2 -4
- package/dist/main.js.map +1 -1
- package/dist/mcp/index.js +3 -465
- package/dist/mcp/index.js.map +1 -1
- package/dist/mcp/mcpHandlers.d.ts +129 -0
- package/dist/mcp/mcpHandlers.d.ts.map +1 -0
- package/dist/mcp/mcpHandlers.js +204 -0
- package/dist/mcp/mcpHandlers.js.map +1 -0
- package/dist/mcp/mcpTools.d.ts +15 -0
- package/dist/mcp/mcpTools.d.ts.map +1 -0
- package/dist/mcp/mcpTools.js +277 -0
- package/dist/mcp/mcpTools.js.map +1 -0
- package/dist/shared/lru.d.ts +23 -0
- package/dist/shared/lru.d.ts.map +1 -1
- package/dist/shared/lru.js +61 -3
- package/dist/shared/lru.js.map +1 -1
- package/dist/shared/skipList.d.ts +10 -2
- package/dist/shared/skipList.d.ts.map +1 -1
- package/dist/shared/skipList.js +22 -1
- package/dist/shared/skipList.js.map +1 -1
- package/package.json +2 -2
- package/dist/cli/dashboard.d.ts +0 -32
- package/dist/cli/dashboard.d.ts.map +0 -1
- package/dist/cli/dashboard.js +0 -183
- package/dist/cli/dashboard.js.map +0 -1
- package/dist/client/tcp/lineBuffer.d.ts +0 -17
- package/dist/client/tcp/lineBuffer.d.ts.map +0 -1
- package/dist/client/tcp/lineBuffer.js +0 -32
- package/dist/client/tcp/lineBuffer.js.map +0 -1
|
@@ -1,14 +1,20 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shard - Container for queues within a shard
|
|
3
3
|
* Each shard manages multiple queues and their state
|
|
4
|
+
*
|
|
5
|
+
* Refactored to compose smaller modules:
|
|
6
|
+
* - UniqueKeyManager: deduplication with TTL
|
|
7
|
+
* - DlqShard: Dead Letter Queue operations
|
|
8
|
+
* - LimiterManager: rate limiting + concurrency
|
|
9
|
+
* - DependencyTracker: job dependency tracking
|
|
10
|
+
* - TemporalManager: temporal index + delayed job tracking
|
|
4
11
|
*/
|
|
5
|
-
import { createQueueState, RateLimiter, ConcurrencyLimiter } from '../types/queue';
|
|
6
|
-
import { DEFAULT_DLQ_CONFIG, createDlqEntry, isDlqEntryExpired, canAutoRetry, } from '../types/dlq';
|
|
7
|
-
import { DEFAULT_STALL_CONFIG } from '../types/stall';
|
|
8
|
-
import { isUniqueKeyExpired, calculateExpiration } from '../types/deduplication';
|
|
9
12
|
import { IndexedPriorityQueue } from './priorityQueue';
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
13
|
+
import { UniqueKeyManager } from './uniqueKeyManager';
|
|
14
|
+
import { DlqShard } from './dlqShard';
|
|
15
|
+
import { LimiterManager } from './limiterManager';
|
|
16
|
+
import { DependencyTracker } from './dependencyTracker';
|
|
17
|
+
import { TemporalManager } from './temporalManager';
|
|
12
18
|
/**
|
|
13
19
|
* Shard contains:
|
|
14
20
|
* - Queues (waiting + delayed jobs)
|
|
@@ -20,56 +26,38 @@ import { MinHeap } from '../../shared/minHeap';
|
|
|
20
26
|
export class Shard {
|
|
21
27
|
/** Priority queues by queue name */
|
|
22
28
|
queues = new Map();
|
|
23
|
-
/**
|
|
24
|
-
|
|
25
|
-
/** DLQ
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
|
|
29
|
+
/** Unique key manager for deduplication */
|
|
30
|
+
uniqueKeyManager = new UniqueKeyManager();
|
|
31
|
+
/** DLQ manager */
|
|
32
|
+
dlqManager;
|
|
33
|
+
/** Limiter manager for rate/concurrency control */
|
|
34
|
+
limiterManager = new LimiterManager();
|
|
35
|
+
/** Dependency tracker */
|
|
36
|
+
dependencyTracker = new DependencyTracker();
|
|
37
|
+
/** Temporal manager for index and delayed jobs */
|
|
38
|
+
temporalManager = new TemporalManager();
|
|
29
39
|
/** Running counters for O(1) stats - updated on every operation */
|
|
30
40
|
stats = {
|
|
31
41
|
queuedJobs: 0,
|
|
32
42
|
delayedJobs: 0,
|
|
33
43
|
dlqJobs: 0,
|
|
34
44
|
};
|
|
35
|
-
/** Set of delayed job IDs for tracking when they become ready */
|
|
36
|
-
delayedJobIds = new Set();
|
|
37
|
-
/**
|
|
38
|
-
* Min-heap of delayed jobs ordered by runAt for O(k) refresh
|
|
39
|
-
* Instead of O(n × queues) iteration
|
|
40
|
-
*/
|
|
41
|
-
delayedHeap = new MinHeap((a, b) => a.runAt - b.runAt);
|
|
42
|
-
/** Map from jobId to current runAt for stale detection in delayedHeap */
|
|
43
|
-
delayedRunAt = new Map();
|
|
44
|
-
/**
|
|
45
|
-
* Temporal index: Skip List for O(log n) insert/delete instead of O(n) splice
|
|
46
|
-
* Ordered by createdAt for efficient cleanQueue range queries
|
|
47
|
-
*/
|
|
48
|
-
temporalIndex = new SkipList((a, b) => a.createdAt - b.createdAt);
|
|
49
|
-
/** Unique keys per queue for deduplication (with TTL support) */
|
|
50
|
-
uniqueKeys = new Map();
|
|
51
|
-
/** Jobs waiting for dependencies */
|
|
52
|
-
waitingDeps = new Map();
|
|
53
|
-
/**
|
|
54
|
-
* Reverse index: depId -> Set of jobIds waiting for that dependency
|
|
55
|
-
* Enables O(1) lookup when a dependency completes instead of O(n) scan
|
|
56
|
-
*/
|
|
57
|
-
dependencyIndex = new Map();
|
|
58
|
-
/** Parent jobs waiting for children to complete */
|
|
59
|
-
waitingChildren = new Map();
|
|
60
|
-
/** Queue state (pause, rate limit, concurrency) */
|
|
61
|
-
queueState = new Map();
|
|
62
45
|
/** Active FIFO groups per queue */
|
|
63
46
|
activeGroups = new Map();
|
|
64
|
-
/** Rate limiters per queue */
|
|
65
|
-
rateLimiters = new Map();
|
|
66
|
-
/** Concurrency limiters per queue */
|
|
67
|
-
concurrencyLimiters = new Map();
|
|
68
47
|
/** Waiter entry with cancellation flag for O(1) cleanup */
|
|
69
48
|
waiters = [];
|
|
49
|
+
constructor() {
|
|
50
|
+
this.dlqManager = new DlqShard({
|
|
51
|
+
incrementDlq: () => {
|
|
52
|
+
this.incrementDlq();
|
|
53
|
+
},
|
|
54
|
+
decrementDlq: (count) => {
|
|
55
|
+
this.decrementDlq(count);
|
|
56
|
+
},
|
|
57
|
+
});
|
|
58
|
+
}
|
|
70
59
|
/** Notify that jobs are available - wakes first non-cancelled waiter */
|
|
71
60
|
notify() {
|
|
72
|
-
// Skip cancelled entries at head - O(k) where k = cancelled
|
|
73
61
|
while (this.waiters.length > 0) {
|
|
74
62
|
const waiter = this.waiters.shift();
|
|
75
63
|
if (!waiter.cancelled) {
|
|
@@ -80,26 +68,21 @@ export class Shard {
|
|
|
80
68
|
}
|
|
81
69
|
/** Wait for a job to become available (with timeout) */
|
|
82
70
|
waitForJob(timeoutMs) {
|
|
83
|
-
if (timeoutMs <= 0)
|
|
71
|
+
if (timeoutMs <= 0)
|
|
84
72
|
return Promise.resolve();
|
|
85
|
-
}
|
|
86
73
|
return new Promise((resolve) => {
|
|
87
74
|
const waiter = { resolve, cancelled: false };
|
|
88
75
|
const cleanup = () => {
|
|
89
76
|
if (waiter.cancelled)
|
|
90
77
|
return;
|
|
91
|
-
// O(1) cancellation - just mark, don't search/splice
|
|
92
78
|
waiter.cancelled = true;
|
|
93
79
|
resolve();
|
|
94
80
|
};
|
|
95
|
-
// Add to waiters
|
|
96
81
|
this.waiters.push(waiter);
|
|
97
|
-
|
|
98
|
-
setTimeout(cleanup, Math.min(timeoutMs, 100)); // Max 100ms wait to allow checking other conditions
|
|
82
|
+
setTimeout(cleanup, Math.min(timeoutMs, 100));
|
|
99
83
|
});
|
|
100
84
|
}
|
|
101
85
|
// ============ Queue Operations ============
|
|
102
|
-
/** Get or create queue */
|
|
103
86
|
getQueue(name) {
|
|
104
87
|
let queue = this.queues.get(name);
|
|
105
88
|
if (!queue) {
|
|
@@ -108,103 +91,48 @@ export class Shard {
|
|
|
108
91
|
}
|
|
109
92
|
return queue;
|
|
110
93
|
}
|
|
111
|
-
/** Get queue state */
|
|
112
94
|
getState(name) {
|
|
113
|
-
|
|
114
|
-
if (!state) {
|
|
115
|
-
state = createQueueState(name);
|
|
116
|
-
this.queueState.set(name, state);
|
|
117
|
-
}
|
|
118
|
-
return state;
|
|
95
|
+
return this.limiterManager.getState(name);
|
|
119
96
|
}
|
|
120
|
-
/** Check if queue is paused */
|
|
121
97
|
isPaused(name) {
|
|
122
|
-
return this.
|
|
98
|
+
return this.limiterManager.isPaused(name);
|
|
123
99
|
}
|
|
124
|
-
/** Pause queue */
|
|
125
100
|
pause(name) {
|
|
126
|
-
this.
|
|
101
|
+
this.limiterManager.pause(name);
|
|
127
102
|
}
|
|
128
|
-
/** Resume queue */
|
|
129
103
|
resume(name) {
|
|
130
|
-
this.
|
|
104
|
+
this.limiterManager.resume(name);
|
|
131
105
|
this.notify();
|
|
132
106
|
}
|
|
133
|
-
// ============ Unique Key Management ============
|
|
134
|
-
/** Check if unique key is available (not registered or expired) */
|
|
107
|
+
// ============ Unique Key Management (delegated) ============
|
|
135
108
|
isUniqueAvailable(queue, key) {
|
|
136
|
-
|
|
137
|
-
if (!entry)
|
|
138
|
-
return true;
|
|
139
|
-
// Check if expired
|
|
140
|
-
if (isUniqueKeyExpired(entry)) {
|
|
141
|
-
// Clean up expired entry
|
|
142
|
-
this.uniqueKeys.get(queue)?.delete(key);
|
|
143
|
-
return true;
|
|
144
|
-
}
|
|
145
|
-
return false;
|
|
109
|
+
return this.uniqueKeyManager.isAvailable(queue, key);
|
|
146
110
|
}
|
|
147
|
-
/** Get unique key entry (returns null if not found or expired) */
|
|
148
111
|
getUniqueKeyEntry(queue, key) {
|
|
149
|
-
|
|
150
|
-
if (!entry)
|
|
151
|
-
return null;
|
|
152
|
-
if (isUniqueKeyExpired(entry)) {
|
|
153
|
-
this.uniqueKeys.get(queue)?.delete(key);
|
|
154
|
-
return null;
|
|
155
|
-
}
|
|
156
|
-
return entry;
|
|
112
|
+
return this.uniqueKeyManager.getEntry(queue, key);
|
|
157
113
|
}
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
this.registerUniqueKeyWithTtl(queue, key, undefined, undefined);
|
|
114
|
+
registerUniqueKey(queue, key, jobId) {
|
|
115
|
+
this.uniqueKeyManager.register(queue, key, jobId);
|
|
161
116
|
}
|
|
162
|
-
/** Register unique key with TTL support */
|
|
163
117
|
registerUniqueKeyWithTtl(queue, key, jobId, ttl) {
|
|
164
|
-
|
|
165
|
-
if (!keys) {
|
|
166
|
-
keys = new Map();
|
|
167
|
-
this.uniqueKeys.set(queue, keys);
|
|
168
|
-
}
|
|
169
|
-
const now = Date.now();
|
|
170
|
-
keys.set(key, {
|
|
171
|
-
jobId: jobId ?? '',
|
|
172
|
-
expiresAt: calculateExpiration(ttl, now),
|
|
173
|
-
registeredAt: now,
|
|
174
|
-
});
|
|
118
|
+
this.uniqueKeyManager.registerWithTtl(queue, key, jobId, ttl);
|
|
175
119
|
}
|
|
176
|
-
/** Extend TTL for an existing unique key */
|
|
177
120
|
extendUniqueKeyTtl(queue, key, ttl) {
|
|
178
|
-
|
|
179
|
-
if (!entry)
|
|
180
|
-
return false;
|
|
181
|
-
entry.expiresAt = calculateExpiration(ttl);
|
|
182
|
-
return true;
|
|
121
|
+
return this.uniqueKeyManager.extendTtl(queue, key, ttl);
|
|
183
122
|
}
|
|
184
|
-
/** Release unique key */
|
|
185
123
|
releaseUniqueKey(queue, key) {
|
|
186
|
-
this.
|
|
124
|
+
this.uniqueKeyManager.release(queue, key);
|
|
187
125
|
}
|
|
188
|
-
/** Clean expired unique keys (call periodically) */
|
|
189
126
|
cleanExpiredUniqueKeys() {
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
if (isUniqueKeyExpired(entry, now)) {
|
|
195
|
-
keys.delete(key);
|
|
196
|
-
cleaned++;
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
return cleaned;
|
|
127
|
+
return this.uniqueKeyManager.cleanExpired();
|
|
128
|
+
}
|
|
129
|
+
get uniqueKeys() {
|
|
130
|
+
return this.uniqueKeyManager.getMap();
|
|
201
131
|
}
|
|
202
132
|
// ============ FIFO Group Management ============
|
|
203
|
-
/** Check if FIFO group is active */
|
|
204
133
|
isGroupActive(queue, groupId) {
|
|
205
134
|
return this.activeGroups.get(queue)?.has(groupId) ?? false;
|
|
206
135
|
}
|
|
207
|
-
/** Mark FIFO group as active */
|
|
208
136
|
activateGroup(queue, groupId) {
|
|
209
137
|
let groups = this.activeGroups.get(queue);
|
|
210
138
|
if (!groups) {
|
|
@@ -213,251 +141,151 @@ export class Shard {
|
|
|
213
141
|
}
|
|
214
142
|
groups.add(groupId);
|
|
215
143
|
}
|
|
216
|
-
/** Release FIFO group */
|
|
217
144
|
releaseGroup(queue, groupId) {
|
|
218
145
|
this.activeGroups.get(queue)?.delete(groupId);
|
|
219
146
|
}
|
|
220
|
-
// ============ Rate & Concurrency Limiting ============
|
|
221
|
-
/** Set rate limit for queue */
|
|
147
|
+
// ============ Rate & Concurrency Limiting (delegated) ============
|
|
222
148
|
setRateLimit(queue, limit) {
|
|
223
|
-
this.
|
|
224
|
-
this.getState(queue).rateLimit = limit;
|
|
149
|
+
this.limiterManager.setRateLimit(queue, limit);
|
|
225
150
|
}
|
|
226
|
-
/** Clear rate limit */
|
|
227
151
|
clearRateLimit(queue) {
|
|
228
|
-
this.
|
|
229
|
-
const state = this.queueState.get(queue);
|
|
230
|
-
if (state)
|
|
231
|
-
state.rateLimit = null;
|
|
152
|
+
this.limiterManager.clearRateLimit(queue);
|
|
232
153
|
}
|
|
233
|
-
/** Try to acquire rate limit token */
|
|
234
154
|
tryAcquireRateLimit(queue) {
|
|
235
|
-
|
|
236
|
-
return !limiter || limiter.tryAcquire();
|
|
155
|
+
return this.limiterManager.tryAcquireRateLimit(queue);
|
|
237
156
|
}
|
|
238
|
-
/** Set concurrency limit for queue */
|
|
239
157
|
setConcurrency(queue, limit) {
|
|
240
|
-
|
|
241
|
-
if (limiter) {
|
|
242
|
-
limiter.setLimit(limit);
|
|
243
|
-
}
|
|
244
|
-
else {
|
|
245
|
-
limiter = new ConcurrencyLimiter(limit);
|
|
246
|
-
this.concurrencyLimiters.set(queue, limiter);
|
|
247
|
-
}
|
|
248
|
-
this.getState(queue).concurrencyLimit = limit;
|
|
158
|
+
this.limiterManager.setConcurrency(queue, limit);
|
|
249
159
|
}
|
|
250
|
-
/** Clear concurrency limit */
|
|
251
160
|
clearConcurrency(queue) {
|
|
252
|
-
this.
|
|
253
|
-
const state = this.queueState.get(queue);
|
|
254
|
-
if (state)
|
|
255
|
-
state.concurrencyLimit = null;
|
|
161
|
+
this.limiterManager.clearConcurrency(queue);
|
|
256
162
|
}
|
|
257
|
-
/** Try to acquire concurrency slot */
|
|
258
163
|
tryAcquireConcurrency(queue) {
|
|
259
|
-
|
|
260
|
-
return !limiter || limiter.tryAcquire();
|
|
164
|
+
return this.limiterManager.tryAcquireConcurrency(queue);
|
|
261
165
|
}
|
|
262
|
-
/** Release concurrency slot */
|
|
263
166
|
releaseConcurrency(queue) {
|
|
264
|
-
this.
|
|
167
|
+
this.limiterManager.releaseConcurrency(queue);
|
|
168
|
+
}
|
|
169
|
+
get queueState() {
|
|
170
|
+
return this.limiterManager.getStateMap();
|
|
171
|
+
}
|
|
172
|
+
/** Clear limiter data for a queue (rate limits, concurrency) */
|
|
173
|
+
clearQueueLimiters(queue) {
|
|
174
|
+
this.limiterManager.deleteQueue(queue);
|
|
265
175
|
}
|
|
266
176
|
// ============ Resource Release ============
|
|
267
|
-
/** Release all resources for a job */
|
|
268
177
|
releaseJobResources(queue, uniqueKey, groupId) {
|
|
269
|
-
if (uniqueKey)
|
|
178
|
+
if (uniqueKey)
|
|
270
179
|
this.releaseUniqueKey(queue, uniqueKey);
|
|
271
|
-
|
|
272
|
-
if (groupId) {
|
|
180
|
+
if (groupId)
|
|
273
181
|
this.releaseGroup(queue, groupId);
|
|
274
|
-
}
|
|
275
182
|
this.releaseConcurrency(queue);
|
|
276
183
|
}
|
|
277
|
-
// ============ Dependency
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
184
|
+
// ============ Dependency Tracking (delegated) ============
|
|
185
|
+
get waitingDeps() {
|
|
186
|
+
return this.dependencyTracker.waitingDeps;
|
|
187
|
+
}
|
|
188
|
+
get dependencyIndex() {
|
|
189
|
+
return this.dependencyTracker.dependencyIndex;
|
|
190
|
+
}
|
|
191
|
+
get waitingChildren() {
|
|
192
|
+
return this.dependencyTracker.waitingChildren;
|
|
193
|
+
}
|
|
282
194
|
registerDependencies(jobId, dependsOn) {
|
|
283
|
-
|
|
284
|
-
let waiters = this.dependencyIndex.get(depId);
|
|
285
|
-
if (!waiters) {
|
|
286
|
-
waiters = new Set();
|
|
287
|
-
this.dependencyIndex.set(depId, waiters);
|
|
288
|
-
}
|
|
289
|
-
waiters.add(jobId);
|
|
290
|
-
}
|
|
195
|
+
this.dependencyTracker.registerDependencies(jobId, dependsOn);
|
|
291
196
|
}
|
|
292
|
-
/**
|
|
293
|
-
* Unregister a job's dependencies from the reverse index
|
|
294
|
-
* Call when removing a job from waitingDeps
|
|
295
|
-
*/
|
|
296
197
|
unregisterDependencies(jobId, dependsOn) {
|
|
297
|
-
|
|
298
|
-
const waiters = this.dependencyIndex.get(depId);
|
|
299
|
-
if (waiters) {
|
|
300
|
-
waiters.delete(jobId);
|
|
301
|
-
if (waiters.size === 0) {
|
|
302
|
-
this.dependencyIndex.delete(depId);
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
}
|
|
198
|
+
this.dependencyTracker.unregisterDependencies(jobId, dependsOn);
|
|
306
199
|
}
|
|
307
|
-
/**
|
|
308
|
-
* Get jobs waiting for a specific dependency - O(1)
|
|
309
|
-
*/
|
|
310
200
|
getJobsWaitingFor(depId) {
|
|
311
|
-
return this.
|
|
201
|
+
return this.dependencyTracker.getJobsWaitingFor(depId);
|
|
202
|
+
}
|
|
203
|
+
// ============ DLQ Operations (delegated) ============
|
|
204
|
+
get dlq() {
|
|
205
|
+
const map = new Map();
|
|
206
|
+
for (const queue of this.dlqManager.getQueueNames()) {
|
|
207
|
+
map.set(queue, this.dlqManager.getEntries(queue));
|
|
208
|
+
}
|
|
209
|
+
for (const queue of this.queues.keys()) {
|
|
210
|
+
if (!map.has(queue))
|
|
211
|
+
map.set(queue, []);
|
|
212
|
+
}
|
|
213
|
+
return map;
|
|
214
|
+
}
|
|
215
|
+
get dlqConfig() {
|
|
216
|
+
const map = new Map();
|
|
217
|
+
for (const queue of this.getQueueNames()) {
|
|
218
|
+
map.set(queue, this.dlqManager.getConfig(queue));
|
|
219
|
+
}
|
|
220
|
+
return map;
|
|
221
|
+
}
|
|
222
|
+
get stallConfig() {
|
|
223
|
+
const map = new Map();
|
|
224
|
+
for (const queue of this.getQueueNames()) {
|
|
225
|
+
map.set(queue, this.dlqManager.getStallConfig(queue));
|
|
226
|
+
}
|
|
227
|
+
return map;
|
|
312
228
|
}
|
|
313
|
-
// ============ DLQ Operations ============
|
|
314
|
-
/** Get DLQ config for queue */
|
|
315
229
|
getDlqConfig(queue) {
|
|
316
|
-
return this.
|
|
230
|
+
return this.dlqManager.getConfig(queue);
|
|
317
231
|
}
|
|
318
|
-
/** Set DLQ config for queue */
|
|
319
232
|
setDlqConfig(queue, config) {
|
|
320
|
-
|
|
321
|
-
this.dlqConfig.set(queue, { ...current, ...config });
|
|
233
|
+
this.dlqManager.setConfig(queue, config);
|
|
322
234
|
}
|
|
323
|
-
/** Get stall config for queue */
|
|
324
235
|
getStallConfig(queue) {
|
|
325
|
-
return this.
|
|
236
|
+
return this.dlqManager.getStallConfig(queue);
|
|
326
237
|
}
|
|
327
|
-
/** Set stall config for queue */
|
|
328
238
|
setStallConfig(queue, config) {
|
|
329
|
-
|
|
330
|
-
this.stallConfig.set(queue, { ...current, ...config });
|
|
239
|
+
this.dlqManager.setStallConfig(queue, config);
|
|
331
240
|
}
|
|
332
|
-
/** Add job to DLQ with full metadata */
|
|
333
241
|
addToDlq(job, reason = "unknown" /* FailureReason.Unknown */, error = null) {
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
const config = this.getDlqConfig(job.queue);
|
|
340
|
-
const entry = createDlqEntry(job, reason, error, config);
|
|
341
|
-
// Enforce max entries
|
|
342
|
-
while (dlq.length >= config.maxEntries) {
|
|
343
|
-
dlq.shift(); // Remove oldest
|
|
344
|
-
this.decrementDlq();
|
|
345
|
-
}
|
|
346
|
-
dlq.push(entry);
|
|
347
|
-
this.incrementDlq();
|
|
348
|
-
return entry;
|
|
242
|
+
return this.dlqManager.add(job, reason, error);
|
|
243
|
+
}
|
|
244
|
+
/** Restore an existing DlqEntry (for recovery from persistence) */
|
|
245
|
+
restoreDlqEntry(queue, entry) {
|
|
246
|
+
this.dlqManager.restoreEntry(queue, entry);
|
|
349
247
|
}
|
|
350
|
-
/** Get DLQ entries (raw) */
|
|
351
248
|
getDlqEntries(queue) {
|
|
352
|
-
return this.
|
|
249
|
+
return this.dlqManager.getEntries(queue);
|
|
353
250
|
}
|
|
354
|
-
/** Get DLQ jobs (for backward compatibility) */
|
|
355
251
|
getDlq(queue, count) {
|
|
356
|
-
|
|
357
|
-
if (!dlq)
|
|
358
|
-
return [];
|
|
359
|
-
const entries = count ? dlq.slice(0, count) : dlq;
|
|
360
|
-
return entries.map((e) => e.job);
|
|
252
|
+
return this.dlqManager.getJobs(queue, count);
|
|
361
253
|
}
|
|
362
|
-
/** Get DLQ entries with filter */
|
|
363
254
|
getDlqFiltered(queue, filter) {
|
|
364
|
-
|
|
365
|
-
if (!dlq)
|
|
366
|
-
return [];
|
|
367
|
-
const now = Date.now();
|
|
368
|
-
let result = dlq.filter((entry) => {
|
|
369
|
-
if (filter.reason && entry.reason !== filter.reason)
|
|
370
|
-
return false;
|
|
371
|
-
if (filter.olderThan && entry.enteredAt >= filter.olderThan)
|
|
372
|
-
return false;
|
|
373
|
-
if (filter.newerThan && entry.enteredAt <= filter.newerThan)
|
|
374
|
-
return false;
|
|
375
|
-
if (filter.retriable && !canAutoRetry(entry, this.getDlqConfig(queue), now))
|
|
376
|
-
return false;
|
|
377
|
-
if (filter.expired && !isDlqEntryExpired(entry, now))
|
|
378
|
-
return false;
|
|
379
|
-
return true;
|
|
380
|
-
});
|
|
381
|
-
if (filter.offset) {
|
|
382
|
-
result = result.slice(filter.offset);
|
|
383
|
-
}
|
|
384
|
-
if (filter.limit) {
|
|
385
|
-
result = result.slice(0, filter.limit);
|
|
386
|
-
}
|
|
387
|
-
return result;
|
|
255
|
+
return this.dlqManager.getFiltered(queue, filter);
|
|
388
256
|
}
|
|
389
|
-
/** Remove entry from DLQ by job ID */
|
|
390
257
|
removeFromDlq(queue, jobId) {
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
return null;
|
|
394
|
-
const idx = dlq.findIndex((e) => e.job.id === jobId);
|
|
395
|
-
if (idx === -1)
|
|
396
|
-
return null;
|
|
397
|
-
this.decrementDlq();
|
|
398
|
-
return dlq.splice(idx, 1)[0];
|
|
399
|
-
}
|
|
400
|
-
/** Get entries ready for auto-retry */
|
|
258
|
+
return this.dlqManager.remove(queue, jobId);
|
|
259
|
+
}
|
|
401
260
|
getAutoRetryEntries(queue, now = Date.now()) {
|
|
402
|
-
|
|
403
|
-
if (!dlq)
|
|
404
|
-
return [];
|
|
405
|
-
const config = this.getDlqConfig(queue);
|
|
406
|
-
return dlq.filter((entry) => canAutoRetry(entry, config, now));
|
|
261
|
+
return this.dlqManager.getAutoRetryEntries(queue, now);
|
|
407
262
|
}
|
|
408
|
-
/** Get expired entries for cleanup */
|
|
409
263
|
getExpiredEntries(queue, now = Date.now()) {
|
|
410
|
-
|
|
411
|
-
if (!dlq)
|
|
412
|
-
return [];
|
|
413
|
-
return dlq.filter((entry) => isDlqEntryExpired(entry, now));
|
|
264
|
+
return this.dlqManager.getExpiredEntries(queue, now);
|
|
414
265
|
}
|
|
415
|
-
/** Remove expired entries */
|
|
416
266
|
purgeExpired(queue, now = Date.now()) {
|
|
417
|
-
|
|
418
|
-
if (!dlq)
|
|
419
|
-
return 0;
|
|
420
|
-
const before = dlq.length;
|
|
421
|
-
const remaining = dlq.filter((entry) => !isDlqEntryExpired(entry, now));
|
|
422
|
-
if (remaining.length < before) {
|
|
423
|
-
this.dlq.set(queue, remaining);
|
|
424
|
-
const removed = before - remaining.length;
|
|
425
|
-
this.decrementDlq(removed);
|
|
426
|
-
return removed;
|
|
427
|
-
}
|
|
428
|
-
return 0;
|
|
267
|
+
return this.dlqManager.purgeExpired(queue, now);
|
|
429
268
|
}
|
|
430
|
-
/** Clear DLQ for queue */
|
|
431
269
|
clearDlq(queue) {
|
|
432
|
-
|
|
433
|
-
if (!dlq)
|
|
434
|
-
return 0;
|
|
435
|
-
const count = dlq.length;
|
|
436
|
-
this.dlq.delete(queue);
|
|
437
|
-
this.decrementDlq(count);
|
|
438
|
-
return count;
|
|
270
|
+
return this.dlqManager.clear(queue);
|
|
439
271
|
}
|
|
440
272
|
// ============ Queue Stats ============
|
|
441
|
-
/** Get waiting job count for queue */
|
|
442
273
|
getWaitingCount(queue) {
|
|
443
274
|
return this.queues.get(queue)?.size ?? 0;
|
|
444
275
|
}
|
|
445
|
-
/** Get DLQ count for queue */
|
|
446
276
|
getDlqCount(queue) {
|
|
447
|
-
return this.
|
|
277
|
+
return this.dlqManager.getCount(queue);
|
|
448
278
|
}
|
|
449
|
-
/** Get all queue names in this shard */
|
|
450
279
|
getQueueNames() {
|
|
451
280
|
const names = new Set();
|
|
452
281
|
for (const name of this.queues.keys())
|
|
453
282
|
names.add(name);
|
|
454
|
-
for (const name of this.
|
|
283
|
+
for (const name of this.dlqManager.getQueueNames())
|
|
455
284
|
names.add(name);
|
|
456
|
-
for (const name of this.
|
|
285
|
+
for (const name of this.limiterManager.getQueueNames())
|
|
457
286
|
names.add(name);
|
|
458
287
|
return Array.from(names);
|
|
459
288
|
}
|
|
460
|
-
/** Get job counts grouped by priority for a queue */
|
|
461
289
|
getCountsPerPriority(queue) {
|
|
462
290
|
const q = this.queues.get(queue);
|
|
463
291
|
const counts = new Map();
|
|
@@ -470,203 +298,105 @@ export class Shard {
|
|
|
470
298
|
return counts;
|
|
471
299
|
}
|
|
472
300
|
// ============ Running Counters (O(1) Stats) ============
|
|
473
|
-
/** Get shard statistics - O(1) */
|
|
474
301
|
getStats() {
|
|
475
302
|
return { ...this.stats };
|
|
476
303
|
}
|
|
477
|
-
/** Get internal structure sizes for memory debugging */
|
|
478
304
|
getInternalSizes() {
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
delayedRunAt: this.delayedRunAt.size,
|
|
483
|
-
temporalIndex: this.temporalIndex.size,
|
|
484
|
-
waiters: this.waiters.length,
|
|
485
|
-
};
|
|
486
|
-
}
|
|
487
|
-
/** Increment queued jobs counter and add to temporal index */
|
|
305
|
+
const sizes = this.temporalManager.getSizes();
|
|
306
|
+
return { ...sizes, waiters: this.waiters.length };
|
|
307
|
+
}
|
|
488
308
|
incrementQueued(jobId, isDelayed, createdAt, queue, runAt) {
|
|
489
309
|
this.stats.queuedJobs++;
|
|
490
310
|
if (isDelayed) {
|
|
491
311
|
this.stats.delayedJobs++;
|
|
492
|
-
this.delayedJobIds.add(jobId);
|
|
493
|
-
// Add to min-heap for O(k) refresh instead of O(n × queues)
|
|
494
|
-
// Only if runAt is provided (for full optimization)
|
|
495
312
|
if (runAt !== undefined) {
|
|
496
|
-
this.
|
|
497
|
-
this.delayedRunAt.set(jobId, runAt);
|
|
313
|
+
this.temporalManager.addDelayed(jobId, runAt);
|
|
498
314
|
}
|
|
499
315
|
}
|
|
500
|
-
// Add to temporal index for efficient cleanQueue
|
|
501
316
|
if (createdAt !== undefined && queue !== undefined) {
|
|
502
|
-
this.
|
|
317
|
+
this.temporalManager.addToIndex(createdAt, jobId, queue);
|
|
503
318
|
}
|
|
504
319
|
}
|
|
505
|
-
/** Decrement queued jobs counter and remove from temporal index */
|
|
506
320
|
decrementQueued(jobId) {
|
|
507
321
|
this.stats.queuedJobs = Math.max(0, this.stats.queuedJobs - 1);
|
|
508
|
-
if (this.
|
|
322
|
+
if (this.temporalManager.removeDelayed(jobId)) {
|
|
509
323
|
this.stats.delayedJobs = Math.max(0, this.stats.delayedJobs - 1);
|
|
510
|
-
this.delayedJobIds.delete(jobId);
|
|
511
|
-
// Mark as stale in heap (lazy removal)
|
|
512
|
-
this.delayedRunAt.delete(jobId);
|
|
513
324
|
}
|
|
514
|
-
// Remove from temporal index (lazy removal - will be cleaned on next cleanQueue)
|
|
515
325
|
}
|
|
516
|
-
/** Increment DLQ counter */
|
|
517
326
|
incrementDlq() {
|
|
518
327
|
this.stats.dlqJobs++;
|
|
519
328
|
}
|
|
520
|
-
/** Decrement DLQ counter */
|
|
521
329
|
decrementDlq(count = 1) {
|
|
522
330
|
this.stats.dlqJobs = Math.max(0, this.stats.dlqJobs - count);
|
|
523
331
|
}
|
|
524
|
-
/**
|
|
525
|
-
* Update delayed jobs that have become ready (call periodically)
|
|
526
|
-
* O(k) where k = jobs that became ready, instead of O(n × queues)
|
|
527
|
-
*/
|
|
528
332
|
refreshDelayedCount(now) {
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
const top = this.delayedHeap.peek();
|
|
532
|
-
if (!top || top.runAt > now)
|
|
533
|
-
break;
|
|
534
|
-
// Pop from heap
|
|
535
|
-
this.delayedHeap.pop();
|
|
536
|
-
// Check if stale (job was removed or runAt changed)
|
|
537
|
-
const currentRunAt = this.delayedRunAt.get(top.jobId);
|
|
538
|
-
if (currentRunAt === undefined) {
|
|
539
|
-
// Job was removed, skip
|
|
540
|
-
continue;
|
|
541
|
-
}
|
|
542
|
-
if (currentRunAt !== top.runAt) {
|
|
543
|
-
// runAt changed, this entry is stale, skip
|
|
544
|
-
continue;
|
|
545
|
-
}
|
|
546
|
-
// Job is ready - remove from delayed tracking
|
|
547
|
-
this.delayedJobIds.delete(top.jobId);
|
|
548
|
-
this.delayedRunAt.delete(top.jobId);
|
|
549
|
-
this.stats.delayedJobs = Math.max(0, this.stats.delayedJobs - 1);
|
|
550
|
-
}
|
|
333
|
+
const readyCount = this.temporalManager.refreshDelayed(now);
|
|
334
|
+
this.stats.delayedJobs = Math.max(0, this.stats.delayedJobs - readyCount);
|
|
551
335
|
}
|
|
552
|
-
/** Reset all counters (used after drain/obliterate) */
|
|
553
336
|
resetQueuedCounters() {
|
|
554
337
|
this.stats.queuedJobs = 0;
|
|
555
338
|
this.stats.delayedJobs = 0;
|
|
556
|
-
this.
|
|
557
|
-
this.delayedHeap.clear();
|
|
558
|
-
this.delayedRunAt.clear();
|
|
339
|
+
this.temporalManager.clearDelayed();
|
|
559
340
|
}
|
|
560
|
-
/** Reset DLQ counter */
|
|
561
341
|
resetDlqCounter() {
|
|
562
342
|
this.stats.dlqJobs = 0;
|
|
563
343
|
}
|
|
564
|
-
// ============ Temporal Index (
|
|
565
|
-
/**
|
|
566
|
-
* Add job to temporal index - O(log n) with Skip List
|
|
567
|
-
* Previously O(n) with array splice
|
|
568
|
-
*/
|
|
569
|
-
addToTemporalIndex(createdAt, jobId, queue) {
|
|
570
|
-
this.temporalIndex.insert({ createdAt, jobId, queue });
|
|
571
|
-
}
|
|
572
|
-
/**
|
|
573
|
-
* Get old jobs from temporal index - O(log n + k) where k = returned jobs
|
|
574
|
-
* Returns jobs older than threshold, up to limit
|
|
575
|
-
*/
|
|
344
|
+
// ============ Temporal Index (delegated) ============
|
|
576
345
|
getOldJobs(queue, thresholdMs, limit) {
|
|
577
|
-
|
|
578
|
-
const threshold = now - thresholdMs;
|
|
579
|
-
const result = [];
|
|
580
|
-
// Use Skip List takeWhile for O(k) iteration from start
|
|
581
|
-
// Stops when createdAt > threshold
|
|
582
|
-
for (const entry of this.temporalIndex.values()) {
|
|
583
|
-
if (entry.createdAt > threshold)
|
|
584
|
-
break;
|
|
585
|
-
if (entry.queue === queue) {
|
|
586
|
-
result.push({ jobId: entry.jobId, createdAt: entry.createdAt });
|
|
587
|
-
if (result.length >= limit)
|
|
588
|
-
break;
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
return result;
|
|
346
|
+
return this.temporalManager.getOldJobs(queue, thresholdMs, limit);
|
|
592
347
|
}
|
|
593
|
-
/**
|
|
594
|
-
* Remove job from temporal index (called after job is cleaned)
|
|
595
|
-
* O(n) in worst case but typically fast with deleteWhere
|
|
596
|
-
*/
|
|
597
348
|
removeFromTemporalIndex(jobId) {
|
|
598
|
-
this.
|
|
349
|
+
this.temporalManager.removeFromIndex(jobId);
|
|
599
350
|
}
|
|
600
|
-
/** Clear temporal index for a queue */
|
|
601
351
|
clearTemporalIndexForQueue(queue) {
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
}
|
|
605
|
-
/**
|
|
606
|
-
* Clean orphaned temporal index entries.
|
|
607
|
-
* Removes entries for jobs that no longer exist in the queue.
|
|
608
|
-
* Call periodically to prevent memory leaks.
|
|
609
|
-
*/
|
|
352
|
+
this.temporalManager.clearIndexForQueue(queue);
|
|
353
|
+
}
|
|
610
354
|
cleanOrphanedTemporalEntries() {
|
|
611
|
-
if (this.
|
|
355
|
+
if (this.temporalManager.indexSize === 0)
|
|
612
356
|
return 0;
|
|
613
|
-
// Build a set of valid job IDs from all queues
|
|
614
357
|
const validJobIds = new Set();
|
|
615
|
-
for (const
|
|
616
|
-
for (const job of
|
|
358
|
+
for (const pq of this.queues.values()) {
|
|
359
|
+
for (const job of pq.values()) {
|
|
617
360
|
validJobIds.add(job.id);
|
|
618
361
|
}
|
|
619
362
|
}
|
|
620
|
-
|
|
621
|
-
const beforeSize = this.temporalIndex.size;
|
|
622
|
-
this.temporalIndex.removeAll((e) => !validJobIds.has(e.jobId));
|
|
623
|
-
return beforeSize - this.temporalIndex.size;
|
|
363
|
+
return this.temporalManager.cleanOrphaned(validJobIds);
|
|
624
364
|
}
|
|
625
|
-
|
|
365
|
+
// ============ Queue Lifecycle ============
|
|
626
366
|
drain(queue) {
|
|
627
367
|
const q = this.queues.get(queue);
|
|
628
368
|
if (!q)
|
|
629
369
|
return { count: 0, jobIds: [] };
|
|
630
370
|
const count = q.size;
|
|
631
371
|
const jobIds = [];
|
|
632
|
-
// Collect job IDs and remove delayed job tracking
|
|
633
372
|
for (const job of q.values()) {
|
|
634
373
|
jobIds.push(job.id);
|
|
635
|
-
this.
|
|
374
|
+
this.temporalManager.removeDelayed(job.id);
|
|
636
375
|
}
|
|
637
376
|
q.clear();
|
|
638
|
-
|
|
639
|
-
this.clearTemporalIndexForQueue(queue);
|
|
640
|
-
// Update counters
|
|
377
|
+
this.temporalManager.clearIndexForQueue(queue);
|
|
641
378
|
this.stats.queuedJobs = Math.max(0, this.stats.queuedJobs - count);
|
|
642
|
-
this.stats.delayedJobs =
|
|
379
|
+
this.stats.delayedJobs = this.temporalManager.delayedCount;
|
|
643
380
|
return { count, jobIds };
|
|
644
381
|
}
|
|
645
|
-
/** Obliterate queue completely */
|
|
646
382
|
obliterate(queue) {
|
|
647
|
-
// Update counters before deleting
|
|
648
383
|
const q = this.queues.get(queue);
|
|
649
384
|
if (q) {
|
|
650
385
|
for (const job of q.values()) {
|
|
651
|
-
this.
|
|
386
|
+
this.temporalManager.removeDelayed(job.id);
|
|
652
387
|
}
|
|
653
388
|
this.stats.queuedJobs = Math.max(0, this.stats.queuedJobs - q.size);
|
|
654
389
|
}
|
|
655
|
-
const
|
|
656
|
-
if (
|
|
657
|
-
this.stats.dlqJobs = Math.max(0, this.stats.dlqJobs -
|
|
390
|
+
const dlqCount = this.dlqManager.deleteQueue(queue);
|
|
391
|
+
if (dlqCount > 0) {
|
|
392
|
+
this.stats.dlqJobs = Math.max(0, this.stats.dlqJobs - dlqCount);
|
|
658
393
|
}
|
|
659
|
-
|
|
660
|
-
this.
|
|
661
|
-
// Clear temporal index for this queue
|
|
662
|
-
this.clearTemporalIndexForQueue(queue);
|
|
394
|
+
this.stats.delayedJobs = this.temporalManager.delayedCount;
|
|
395
|
+
this.temporalManager.clearIndexForQueue(queue);
|
|
663
396
|
this.queues.delete(queue);
|
|
664
|
-
this.
|
|
665
|
-
this.
|
|
666
|
-
this.queueState.delete(queue);
|
|
397
|
+
this.uniqueKeyManager.clearQueue(queue);
|
|
398
|
+
this.limiterManager.deleteQueue(queue);
|
|
667
399
|
this.activeGroups.delete(queue);
|
|
668
|
-
this.rateLimiters.delete(queue);
|
|
669
|
-
this.concurrencyLimiters.delete(queue);
|
|
670
400
|
}
|
|
671
401
|
}
|
|
672
402
|
//# sourceMappingURL=shard.js.map
|