bunqueue 2.8.56 → 2.8.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/client/bunqueue/aging.d.ts +1 -0
- package/dist/client/bunqueue/aging.js +17 -6
- package/dist/client/bunqueue/cancellation.d.ts +4 -2
- package/dist/client/bunqueue/cancellation.js +54 -22
- package/dist/client/bunqueue/circuitBreaker.d.ts +1 -0
- package/dist/client/bunqueue/circuitBreaker.js +19 -1
- package/dist/client/bunqueue/retry.d.ts +1 -1
- package/dist/client/bunqueue/retry.js +45 -7
- package/dist/client/bunqueue/runtime.js +25 -7
- package/dist/client/manager.d.ts +4 -1
- package/dist/client/manager.js +45 -7
- package/dist/client/queue/runtime/state.d.ts +1 -0
- package/dist/client/queue/runtime/state.js +4 -0
- package/dist/client/types/connection.d.ts +1 -0
- package/dist/client/types/events.d.ts +1 -1
- package/dist/client/types/worker.d.ts +1 -0
- package/dist/client/worker/runtime/control.js +2 -8
- package/dist/client/worker/runtime/lifecycle.js +1 -4
- package/dist/client/worker/runtime/polling.d.ts +1 -0
- package/dist/client/worker/runtime/polling.js +24 -17
- package/dist/client/worker/runtime/state.d.ts +2 -0
- package/dist/client/worker/runtime/state.js +8 -0
- package/dist/infrastructure/backup/s3Backup.d.ts +1 -0
- package/dist/infrastructure/backup/s3Backup.js +16 -2
- package/dist/infrastructure/cloud/cloudAgent.d.ts +1 -0
- package/dist/infrastructure/cloud/cloudAgent.js +10 -2
- package/package.json +1 -1
- package/dist/application/types.d.ts +0 -148
- package/dist/application/types.js +0 -16
- package/dist/client/queue/queueTypes.d.ts +0 -33
- package/dist/client/queue/queueTypes.js +0 -5
- package/dist/client/sandboxed/types.d.ts +0 -79
- package/dist/client/sandboxed/types.js +0 -5
- package/dist/client/worker/types.d.ts +0 -50
- package/dist/client/worker/types.js +0 -13
package/README.md
CHANGED
|
@@ -242,13 +242,18 @@ implementation:
|
|
|
242
242
|
|
|
243
243
|
| SDK | Runtime invariants | Generated tests | Mutation engine |
|
|
244
244
|
| --- | --- | --- | --- |
|
|
245
|
-
| [TypeScript](./sdk/typescript/README.md) | [contract](./sdk/typescript/INVARIANTS.md) | fast-check |
|
|
245
|
+
| [TypeScript](./sdk/typescript/README.md) | [contract](./sdk/typescript/INVARIANTS.md) | fast-check | none¹ |
|
|
246
246
|
| [Python](./sdk/python/README.md) | [contract](./sdk/python/INVARIANTS.md) | Hypothesis | mutmut |
|
|
247
247
|
| [PHP](./sdk/php/README.md) | [contract](./sdk/php/INVARIANTS.md) | Eris | Infection |
|
|
248
248
|
| [Go](./sdk/go/README.md) | [contract](./sdk/go/INVARIANTS.md) | Rapid | Gremlins |
|
|
249
249
|
| [Rust](./sdk/rust/README.md) | [contract](./sdk/rust/INVARIANTS.md) | proptest | cargo-mutants |
|
|
250
250
|
| [Elixir](./sdk/elixir/README.md) | [contract](./sdk/elixir/INVARIANTS.md) | StreamData | Muex |
|
|
251
251
|
|
|
252
|
+
¹ The TypeScript SDK has no mutation engine. StrykerJS was removed because its
|
|
253
|
+
dependency graph produced every advisory the weekly audit had to answer for,
|
|
254
|
+
none of it reachable from the published client; the planners keep their
|
|
255
|
+
fast-check coverage.
|
|
256
|
+
|
|
252
257
|
Property campaigns run in the ordinary SDK gate with deterministic replay
|
|
253
258
|
seeds. Mutation campaigns run separately against the pure planners and
|
|
254
259
|
snapshot validators. Contributors can reproduce the complete isolated SDK
|
|
@@ -6,6 +6,7 @@ import type { Queue } from '../queue/queue';
|
|
|
6
6
|
import type { PriorityAgingConfig } from './types';
|
|
7
7
|
export declare class PriorityAger<T = unknown> {
|
|
8
8
|
private timer;
|
|
9
|
+
private generation;
|
|
9
10
|
private readonly config;
|
|
10
11
|
private readonly queue;
|
|
11
12
|
constructor(config: PriorityAgingConfig, queue: Queue<T>);
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
export class PriorityAger {
|
|
6
6
|
timer = null;
|
|
7
|
+
generation = 0;
|
|
7
8
|
config;
|
|
8
9
|
queue;
|
|
9
10
|
constructor(config, queue) {
|
|
@@ -11,12 +12,17 @@ export class PriorityAger {
|
|
|
11
12
|
this.queue = queue;
|
|
12
13
|
}
|
|
13
14
|
start() {
|
|
15
|
+
if (this.timer !== null)
|
|
16
|
+
return;
|
|
14
17
|
const interval = this.config.interval ?? 60000;
|
|
18
|
+
const generation = ++this.generation;
|
|
15
19
|
this.timer = setInterval(() => {
|
|
16
|
-
void this.tick();
|
|
20
|
+
void this.tick(generation);
|
|
17
21
|
}, interval);
|
|
18
22
|
}
|
|
19
|
-
async tick() {
|
|
23
|
+
async tick(generation) {
|
|
24
|
+
if (generation !== this.generation)
|
|
25
|
+
return;
|
|
20
26
|
const minAge = this.config.minAge ?? 60000;
|
|
21
27
|
const boost = this.config.boost ?? 1;
|
|
22
28
|
const maxPriority = this.config.maxPriority ?? 100;
|
|
@@ -26,9 +32,13 @@ export class PriorityAger {
|
|
|
26
32
|
this.queue.getWaitingAsync(0, maxScan),
|
|
27
33
|
this.queue.getJobsAsync({ state: 'prioritized', start: 0, end: maxScan }),
|
|
28
34
|
]);
|
|
35
|
+
if (generation !== this.generation)
|
|
36
|
+
return;
|
|
29
37
|
const jobs = [...waiting, ...prioritized];
|
|
30
38
|
const now = Date.now();
|
|
31
39
|
for (const job of jobs) {
|
|
40
|
+
if (generation !== this.generation)
|
|
41
|
+
return;
|
|
32
42
|
const age = now - job.timestamp;
|
|
33
43
|
if (age >= minAge && job.priority < maxPriority) {
|
|
34
44
|
const newPriority = Math.min(job.priority + boost, maxPriority);
|
|
@@ -44,9 +54,10 @@ export class PriorityAger {
|
|
|
44
54
|
}
|
|
45
55
|
}
|
|
46
56
|
destroy() {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
57
|
+
this.generation++;
|
|
58
|
+
const timer = this.timer;
|
|
59
|
+
this.timer = null;
|
|
60
|
+
if (timer !== null)
|
|
61
|
+
clearInterval(timer);
|
|
51
62
|
}
|
|
52
63
|
}
|
|
@@ -3,11 +3,12 @@
|
|
|
3
3
|
* AbortController-based cancellation with optional grace period.
|
|
4
4
|
*/
|
|
5
5
|
export declare class CancellationManager {
|
|
6
|
-
private readonly
|
|
6
|
+
private readonly currentByJob;
|
|
7
|
+
private readonly registrations;
|
|
7
8
|
/** Register a new AbortController for a job */
|
|
8
9
|
register(jobId: string): AbortController;
|
|
9
10
|
/** Remove a job's controller (on completion) */
|
|
10
|
-
unregister(jobId: string): void;
|
|
11
|
+
unregister(jobId: string, controller?: AbortController): void;
|
|
11
12
|
/** Cancel a job with optional grace period */
|
|
12
13
|
cancel(jobId: string, gracePeriodMs?: number): void;
|
|
13
14
|
/** Check if a job is cancelled */
|
|
@@ -16,4 +17,5 @@ export declare class CancellationManager {
|
|
|
16
17
|
getSignal(jobId: string): AbortSignal | null;
|
|
17
18
|
/** Cancel all and clear */
|
|
18
19
|
destroyAll(): void;
|
|
20
|
+
private clearTimer;
|
|
19
21
|
}
|
|
@@ -3,45 +3,77 @@
|
|
|
3
3
|
* AbortController-based cancellation with optional grace period.
|
|
4
4
|
*/
|
|
5
5
|
export class CancellationManager {
|
|
6
|
-
|
|
6
|
+
currentByJob = new Map();
|
|
7
|
+
registrations = new Map();
|
|
7
8
|
/** Register a new AbortController for a job */
|
|
8
9
|
register(jobId) {
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
const controller = new AbortController();
|
|
11
|
+
const registration = { jobId, controller, timer: null, deadline: null };
|
|
12
|
+
this.currentByJob.set(jobId, registration);
|
|
13
|
+
this.registrations.set(controller, registration);
|
|
14
|
+
return controller;
|
|
12
15
|
}
|
|
13
16
|
/** Remove a job's controller (on completion) */
|
|
14
|
-
unregister(jobId) {
|
|
15
|
-
|
|
17
|
+
unregister(jobId, controller) {
|
|
18
|
+
const registration = controller
|
|
19
|
+
? this.registrations.get(controller)
|
|
20
|
+
: this.currentByJob.get(jobId);
|
|
21
|
+
if (!registration || registration.jobId !== jobId)
|
|
22
|
+
return;
|
|
23
|
+
this.clearTimer(registration);
|
|
24
|
+
this.registrations.delete(registration.controller);
|
|
25
|
+
if (this.currentByJob.get(jobId) === registration)
|
|
26
|
+
this.currentByJob.delete(jobId);
|
|
16
27
|
}
|
|
17
28
|
/** Cancel a job with optional grace period */
|
|
18
29
|
cancel(jobId, gracePeriodMs = 0) {
|
|
19
|
-
const
|
|
20
|
-
if (!
|
|
30
|
+
const registration = this.currentByJob.get(jobId);
|
|
31
|
+
if (!registration)
|
|
32
|
+
return;
|
|
33
|
+
if (gracePeriodMs <= 0) {
|
|
34
|
+
this.clearTimer(registration);
|
|
35
|
+
registration.controller.abort();
|
|
21
36
|
return;
|
|
22
|
-
if (gracePeriodMs > 0) {
|
|
23
|
-
setTimeout(() => {
|
|
24
|
-
ac.abort();
|
|
25
|
-
}, gracePeriodMs);
|
|
26
|
-
}
|
|
27
|
-
else {
|
|
28
|
-
ac.abort();
|
|
29
37
|
}
|
|
38
|
+
if (registration.controller.signal.aborted)
|
|
39
|
+
return;
|
|
40
|
+
const deadline = Date.now() + gracePeriodMs;
|
|
41
|
+
if (registration.deadline !== null && registration.deadline <= deadline)
|
|
42
|
+
return;
|
|
43
|
+
this.clearTimer(registration);
|
|
44
|
+
const timer = setTimeout(() => {
|
|
45
|
+
if (registration.timer !== timer)
|
|
46
|
+
return;
|
|
47
|
+
registration.timer = null;
|
|
48
|
+
registration.deadline = null;
|
|
49
|
+
registration.controller.abort();
|
|
50
|
+
}, gracePeriodMs);
|
|
51
|
+
registration.timer = timer;
|
|
52
|
+
registration.deadline = deadline;
|
|
30
53
|
}
|
|
31
54
|
/** Check if a job is cancelled */
|
|
32
55
|
isCancelled(jobId) {
|
|
33
|
-
|
|
34
|
-
return ac ? ac.signal.aborted : false;
|
|
56
|
+
return this.currentByJob.get(jobId)?.controller.signal.aborted ?? false;
|
|
35
57
|
}
|
|
36
58
|
/** Get the AbortSignal for a job */
|
|
37
59
|
getSignal(jobId) {
|
|
38
|
-
return this.
|
|
60
|
+
return this.currentByJob.get(jobId)?.controller.signal ?? null;
|
|
39
61
|
}
|
|
40
62
|
/** Cancel all and clear */
|
|
41
63
|
destroyAll() {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
64
|
+
const registrations = [...this.registrations.values()];
|
|
65
|
+
this.currentByJob.clear();
|
|
66
|
+
this.registrations.clear();
|
|
67
|
+
for (const registration of registrations)
|
|
68
|
+
this.clearTimer(registration);
|
|
69
|
+
for (const registration of registrations)
|
|
70
|
+
registration.controller.abort();
|
|
71
|
+
}
|
|
72
|
+
clearTimer(registration) {
|
|
73
|
+
const timer = registration.timer;
|
|
74
|
+
registration.timer = null;
|
|
75
|
+
registration.deadline = null;
|
|
76
|
+
if (timer !== null)
|
|
77
|
+
clearTimeout(timer);
|
|
46
78
|
}
|
|
47
79
|
}
|
|
@@ -5,6 +5,7 @@ export class WorkerCircuitBreaker {
|
|
|
5
5
|
state = 'closed';
|
|
6
6
|
failures = 0;
|
|
7
7
|
timer = null;
|
|
8
|
+
destroyed = false;
|
|
8
9
|
config;
|
|
9
10
|
worker;
|
|
10
11
|
constructor(config, worker) {
|
|
@@ -18,6 +19,8 @@ export class WorkerCircuitBreaker {
|
|
|
18
19
|
return this.state === 'open';
|
|
19
20
|
}
|
|
20
21
|
onSuccess() {
|
|
22
|
+
if (this.destroyed)
|
|
23
|
+
return;
|
|
21
24
|
if (this.state === 'half-open') {
|
|
22
25
|
this.state = 'closed';
|
|
23
26
|
this.failures = 0;
|
|
@@ -28,6 +31,8 @@ export class WorkerCircuitBreaker {
|
|
|
28
31
|
}
|
|
29
32
|
}
|
|
30
33
|
onFailure() {
|
|
34
|
+
if (this.destroyed)
|
|
35
|
+
return;
|
|
31
36
|
this.failures++;
|
|
32
37
|
const threshold = this.config.threshold ?? 5;
|
|
33
38
|
if (this.state === 'half-open' || this.failures >= threshold) {
|
|
@@ -35,19 +40,31 @@ export class WorkerCircuitBreaker {
|
|
|
35
40
|
}
|
|
36
41
|
}
|
|
37
42
|
open() {
|
|
43
|
+
if (this.destroyed)
|
|
44
|
+
return;
|
|
38
45
|
this.state = 'open';
|
|
39
46
|
this.config.onOpen?.(this.failures);
|
|
47
|
+
if (this.destroyed)
|
|
48
|
+
return;
|
|
40
49
|
this.worker.pause();
|
|
41
50
|
const resetTimeout = this.config.resetTimeout ?? 30000;
|
|
42
51
|
if (this.timer)
|
|
43
52
|
clearTimeout(this.timer);
|
|
44
|
-
|
|
53
|
+
const timer = setTimeout(() => {
|
|
54
|
+
if (this.destroyed || this.timer !== timer)
|
|
55
|
+
return;
|
|
56
|
+
this.timer = null;
|
|
45
57
|
this.state = 'half-open';
|
|
46
58
|
this.config.onHalfOpen?.();
|
|
59
|
+
if (this.destroyed)
|
|
60
|
+
return;
|
|
47
61
|
this.worker.resume();
|
|
48
62
|
}, resetTimeout);
|
|
63
|
+
this.timer = timer;
|
|
49
64
|
}
|
|
50
65
|
reset() {
|
|
66
|
+
if (this.destroyed)
|
|
67
|
+
return;
|
|
51
68
|
this.state = 'closed';
|
|
52
69
|
this.failures = 0;
|
|
53
70
|
if (this.timer) {
|
|
@@ -59,6 +76,7 @@ export class WorkerCircuitBreaker {
|
|
|
59
76
|
}
|
|
60
77
|
}
|
|
61
78
|
destroy() {
|
|
79
|
+
this.destroyed = true;
|
|
62
80
|
if (this.timer) {
|
|
63
81
|
clearTimeout(this.timer);
|
|
64
82
|
this.timer = null;
|
|
@@ -5,4 +5,4 @@ import type { RetryConfig, RetryStrategy } from './types';
|
|
|
5
5
|
/** Calculate backoff delay based on strategy */
|
|
6
6
|
export declare function calculateBackoff(strategy: RetryStrategy, attempt: number, baseDelay: number, error: Error, config: RetryConfig): number;
|
|
7
7
|
/** Execute a function with retry logic */
|
|
8
|
-
export declare function executeWithRetry<R>(fn: () => Promise<R>, config: RetryConfig): Promise<R>;
|
|
8
|
+
export declare function executeWithRetry<R>(fn: () => Promise<R>, config: RetryConfig, signal?: AbortSignal): Promise<R>;
|
|
@@ -30,24 +30,62 @@ export function calculateBackoff(strategy, attempt, baseDelay, error, config) {
|
|
|
30
30
|
return baseDelay;
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
|
+
function cancellationError() {
|
|
34
|
+
return new Error('Job cancelled');
|
|
35
|
+
}
|
|
36
|
+
function waitForRetry(delay, next, signal) {
|
|
37
|
+
return new Promise((resolve, reject) => {
|
|
38
|
+
if (signal?.aborted) {
|
|
39
|
+
reject(cancellationError());
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
let timer = null;
|
|
43
|
+
const onAbort = () => {
|
|
44
|
+
if (timer !== null)
|
|
45
|
+
clearTimeout(timer);
|
|
46
|
+
timer = null;
|
|
47
|
+
signal?.removeEventListener('abort', onAbort);
|
|
48
|
+
reject(cancellationError());
|
|
49
|
+
};
|
|
50
|
+
timer = setTimeout(() => {
|
|
51
|
+
timer = null;
|
|
52
|
+
signal?.removeEventListener('abort', onAbort);
|
|
53
|
+
if (signal?.aborted) {
|
|
54
|
+
reject(cancellationError());
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
resolve(next());
|
|
58
|
+
}, delay);
|
|
59
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
60
|
+
if (signal?.aborted)
|
|
61
|
+
onAbort();
|
|
62
|
+
});
|
|
63
|
+
}
|
|
33
64
|
/** Execute a function with retry logic */
|
|
34
|
-
export function executeWithRetry(fn, config) {
|
|
65
|
+
export function executeWithRetry(fn, config, signal) {
|
|
35
66
|
const maxAttempts = config.maxAttempts ?? 3;
|
|
36
67
|
const baseDelay = config.delay ?? 1000;
|
|
37
68
|
const strategy = config.strategy ?? 'exponential';
|
|
38
69
|
const attempt = (n) => {
|
|
39
|
-
|
|
70
|
+
if (signal?.aborted)
|
|
71
|
+
return Promise.reject(cancellationError());
|
|
72
|
+
let execution;
|
|
73
|
+
try {
|
|
74
|
+
execution = fn();
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
execution = Promise.reject(error);
|
|
78
|
+
}
|
|
79
|
+
return execution.catch((err) => {
|
|
40
80
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
81
|
+
if (signal?.aborted)
|
|
82
|
+
throw cancellationError();
|
|
41
83
|
if (n >= maxAttempts)
|
|
42
84
|
throw error;
|
|
43
85
|
if (config.retryIf && !config.retryIf(error, n))
|
|
44
86
|
throw error;
|
|
45
87
|
const delay = calculateBackoff(strategy, n, baseDelay, error, config);
|
|
46
|
-
return
|
|
47
|
-
setTimeout(() => {
|
|
48
|
-
resolve(attempt(n + 1));
|
|
49
|
-
}, delay);
|
|
50
|
-
});
|
|
88
|
+
return waitForRetry(delay, () => attempt(n + 1), signal);
|
|
51
89
|
});
|
|
52
90
|
};
|
|
53
91
|
return attempt(1);
|
|
@@ -103,18 +103,36 @@ export class BunqueueRuntime {
|
|
|
103
103
|
}
|
|
104
104
|
const abortController = this.cancellation.register(job.id);
|
|
105
105
|
const runChain = () => this.runMiddlewareChain(job, abortController);
|
|
106
|
-
|
|
106
|
+
let execution;
|
|
107
|
+
try {
|
|
108
|
+
execution = this.retryConfig
|
|
109
|
+
? executeWithRetry(runChain, this.retryConfig, abortController.signal)
|
|
110
|
+
: runChain();
|
|
111
|
+
}
|
|
112
|
+
catch (error) {
|
|
113
|
+
execution = Promise.reject(error);
|
|
114
|
+
}
|
|
107
115
|
return execution.then((result) => {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
116
|
+
try {
|
|
117
|
+
this.cb?.onSuccess();
|
|
118
|
+
return result;
|
|
119
|
+
}
|
|
120
|
+
finally {
|
|
121
|
+
this.cancellation.unregister(job.id, abortController);
|
|
122
|
+
}
|
|
111
123
|
}, (error) => {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
124
|
+
try {
|
|
125
|
+
this.cb?.onFailure();
|
|
126
|
+
throw error;
|
|
127
|
+
}
|
|
128
|
+
finally {
|
|
129
|
+
this.cancellation.unregister(job.id, abortController);
|
|
130
|
+
}
|
|
115
131
|
});
|
|
116
132
|
}
|
|
117
133
|
runMiddlewareChain(job, abortController) {
|
|
134
|
+
if (abortController.signal.aborted)
|
|
135
|
+
return Promise.reject(new Error('Job cancelled'));
|
|
118
136
|
const publicJob = job;
|
|
119
137
|
if (this.middlewares.length === 0) {
|
|
120
138
|
const result = this.baseProcessor(job);
|
package/dist/client/manager.d.ts
CHANGED
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
import { QueueManager } from '../application/queueManager';
|
|
5
5
|
/** Shared manager type export */
|
|
6
6
|
export type SharedManager = QueueManager;
|
|
7
|
-
/**
|
|
7
|
+
/**
|
|
8
|
+
* Get the process-wide QueueManager.
|
|
9
|
+
* A later explicit dataPath must identify the database selected on first use.
|
|
10
|
+
*/
|
|
8
11
|
export declare function getSharedManager(dataPath?: string): QueueManager;
|
|
9
12
|
/** Shutdown shared manager */
|
|
10
13
|
export declare function shutdownManager(): void;
|
package/dist/client/manager.js
CHANGED
|
@@ -1,21 +1,59 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared QueueManager singleton
|
|
3
3
|
*/
|
|
4
|
+
import { realpathSync } from 'node:fs';
|
|
5
|
+
import { basename, dirname, join, resolve } from 'node:path';
|
|
4
6
|
import { QueueManager } from '../application/queueManager';
|
|
5
|
-
let
|
|
7
|
+
let shared = null;
|
|
6
8
|
/** Get data path from environment (priority: BUNQUEUE_DATA_PATH > BQ_DATA_PATH > DATA_PATH) */
|
|
7
9
|
function getDataPath() {
|
|
8
10
|
return (Bun.env.BUNQUEUE_DATA_PATH ?? Bun.env.BQ_DATA_PATH ?? Bun.env.DATA_PATH ?? Bun.env.SQLITE_PATH);
|
|
9
11
|
}
|
|
10
|
-
|
|
12
|
+
function normalizeDataPath(dataPath) {
|
|
13
|
+
if (!dataPath)
|
|
14
|
+
return undefined;
|
|
15
|
+
if (dataPath === ':memory:')
|
|
16
|
+
return dataPath;
|
|
17
|
+
const absolutePath = resolve(dataPath);
|
|
18
|
+
try {
|
|
19
|
+
return realpathSync.native(absolutePath);
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
try {
|
|
23
|
+
return join(realpathSync.native(dirname(absolutePath)), basename(absolutePath));
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return absolutePath;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
function displayDataPath(dataPath) {
|
|
31
|
+
return JSON.stringify(dataPath ?? '<in-memory>');
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Get the process-wide QueueManager.
|
|
35
|
+
* A later explicit dataPath must identify the database selected on first use.
|
|
36
|
+
*/
|
|
11
37
|
export function getSharedManager(dataPath) {
|
|
12
|
-
|
|
38
|
+
if (shared) {
|
|
39
|
+
if (dataPath !== undefined) {
|
|
40
|
+
const requestedDataPath = normalizeDataPath(dataPath);
|
|
41
|
+
if (requestedDataPath !== shared.dataPath) {
|
|
42
|
+
throw new Error(`Embedded QueueManager dataPath conflict: already initialized with ${displayDataPath(shared.dataPath)}; ` +
|
|
43
|
+
`cannot use ${displayDataPath(requestedDataPath)}. Reuse the active dataPath, or close all ` +
|
|
44
|
+
'embedded clients and call shutdownManager() before switching databases.');
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return shared.instance;
|
|
48
|
+
}
|
|
49
|
+
const selectedDataPath = normalizeDataPath(dataPath ?? getDataPath());
|
|
50
|
+
const instance = new QueueManager({ dataPath: selectedDataPath });
|
|
51
|
+
shared = { instance, dataPath: selectedDataPath };
|
|
13
52
|
return instance;
|
|
14
53
|
}
|
|
15
54
|
/** Shutdown shared manager */
|
|
16
55
|
export function shutdownManager() {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
}
|
|
56
|
+
const current = shared;
|
|
57
|
+
shared = null;
|
|
58
|
+
current?.instance.shutdown();
|
|
21
59
|
}
|
|
@@ -11,6 +11,7 @@ export declare class QueueState<T> {
|
|
|
11
11
|
protected readonly tcpPool: TcpConnectionPool | null;
|
|
12
12
|
protected readonly useSharedPool: boolean;
|
|
13
13
|
protected readonly addBatcher: AddBatcher<unknown> | null;
|
|
14
|
+
private connectionReleased;
|
|
14
15
|
constructor(name: string, opts?: QueueOptions);
|
|
15
16
|
protected get ctx(): {
|
|
16
17
|
name: string;
|
|
@@ -14,6 +14,7 @@ export class QueueState {
|
|
|
14
14
|
tcpPool;
|
|
15
15
|
useSharedPool;
|
|
16
16
|
addBatcher;
|
|
17
|
+
connectionReleased = false;
|
|
17
18
|
constructor(name, opts = {}) {
|
|
18
19
|
this.name = name;
|
|
19
20
|
this.prefixKey = opts.prefixKey ?? '';
|
|
@@ -139,6 +140,9 @@ export class QueueState {
|
|
|
139
140
|
};
|
|
140
141
|
}
|
|
141
142
|
releaseConnection() {
|
|
143
|
+
if (this.connectionReleased)
|
|
144
|
+
return;
|
|
145
|
+
this.connectionReleased = true;
|
|
142
146
|
if (!this.tcpPool)
|
|
143
147
|
return;
|
|
144
148
|
if (this.useSharedPool)
|
|
@@ -23,6 +23,7 @@ export interface QueueOptions {
|
|
|
23
23
|
defaultJobOptions?: JobOptions;
|
|
24
24
|
connection?: ConnectionOptions;
|
|
25
25
|
embedded?: boolean;
|
|
26
|
+
/** Must match the process-wide embedded manager when one is already active. */
|
|
26
27
|
dataPath?: string;
|
|
27
28
|
autoBatch?: AutoBatchOptions;
|
|
28
29
|
prefixKey?: string;
|
|
@@ -4,7 +4,7 @@ export interface QueueEventsOptions {
|
|
|
4
4
|
embedded?: boolean;
|
|
5
5
|
/** TCP connection used when embedded is false. */
|
|
6
6
|
connection?: ConnectionOptions;
|
|
7
|
-
/** Embedded SQLite path. */
|
|
7
|
+
/** Embedded SQLite path; must match an already-active shared manager. */
|
|
8
8
|
dataPath?: string;
|
|
9
9
|
/** Namespace prefix applied to the queue name. */
|
|
10
10
|
prefixKey?: string;
|
|
@@ -10,6 +10,7 @@ export interface WorkerOptions {
|
|
|
10
10
|
heartbeatInterval?: number;
|
|
11
11
|
connection?: ConnectionOptions;
|
|
12
12
|
embedded?: boolean;
|
|
13
|
+
/** Must match the process-wide embedded manager when one is already active. */
|
|
13
14
|
dataPath?: string;
|
|
14
15
|
batchSize?: number;
|
|
15
16
|
pollTimeout?: number;
|
|
@@ -6,13 +6,10 @@ import { startHeartbeat } from '../workerHeartbeat';
|
|
|
6
6
|
import { WorkerState } from './state';
|
|
7
7
|
export class WorkerControl extends WorkerState {
|
|
8
8
|
run() {
|
|
9
|
-
if (this.running || this.closed)
|
|
9
|
+
if (this.running || this.closed || this._closing || this._closingPromise !== null)
|
|
10
10
|
return;
|
|
11
11
|
this.running = true;
|
|
12
12
|
this.paused = false;
|
|
13
|
-
this._closing = false;
|
|
14
|
-
this._forceClose = false;
|
|
15
|
-
this._closingPromise = null;
|
|
16
13
|
queueMicrotask(() => {
|
|
17
14
|
if (!this.closed)
|
|
18
15
|
this.emit('ready');
|
|
@@ -88,10 +85,7 @@ export class WorkerControl extends WorkerState {
|
|
|
88
85
|
return;
|
|
89
86
|
this.running = false;
|
|
90
87
|
this.paused = true;
|
|
91
|
-
|
|
92
|
-
clearTimeout(this.pollTimer);
|
|
93
|
-
this.pollTimer = null;
|
|
94
|
-
}
|
|
88
|
+
this.clearPollTimer();
|
|
95
89
|
}
|
|
96
90
|
resume() {
|
|
97
91
|
if (this.closed)
|
|
@@ -19,10 +19,7 @@ export class WorkerLifecycle extends WorkerManual {
|
|
|
19
19
|
this._closing = true;
|
|
20
20
|
this.running = false;
|
|
21
21
|
this.paused = false;
|
|
22
|
-
|
|
23
|
-
clearTimeout(this.pollTimer);
|
|
24
|
-
this.pollTimer = null;
|
|
25
|
-
}
|
|
22
|
+
this.clearPollTimer();
|
|
26
23
|
if (this.heartbeatTimer) {
|
|
27
24
|
clearInterval(this.heartbeatTimer);
|
|
28
25
|
this.heartbeatTimer = null;
|
|
@@ -6,6 +6,7 @@ export declare abstract class WorkerPolling<T = unknown, R = unknown> extends Wo
|
|
|
6
6
|
protected tryProcess(): Promise<void>;
|
|
7
7
|
protected doPullBatch(): Promise<PulledJob[]>;
|
|
8
8
|
private scheduleRateLimitPoll;
|
|
9
|
+
private schedulePoll;
|
|
9
10
|
protected handlePullError(errorValue: unknown): void;
|
|
10
11
|
protected abstract startJob(delivery: WorkerDelivery): boolean;
|
|
11
12
|
protected abstract getPullConfig(): PullConfig;
|
|
@@ -3,12 +3,11 @@ import { pullEmbedded, pullTcp } from '../workerPull';
|
|
|
3
3
|
import { WorkerBuffer } from './buffer';
|
|
4
4
|
export class WorkerPolling extends WorkerBuffer {
|
|
5
5
|
poll() {
|
|
6
|
+
this.clearPollTimer();
|
|
6
7
|
if (!this.running || this._closing)
|
|
7
8
|
return;
|
|
8
9
|
if (this.activeJobs >= this.opts.concurrency) {
|
|
9
|
-
this.
|
|
10
|
-
this.poll();
|
|
11
|
-
}, 10);
|
|
10
|
+
this.schedulePoll(10);
|
|
12
11
|
return;
|
|
13
12
|
}
|
|
14
13
|
if (!this.rateLimiter.canProcessWithinLimit()) {
|
|
@@ -46,9 +45,7 @@ export class WorkerPolling extends WorkerBuffer {
|
|
|
46
45
|
if (item) {
|
|
47
46
|
if (this.activeJobs >= this.opts.concurrency) {
|
|
48
47
|
this.requeueItem(item);
|
|
49
|
-
this.
|
|
50
|
-
this.poll();
|
|
51
|
-
}, 10);
|
|
48
|
+
this.schedulePoll(10);
|
|
52
49
|
return;
|
|
53
50
|
}
|
|
54
51
|
this.consecutiveErrors = 0;
|
|
@@ -60,9 +57,7 @@ export class WorkerPolling extends WorkerBuffer {
|
|
|
60
57
|
else {
|
|
61
58
|
const hasBuffered = this.pendingJobsHead < this.pendingJobs.length;
|
|
62
59
|
if (hasBuffered && this.groupLimiter) {
|
|
63
|
-
this.
|
|
64
|
-
this.poll();
|
|
65
|
-
}, 10);
|
|
60
|
+
this.schedulePoll(10);
|
|
66
61
|
return;
|
|
67
62
|
}
|
|
68
63
|
const now = Date.now();
|
|
@@ -71,9 +66,7 @@ export class WorkerPolling extends WorkerBuffer {
|
|
|
71
66
|
this.emit('drained');
|
|
72
67
|
}
|
|
73
68
|
const waitTime = this.opts.pollTimeout > 0 ? 10 : this.opts.drainDelay;
|
|
74
|
-
this.
|
|
75
|
-
this.poll();
|
|
76
|
-
}, waitTime);
|
|
69
|
+
this.schedulePoll(waitTime);
|
|
77
70
|
}
|
|
78
71
|
}
|
|
79
72
|
catch (error) {
|
|
@@ -103,9 +96,25 @@ export class WorkerPolling extends WorkerBuffer {
|
|
|
103
96
|
}
|
|
104
97
|
scheduleRateLimitPoll() {
|
|
105
98
|
const waitTime = this.rateLimiter.getTimeUntilNextSlot();
|
|
106
|
-
this.
|
|
99
|
+
this.schedulePoll(Math.max(waitTime, 10));
|
|
100
|
+
}
|
|
101
|
+
schedulePoll(delay) {
|
|
102
|
+
if (!this.running || this._closing || this.closed)
|
|
103
|
+
return;
|
|
104
|
+
const deadline = Date.now() + delay;
|
|
105
|
+
if (this.pollTimer !== null && this.pollDeadline !== null && this.pollDeadline <= deadline) {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
this.clearPollTimer();
|
|
109
|
+
const timer = setTimeout(() => {
|
|
110
|
+
if (this.pollTimer !== timer)
|
|
111
|
+
return;
|
|
112
|
+
this.pollTimer = null;
|
|
113
|
+
this.pollDeadline = null;
|
|
107
114
|
this.poll();
|
|
108
|
-
},
|
|
115
|
+
}, delay);
|
|
116
|
+
this.pollTimer = timer;
|
|
117
|
+
this.pollDeadline = deadline;
|
|
109
118
|
}
|
|
110
119
|
handlePullError(errorValue) {
|
|
111
120
|
this.consecutiveErrors++;
|
|
@@ -116,8 +125,6 @@ export class WorkerPolling extends WorkerBuffer {
|
|
|
116
125
|
context: 'pull',
|
|
117
126
|
}));
|
|
118
127
|
const backoffMs = Math.min(WORKER_CONSTANTS.BASE_BACKOFF_MS * Math.pow(2, this.consecutiveErrors - 1), WORKER_CONSTANTS.MAX_BACKOFF_MS);
|
|
119
|
-
this.
|
|
120
|
-
this.poll();
|
|
121
|
-
}, backoffMs);
|
|
128
|
+
this.schedulePoll(backoffMs);
|
|
122
129
|
}
|
|
123
130
|
}
|
|
@@ -34,6 +34,7 @@ export declare abstract class WorkerState<T = unknown, R = unknown> extends Even
|
|
|
34
34
|
protected closed: boolean;
|
|
35
35
|
protected activeJobs: number;
|
|
36
36
|
protected pollTimer: ReturnType<typeof setTimeout> | null;
|
|
37
|
+
protected pollDeadline: number | null;
|
|
37
38
|
protected consecutiveErrors: number;
|
|
38
39
|
protected readonly pulledJobIds: Set<string>;
|
|
39
40
|
protected readonly jobTokens: Map<string, string>;
|
|
@@ -102,6 +103,7 @@ export declare abstract class WorkerState<T = unknown, R = unknown> extends Even
|
|
|
102
103
|
protected hasActiveDelivery(jobId: string): boolean;
|
|
103
104
|
protected activeDeliveryIds(): IterableIterator<string>;
|
|
104
105
|
protected clearDeliveries(): void;
|
|
106
|
+
protected clearPollTimer(): void;
|
|
105
107
|
abstract run(): void;
|
|
106
108
|
protected abstract registerWithServer(): void;
|
|
107
109
|
protected abstract getHeartbeatDeps(): HeartbeatDeps;
|
|
@@ -24,6 +24,7 @@ export class WorkerState extends EventEmitter {
|
|
|
24
24
|
closed = false;
|
|
25
25
|
activeJobs = 0;
|
|
26
26
|
pollTimer = null;
|
|
27
|
+
pollDeadline = null;
|
|
27
28
|
consecutiveErrors = 0;
|
|
28
29
|
pulledJobIds = new Set();
|
|
29
30
|
jobTokens = new Map();
|
|
@@ -153,4 +154,11 @@ export class WorkerState extends EventEmitter {
|
|
|
153
154
|
this.currentDeliveries.clear();
|
|
154
155
|
this.activeDeliveries.clear();
|
|
155
156
|
}
|
|
157
|
+
clearPollTimer() {
|
|
158
|
+
const timer = this.pollTimer;
|
|
159
|
+
this.pollTimer = null;
|
|
160
|
+
this.pollDeadline = null;
|
|
161
|
+
if (timer !== null)
|
|
162
|
+
clearTimeout(timer);
|
|
163
|
+
}
|
|
156
164
|
}
|
|
@@ -19,6 +19,7 @@ export declare class S3BackupManager {
|
|
|
19
19
|
private readonly telemetry;
|
|
20
20
|
private backupInterval;
|
|
21
21
|
private initialBackupTimeout;
|
|
22
|
+
private schedulerGeneration;
|
|
22
23
|
private dashboardEmit;
|
|
23
24
|
/** Set the dashboard event emitter callback */
|
|
24
25
|
setDashboardEmit(callback: (event: string, data: Record<string, unknown>) => void): void;
|
|
@@ -20,6 +20,7 @@ export class S3BackupManager {
|
|
|
20
20
|
telemetry;
|
|
21
21
|
backupInterval = null;
|
|
22
22
|
initialBackupTimeout = null;
|
|
23
|
+
schedulerGeneration = 0;
|
|
23
24
|
dashboardEmit = null;
|
|
24
25
|
/** Set the dashboard event emitter callback */
|
|
25
26
|
setDashboardEmit(callback) {
|
|
@@ -73,30 +74,43 @@ export class S3BackupManager {
|
|
|
73
74
|
if (!this.config.enabled) {
|
|
74
75
|
return;
|
|
75
76
|
}
|
|
77
|
+
if (this.initialBackupTimeout !== null || this.backupInterval !== null)
|
|
78
|
+
return;
|
|
76
79
|
const validation = this.validate();
|
|
77
80
|
if (!validation.valid) {
|
|
78
81
|
backupLog.error('S3 backup configuration invalid', { errors: validation.errors });
|
|
79
82
|
return;
|
|
80
83
|
}
|
|
84
|
+
const generation = ++this.schedulerGeneration;
|
|
81
85
|
// Run initial backup after 1 minute
|
|
82
|
-
|
|
86
|
+
const initialBackupTimeout = setTimeout(() => {
|
|
87
|
+
if (generation !== this.schedulerGeneration ||
|
|
88
|
+
this.initialBackupTimeout !== initialBackupTimeout) {
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
83
91
|
this.initialBackupTimeout = null;
|
|
84
92
|
this.backup().catch((err) => {
|
|
85
93
|
backupLog.error('Initial backup failed', { error: String(err) });
|
|
86
94
|
});
|
|
87
95
|
}, 60 * 1000);
|
|
96
|
+
this.initialBackupTimeout = initialBackupTimeout;
|
|
88
97
|
// Schedule periodic backups
|
|
89
|
-
|
|
98
|
+
const backupInterval = setInterval(() => {
|
|
99
|
+
if (generation !== this.schedulerGeneration || this.backupInterval !== backupInterval) {
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
90
102
|
this.backup().catch((err) => {
|
|
91
103
|
backupLog.error('Scheduled backup failed', { error: String(err) });
|
|
92
104
|
});
|
|
93
105
|
}, this.config.intervalMs);
|
|
106
|
+
this.backupInterval = backupInterval;
|
|
94
107
|
this.telemetry.setSchedulerRunning(true);
|
|
95
108
|
}
|
|
96
109
|
/**
|
|
97
110
|
* Stop automated backup scheduler
|
|
98
111
|
*/
|
|
99
112
|
stop() {
|
|
113
|
+
this.schedulerGeneration++;
|
|
100
114
|
if (this.initialBackupTimeout) {
|
|
101
115
|
clearTimeout(this.initialBackupTimeout);
|
|
102
116
|
this.initialBackupTimeout = null;
|
|
@@ -27,6 +27,7 @@ export class CloudAgent {
|
|
|
27
27
|
statsUpdateTimer = null;
|
|
28
28
|
unsubscribeEvents = null;
|
|
29
29
|
sequenceId = 0;
|
|
30
|
+
started = false;
|
|
30
31
|
stopped = false;
|
|
31
32
|
serverHandles;
|
|
32
33
|
/** Event buffer — flushed into each HTTP snapshot */
|
|
@@ -59,6 +60,9 @@ export class CloudAgent {
|
|
|
59
60
|
}
|
|
60
61
|
/** Start both channels */
|
|
61
62
|
start() {
|
|
63
|
+
if (this.started || this.stopped)
|
|
64
|
+
return;
|
|
65
|
+
this.started = true;
|
|
62
66
|
cloudLog.info('Connecting to dashboard', {
|
|
63
67
|
url: this.config.url,
|
|
64
68
|
instance: this.config.instanceName,
|
|
@@ -156,15 +160,19 @@ export class CloudAgent {
|
|
|
156
160
|
}
|
|
157
161
|
/** Schedule next snapshot with dynamic interval */
|
|
158
162
|
scheduleNext() {
|
|
159
|
-
if (this.stopped)
|
|
163
|
+
if (this.stopped || !this.started)
|
|
160
164
|
return;
|
|
161
165
|
const intervalMs = this.computeInterval();
|
|
162
166
|
cloudLog.debug('Next snapshot', { intervalMs, compressedKB: this.httpSender.lastCompressedKB });
|
|
163
|
-
|
|
167
|
+
const snapshotTimer = setTimeout(() => {
|
|
168
|
+
if (this.stopped || this.snapshotTimer !== snapshotTimer)
|
|
169
|
+
return;
|
|
170
|
+
this.snapshotTimer = null;
|
|
164
171
|
void this.sendSnapshot().then(() => {
|
|
165
172
|
this.scheduleNext();
|
|
166
173
|
});
|
|
167
174
|
}, intervalMs);
|
|
175
|
+
this.snapshotTimer = snapshotTimer;
|
|
168
176
|
}
|
|
169
177
|
/** Collect and send a snapshot via HTTP */
|
|
170
178
|
async sendSnapshot(_forceHeavy = false) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bunqueue",
|
|
3
|
-
"version": "2.8.
|
|
3
|
+
"version": "2.8.58",
|
|
4
4
|
"description": "High-performance job queue for Bun & AI agents. SQLite persistence, cron scheduling, priorities, retries, DLQ, webhooks, native MCP server. Zero external infrastructure.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/main.js",
|
|
@@ -1,148 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Shared types for QueueManager modules
|
|
3
|
-
*/
|
|
4
|
-
import type { Job, JobId, JobLock } from '../domain/types/job';
|
|
5
|
-
import type { JobLocation } from '../domain/types/queue';
|
|
6
|
-
import type { JobLogEntry } from '../domain/types/worker';
|
|
7
|
-
import type { Shard } from '../domain/queue/shard';
|
|
8
|
-
import type { SqliteStorage } from '../infrastructure/persistence/sqlite';
|
|
9
|
-
import type { RWLock } from '../shared/lock';
|
|
10
|
-
import type { LRUMap, BoundedSet, BoundedMap, SetLike, MapLike } from '../shared/lru';
|
|
11
|
-
import type { EventsManager } from './eventsManager';
|
|
12
|
-
import type { WebhookManager } from './webhookManager';
|
|
13
|
-
import type { WorkerManager } from './workerManager';
|
|
14
|
-
import type { MonitoringState } from './monitoringChecks';
|
|
15
|
-
import type { DependencyResultTracker } from './dependencyResultTracker';
|
|
16
|
-
import type { DependencyCompletionTracker } from './dependencyCompletions';
|
|
17
|
-
/** Queue Manager configuration */
|
|
18
|
-
export interface QueueManagerConfig {
|
|
19
|
-
dataPath?: string;
|
|
20
|
-
maxCompletedJobs?: number;
|
|
21
|
-
maxJobResults?: number;
|
|
22
|
-
maxJobLogs?: number;
|
|
23
|
-
maxCustomIds?: number;
|
|
24
|
-
maxWaitingDeps?: number;
|
|
25
|
-
/** Maximum queue label values emitted in one Prometheus scrape; zero disables them. */
|
|
26
|
-
maxPrometheusQueues?: number;
|
|
27
|
-
cleanupIntervalMs?: number;
|
|
28
|
-
jobTimeoutCheckMs?: number;
|
|
29
|
-
dependencyCheckMs?: number;
|
|
30
|
-
stallCheckMs?: number;
|
|
31
|
-
dlqMaintenanceMs?: number;
|
|
32
|
-
validateWebhookUrls?: boolean;
|
|
33
|
-
}
|
|
34
|
-
export declare const DEFAULT_CONFIG: {
|
|
35
|
-
maxCompletedJobs: number;
|
|
36
|
-
maxJobResults: number;
|
|
37
|
-
maxJobLogs: number;
|
|
38
|
-
maxCustomIds: number;
|
|
39
|
-
maxWaitingDeps: number;
|
|
40
|
-
maxPrometheusQueues: number;
|
|
41
|
-
cleanupIntervalMs: number;
|
|
42
|
-
jobTimeoutCheckMs: number;
|
|
43
|
-
dependencyCheckMs: number;
|
|
44
|
-
stallCheckMs: number;
|
|
45
|
-
dlqMaintenanceMs: number;
|
|
46
|
-
};
|
|
47
|
-
/** Shared state accessible by all modules */
|
|
48
|
-
export interface QueueManagerState {
|
|
49
|
-
readonly config: typeof DEFAULT_CONFIG & {
|
|
50
|
-
dataPath?: string;
|
|
51
|
-
};
|
|
52
|
-
readonly storage: SqliteStorage | null;
|
|
53
|
-
readonly shards: Shard[];
|
|
54
|
-
readonly shardLocks: RWLock[];
|
|
55
|
-
readonly processingShards: Map<JobId, Job>[];
|
|
56
|
-
readonly processingLocks: RWLock[];
|
|
57
|
-
readonly jobIndex: Map<JobId, JobLocation>;
|
|
58
|
-
readonly completedJobs: BoundedSet<JobId>;
|
|
59
|
-
readonly jobResults: LRUMap<JobId, unknown>;
|
|
60
|
-
readonly dependencyResults: DependencyResultTracker;
|
|
61
|
-
readonly customIdMap: LRUMap<string, JobId>;
|
|
62
|
-
readonly jobLogs: LRUMap<JobId, JobLogEntry[]>;
|
|
63
|
-
readonly jobLocks: Map<JobId, JobLock>;
|
|
64
|
-
readonly clientJobs: Map<string, Set<JobId>>;
|
|
65
|
-
readonly stalledCandidates: Set<JobId>;
|
|
66
|
-
readonly pendingDepChecks: Set<JobId>;
|
|
67
|
-
readonly queueNamesCache: Set<string>;
|
|
68
|
-
readonly eventsManager: EventsManager;
|
|
69
|
-
readonly webhookManager: WebhookManager;
|
|
70
|
-
readonly metrics: {
|
|
71
|
-
totalPushed: {
|
|
72
|
-
value: bigint;
|
|
73
|
-
};
|
|
74
|
-
totalPulled: {
|
|
75
|
-
value: bigint;
|
|
76
|
-
};
|
|
77
|
-
totalCompleted: {
|
|
78
|
-
value: bigint;
|
|
79
|
-
};
|
|
80
|
-
totalFailed: {
|
|
81
|
-
value: bigint;
|
|
82
|
-
};
|
|
83
|
-
};
|
|
84
|
-
readonly startTime: number;
|
|
85
|
-
readonly perQueueMetrics: MapLike<string, {
|
|
86
|
-
totalCompleted: bigint;
|
|
87
|
-
totalFailed: bigint;
|
|
88
|
-
}>;
|
|
89
|
-
}
|
|
90
|
-
/** Context for lock operations */
|
|
91
|
-
export interface LockContext {
|
|
92
|
-
jobIndex: Map<JobId, JobLocation>;
|
|
93
|
-
jobLocks: Map<JobId, JobLock>;
|
|
94
|
-
clientJobs: Map<string, Set<JobId>>;
|
|
95
|
-
processingShards: Map<JobId, Job>[];
|
|
96
|
-
processingLocks: RWLock[];
|
|
97
|
-
shards: Shard[];
|
|
98
|
-
shardLocks: RWLock[];
|
|
99
|
-
eventsManager: EventsManager;
|
|
100
|
-
dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
|
|
101
|
-
storage: SqliteStorage | null;
|
|
102
|
-
}
|
|
103
|
-
/** Context for background tasks */
|
|
104
|
-
export interface BackgroundContext extends QueueManagerState {
|
|
105
|
-
fail: (jobId: JobId, error?: string) => Promise<void>;
|
|
106
|
-
registerQueueName: (queue: string) => void;
|
|
107
|
-
unregisterQueueName: (queue: string) => void;
|
|
108
|
-
dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
|
|
109
|
-
workerManager: WorkerManager;
|
|
110
|
-
monitoringState: MonitoringState;
|
|
111
|
-
completedJobsData: BoundedMap<JobId, Job>;
|
|
112
|
-
depCompletions?: DependencyCompletionTracker;
|
|
113
|
-
maxDependencyCompletions: number;
|
|
114
|
-
timedOutJobs?: BoundedSet<JobId>;
|
|
115
|
-
}
|
|
116
|
-
/** Context for stats operations */
|
|
117
|
-
export interface StatsContext {
|
|
118
|
-
shards: Shard[];
|
|
119
|
-
processingShards: Map<JobId, Job>[];
|
|
120
|
-
completedJobs: SetLike<JobId>;
|
|
121
|
-
jobIndex: Map<JobId, JobLocation>;
|
|
122
|
-
jobResults: LRUMap<JobId, unknown>;
|
|
123
|
-
jobLogs: LRUMap<JobId, JobLogEntry[]>;
|
|
124
|
-
customIdMap: LRUMap<string, JobId>;
|
|
125
|
-
jobLocks: Map<JobId, JobLock>;
|
|
126
|
-
clientJobs: Map<string, Set<JobId>>;
|
|
127
|
-
pendingDepChecks: Set<JobId>;
|
|
128
|
-
stalledCandidates: Set<JobId>;
|
|
129
|
-
metrics: {
|
|
130
|
-
totalPushed: {
|
|
131
|
-
value: bigint;
|
|
132
|
-
};
|
|
133
|
-
totalPulled: {
|
|
134
|
-
value: bigint;
|
|
135
|
-
};
|
|
136
|
-
totalCompleted: {
|
|
137
|
-
value: bigint;
|
|
138
|
-
};
|
|
139
|
-
totalFailed: {
|
|
140
|
-
value: bigint;
|
|
141
|
-
};
|
|
142
|
-
};
|
|
143
|
-
startTime: number;
|
|
144
|
-
perQueueMetrics?: MapLike<string, {
|
|
145
|
-
totalCompleted: bigint;
|
|
146
|
-
totalFailed: bigint;
|
|
147
|
-
}>;
|
|
148
|
-
}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Shared types for QueueManager modules
|
|
3
|
-
*/
|
|
4
|
-
export const DEFAULT_CONFIG = {
|
|
5
|
-
maxCompletedJobs: 50_000,
|
|
6
|
-
maxJobResults: 10_000,
|
|
7
|
-
maxJobLogs: 10_000,
|
|
8
|
-
maxCustomIds: 50_000,
|
|
9
|
-
maxWaitingDeps: 10_000,
|
|
10
|
-
maxPrometheusQueues: 100,
|
|
11
|
-
cleanupIntervalMs: 10_000,
|
|
12
|
-
jobTimeoutCheckMs: 5_000,
|
|
13
|
-
dependencyCheckMs: 30_000, // Safety fallback only; event-driven handles fast path
|
|
14
|
-
stallCheckMs: 5_000,
|
|
15
|
-
dlqMaintenanceMs: 60_000,
|
|
16
|
-
};
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Queue Internal Types
|
|
3
|
-
* Shared types for queue module operations
|
|
4
|
-
*/
|
|
5
|
-
import type { TcpConnectionPool } from '../tcpPool';
|
|
6
|
-
import type { QueueOptions, JobStateType } from '../types';
|
|
7
|
-
/** Internal queue context for operations */
|
|
8
|
-
export interface QueueContext {
|
|
9
|
-
name: string;
|
|
10
|
-
opts: QueueOptions;
|
|
11
|
-
embedded: boolean;
|
|
12
|
-
tcp: TcpConnectionPool | null;
|
|
13
|
-
getJobState: (id: string) => Promise<JobStateType>;
|
|
14
|
-
removeAsync: (id: string) => Promise<void>;
|
|
15
|
-
retryJob: (id: string) => Promise<void>;
|
|
16
|
-
getChildrenValues: (id: string) => Promise<Record<string, unknown>>;
|
|
17
|
-
}
|
|
18
|
-
/** Job creation context for proxy */
|
|
19
|
-
export interface JobCreationContext {
|
|
20
|
-
queueName: string;
|
|
21
|
-
tcp: TcpConnectionPool | null;
|
|
22
|
-
getJobState: (id: string) => Promise<string>;
|
|
23
|
-
removeAsync: (id: string) => Promise<void>;
|
|
24
|
-
retryJob: (id: string) => Promise<void>;
|
|
25
|
-
getChildrenValues: (id: string) => Promise<Record<string, unknown>>;
|
|
26
|
-
}
|
|
27
|
-
/** TCP response with optional data */
|
|
28
|
-
export interface TcpResponse {
|
|
29
|
-
ok: boolean;
|
|
30
|
-
error?: string;
|
|
31
|
-
data?: unknown;
|
|
32
|
-
[key: string]: unknown;
|
|
33
|
-
}
|
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Sandboxed Worker Types
|
|
3
|
-
* Type definitions for sandboxed worker processes
|
|
4
|
-
*/
|
|
5
|
-
import type { SharedManager } from '../manager';
|
|
6
|
-
import type { ConnectionOptions } from '../types';
|
|
7
|
-
import type { Job as DomainJob } from '../../domain/types/job';
|
|
8
|
-
/** Sandboxed worker configuration */
|
|
9
|
-
export interface SandboxedWorkerOptions {
|
|
10
|
-
/** Path to processor file (must export default async function) */
|
|
11
|
-
processor: string;
|
|
12
|
-
/** Number of worker processes (default: 1) */
|
|
13
|
-
concurrency?: number;
|
|
14
|
-
/** Max memory per worker in MB - uses smol mode if <= 64 (default: 256) */
|
|
15
|
-
maxMemory?: number;
|
|
16
|
-
/** Job timeout in ms (default: 30000) */
|
|
17
|
-
timeout?: number;
|
|
18
|
-
/** Auto-restart crashed workers (default: true) */
|
|
19
|
-
autoRestart?: boolean;
|
|
20
|
-
/** Max restarts before giving up (default: 10) */
|
|
21
|
-
maxRestarts?: number;
|
|
22
|
-
/** Poll interval when no workers are idle (default: 10ms) */
|
|
23
|
-
pollInterval?: number;
|
|
24
|
-
/** Custom QueueManager (for testing, defaults to shared manager) */
|
|
25
|
-
manager?: SharedManager;
|
|
26
|
-
/** TCP connection options (if provided, uses TCP mode instead of embedded) */
|
|
27
|
-
connection?: ConnectionOptions;
|
|
28
|
-
/** Heartbeat interval in ms for TCP lock renewal (default: 10000 for TCP, 0 for embedded) */
|
|
29
|
-
heartbeatInterval?: number;
|
|
30
|
-
/** Auto-stop after this many ms of inactivity (0 = disabled, default: 0) */
|
|
31
|
-
idleTimeout?: number;
|
|
32
|
-
/** Recycle individual idle worker processes after this many ms (default: 30000, 0 = disabled) */
|
|
33
|
-
idleRecycleMs?: number;
|
|
34
|
-
/** Auto-restart worker pool when new jobs arrive after idle shutdown (default: false) */
|
|
35
|
-
autoStart?: boolean;
|
|
36
|
-
/** Poll interval in ms for checking new jobs while in idle-shutdown state (default: 5000) */
|
|
37
|
-
autoStartPollMs?: number;
|
|
38
|
-
}
|
|
39
|
-
/** Required options with defaults applied */
|
|
40
|
-
export interface RequiredSandboxedWorkerOptions {
|
|
41
|
-
processor: string;
|
|
42
|
-
concurrency: number;
|
|
43
|
-
maxMemory: number;
|
|
44
|
-
timeout: number;
|
|
45
|
-
autoRestart: boolean;
|
|
46
|
-
maxRestarts: number;
|
|
47
|
-
pollInterval: number;
|
|
48
|
-
}
|
|
49
|
-
/** Worker process state */
|
|
50
|
-
export interface WorkerProcess {
|
|
51
|
-
worker: Worker;
|
|
52
|
-
busy: boolean;
|
|
53
|
-
currentJob: DomainJob | null;
|
|
54
|
-
currentToken: string | null;
|
|
55
|
-
restarts: number;
|
|
56
|
-
timeoutId: Timer | null;
|
|
57
|
-
lastIdleAt: number;
|
|
58
|
-
terminated: boolean;
|
|
59
|
-
}
|
|
60
|
-
/** IPC message from main to worker */
|
|
61
|
-
export interface IPCRequest {
|
|
62
|
-
type: 'job';
|
|
63
|
-
job: {
|
|
64
|
-
id: string;
|
|
65
|
-
data: unknown;
|
|
66
|
-
queue: string;
|
|
67
|
-
attempts: number;
|
|
68
|
-
parentId?: string;
|
|
69
|
-
};
|
|
70
|
-
}
|
|
71
|
-
/** IPC message from worker to main */
|
|
72
|
-
export interface IPCResponse {
|
|
73
|
-
type: 'result' | 'error' | 'progress' | 'log' | 'fail' | 'ready';
|
|
74
|
-
jobId?: string;
|
|
75
|
-
result?: unknown;
|
|
76
|
-
error?: string;
|
|
77
|
-
progress?: number;
|
|
78
|
-
message?: string;
|
|
79
|
-
}
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Worker Types
|
|
3
|
-
* Type definitions for worker module
|
|
4
|
-
*/
|
|
5
|
-
import type { ConnectionOptions } from '../types';
|
|
6
|
-
/** Pending ACK item with result and optional lock token */
|
|
7
|
-
export interface PendingAck {
|
|
8
|
-
id: string;
|
|
9
|
-
result: unknown;
|
|
10
|
-
token?: string;
|
|
11
|
-
resolve: () => void;
|
|
12
|
-
reject: (err: Error) => void;
|
|
13
|
-
}
|
|
14
|
-
/** Extended options with all defaults */
|
|
15
|
-
export interface ExtendedWorkerOptions {
|
|
16
|
-
concurrency: number;
|
|
17
|
-
autorun: boolean;
|
|
18
|
-
heartbeatInterval: number;
|
|
19
|
-
batchSize: number;
|
|
20
|
-
pollTimeout: number;
|
|
21
|
-
embedded: boolean;
|
|
22
|
-
useLocks: boolean;
|
|
23
|
-
skipLockRenewal: boolean;
|
|
24
|
-
skipStalledCheck: boolean;
|
|
25
|
-
drainDelay: number;
|
|
26
|
-
lockDuration: number;
|
|
27
|
-
maxStalledCount: number;
|
|
28
|
-
removeOnComplete?: boolean | number | {
|
|
29
|
-
age?: number;
|
|
30
|
-
count?: number;
|
|
31
|
-
};
|
|
32
|
-
removeOnFail?: boolean | number | {
|
|
33
|
-
age?: number;
|
|
34
|
-
count?: number;
|
|
35
|
-
};
|
|
36
|
-
connection?: ConnectionOptions;
|
|
37
|
-
}
|
|
38
|
-
/** TCP connection interface */
|
|
39
|
-
export interface TcpConnection {
|
|
40
|
-
send: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
|
41
|
-
}
|
|
42
|
-
/** Check if embedded mode should be forced (for tests) */
|
|
43
|
-
export declare const FORCE_EMBEDDED: boolean;
|
|
44
|
-
/** Worker constants */
|
|
45
|
-
export declare const WORKER_CONSTANTS: {
|
|
46
|
-
readonly MAX_BACKOFF_MS: 30000;
|
|
47
|
-
readonly BASE_BACKOFF_MS: 100;
|
|
48
|
-
readonly MAX_POLL_TIMEOUT: 30000;
|
|
49
|
-
readonly DEFAULT_ACK_INTERVAL: 50;
|
|
50
|
-
};
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Worker Types
|
|
3
|
-
* Type definitions for worker module
|
|
4
|
-
*/
|
|
5
|
-
/** Check if embedded mode should be forced (for tests) */
|
|
6
|
-
export const FORCE_EMBEDDED = Bun.env.BUNQUEUE_EMBEDDED === '1';
|
|
7
|
-
/** Worker constants */
|
|
8
|
-
export const WORKER_CONSTANTS = {
|
|
9
|
-
MAX_BACKOFF_MS: 30_000,
|
|
10
|
-
BASE_BACKOFF_MS: 100,
|
|
11
|
-
MAX_POLL_TIMEOUT: 30_000,
|
|
12
|
-
DEFAULT_ACK_INTERVAL: 50,
|
|
13
|
-
};
|