bunqueue 1.6.1 → 1.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/eventsManager.d.ts +2 -1
- package/dist/application/eventsManager.d.ts.map +1 -1
- package/dist/application/eventsManager.js +27 -27
- package/dist/application/eventsManager.js.map +1 -1
- package/dist/application/operations/pull.d.ts.map +1 -1
- package/dist/application/operations/pull.js +7 -5
- package/dist/application/operations/pull.js.map +1 -1
- package/dist/client/index.d.ts +2 -0
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +1 -0
- package/dist/client/index.js.map +1 -1
- package/dist/client/manager.d.ts +2 -0
- package/dist/client/manager.d.ts.map +1 -1
- package/dist/client/manager.js.map +1 -1
- package/dist/client/sandboxedWorker.d.ts +90 -0
- package/dist/client/sandboxedWorker.d.ts.map +1 -0
- package/dist/client/sandboxedWorker.js +299 -0
- package/dist/client/sandboxedWorker.js.map +1 -0
- package/dist/domain/queue/priorityQueue.d.ts +7 -2
- package/dist/domain/queue/priorityQueue.d.ts.map +1 -1
- package/dist/domain/queue/priorityQueue.js +29 -16
- package/dist/domain/queue/priorityQueue.js.map +1 -1
- package/dist/domain/queue/shard.d.ts +2 -2
- package/dist/domain/queue/shard.d.ts.map +1 -1
- package/dist/domain/queue/shard.js +14 -15
- package/dist/domain/queue/shard.js.map +1 -1
- package/dist/infrastructure/persistence/sqlite.d.ts +3 -0
- package/dist/infrastructure/persistence/sqlite.d.ts.map +1 -1
- package/dist/infrastructure/persistence/sqlite.js +24 -9
- package/dist/infrastructure/persistence/sqlite.js.map +1 -1
- package/dist/infrastructure/server/tcp.d.ts.map +1 -1
- package/dist/infrastructure/server/tcp.js +20 -7
- package/dist/infrastructure/server/tcp.js.map +1 -1
- package/dist/shared/hash.d.ts +2 -1
- package/dist/shared/hash.d.ts.map +1 -1
- package/dist/shared/hash.js +3 -18
- package/dist/shared/hash.js.map +1 -1
- package/dist/shared/minHeap.d.ts +11 -2
- package/dist/shared/minHeap.d.ts.map +1 -1
- package/dist/shared/minHeap.js +29 -12
- package/dist/shared/minHeap.js.map +1 -1
- package/package.json +10 -7
|
@@ -10,11 +10,12 @@ export type EventSubscriber = (event: JobEvent) => void;
|
|
|
10
10
|
/** Events manager class */
|
|
11
11
|
export declare class EventsManager {
|
|
12
12
|
private readonly webhookManager;
|
|
13
|
+
/** Use Set for O(1) subscribe/unsubscribe instead of indexOf+splice */
|
|
13
14
|
private readonly subscribers;
|
|
14
15
|
/** Waiters for specific job completions - for efficient WaitJob implementation */
|
|
15
16
|
private readonly completionWaiters;
|
|
16
17
|
constructor(webhookManager: WebhookManager);
|
|
17
|
-
/** Subscribe to job events */
|
|
18
|
+
/** Subscribe to job events - O(1) add and remove */
|
|
18
19
|
subscribe(callback: EventSubscriber): () => void;
|
|
19
20
|
/** Clear all subscribers (for shutdown) */
|
|
20
21
|
clear(): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eventsManager.d.ts","sourceRoot":"","sources":["../../src/application/eventsManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,KAAK,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAIvD,gCAAgC;AAChC,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"eventsManager.d.ts","sourceRoot":"","sources":["../../src/application/eventsManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,KAAK,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAIvD,gCAAgC;AAChC,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;AAQxD,2BAA2B;AAC3B,qBAAa,aAAa;IAMZ,OAAO,CAAC,QAAQ,CAAC,cAAc;IAL3C,uEAAuE;IACvE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8B;IAC1D,kFAAkF;IAClF,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAyC;gBAE9C,cAAc,EAAE,cAAc;IAE3D,oDAAoD;IACpD,SAAS,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,IAAI;IAOhD,2CAA2C;IAC3C,KAAK,IAAI,IAAI;IAab;;;OAGG;IACH,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA4BvE,qEAAqE;IACrE,cAAc,IAAI,OAAO;IAQzB,oFAAoF;IACpF,SAAS,CACP,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG;QACzB,SAAS,EAAE,SAAS,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,KAAK,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GACA,IAAI;IA0DP,+CAA+C;IAC/C,OAAO,CAAC,iBAAiB;CAgB1B"}
|
|
@@ -6,28 +6,29 @@ import { webhookLog } from '../shared/logger';
|
|
|
6
6
|
/** Events manager class */
|
|
7
7
|
export class EventsManager {
|
|
8
8
|
webhookManager;
|
|
9
|
-
|
|
9
|
+
/** Use Set for O(1) subscribe/unsubscribe instead of indexOf+splice */
|
|
10
|
+
subscribers = new Set();
|
|
10
11
|
/** Waiters for specific job completions - for efficient WaitJob implementation */
|
|
11
12
|
completionWaiters = new Map();
|
|
12
13
|
constructor(webhookManager) {
|
|
13
14
|
this.webhookManager = webhookManager;
|
|
14
15
|
}
|
|
15
|
-
/** Subscribe to job events */
|
|
16
|
+
/** Subscribe to job events - O(1) add and remove */
|
|
16
17
|
subscribe(callback) {
|
|
17
|
-
this.subscribers.
|
|
18
|
+
this.subscribers.add(callback);
|
|
18
19
|
return () => {
|
|
19
|
-
|
|
20
|
-
if (idx !== -1)
|
|
21
|
-
this.subscribers.splice(idx, 1);
|
|
20
|
+
this.subscribers.delete(callback); // O(1) instead of O(n)
|
|
22
21
|
};
|
|
23
22
|
}
|
|
24
23
|
/** Clear all subscribers (for shutdown) */
|
|
25
24
|
clear() {
|
|
26
|
-
this.subscribers.
|
|
25
|
+
this.subscribers.clear();
|
|
27
26
|
// Clear all waiters
|
|
28
27
|
for (const waiters of this.completionWaiters.values()) {
|
|
29
|
-
for (const
|
|
30
|
-
|
|
28
|
+
for (const waiter of waiters) {
|
|
29
|
+
if (!waiter.cancelled) {
|
|
30
|
+
waiter.resolve();
|
|
31
|
+
}
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
34
|
this.completionWaiters.clear();
|
|
@@ -39,40 +40,36 @@ export class EventsManager {
|
|
|
39
40
|
waitForJobCompletion(jobId, timeoutMs) {
|
|
40
41
|
const jobKey = String(jobId);
|
|
41
42
|
return new Promise((resolve) => {
|
|
43
|
+
const waiter = {
|
|
44
|
+
resolve: () => {
|
|
45
|
+
clearTimeout(timer);
|
|
46
|
+
resolve(true);
|
|
47
|
+
},
|
|
48
|
+
cancelled: false,
|
|
49
|
+
};
|
|
42
50
|
const timer = setTimeout(() => {
|
|
43
|
-
// Timeout -
|
|
44
|
-
|
|
45
|
-
if (waiters) {
|
|
46
|
-
const idx = waiters.indexOf(resolveWaiter);
|
|
47
|
-
if (idx !== -1)
|
|
48
|
-
waiters.splice(idx, 1);
|
|
49
|
-
if (waiters.length === 0)
|
|
50
|
-
this.completionWaiters.delete(jobKey);
|
|
51
|
-
}
|
|
51
|
+
// Timeout - mark as cancelled instead of O(n) splice
|
|
52
|
+
waiter.cancelled = true;
|
|
52
53
|
resolve(false);
|
|
53
54
|
}, timeoutMs);
|
|
54
|
-
const resolveWaiter = () => {
|
|
55
|
-
clearTimeout(timer);
|
|
56
|
-
resolve(true);
|
|
57
|
-
};
|
|
58
55
|
// Add to waiters
|
|
59
56
|
let waiters = this.completionWaiters.get(jobKey);
|
|
60
57
|
if (!waiters) {
|
|
61
58
|
waiters = [];
|
|
62
59
|
this.completionWaiters.set(jobKey, waiters);
|
|
63
60
|
}
|
|
64
|
-
waiters.push(
|
|
61
|
+
waiters.push(waiter);
|
|
65
62
|
});
|
|
66
63
|
}
|
|
67
64
|
/** Check if broadcast has any listeners - for batch optimizations */
|
|
68
65
|
needsBroadcast() {
|
|
69
|
-
return (this.subscribers.
|
|
66
|
+
return (this.subscribers.size > 0 ||
|
|
70
67
|
this.webhookManager.hasEnabledWebhooks() ||
|
|
71
68
|
this.completionWaiters.size > 0);
|
|
72
69
|
}
|
|
73
70
|
/** Broadcast event to all subscribers - optimized to skip work when no listeners */
|
|
74
71
|
broadcast(event) {
|
|
75
|
-
const hasSubscribers = this.subscribers.
|
|
72
|
+
const hasSubscribers = this.subscribers.size > 0;
|
|
76
73
|
const hasWebhooks = this.webhookManager.hasEnabledWebhooks();
|
|
77
74
|
const isCompletion = event.eventType === "completed" /* EventType.Completed */;
|
|
78
75
|
const hasWaiters = isCompletion && this.completionWaiters.size > 0;
|
|
@@ -97,8 +94,11 @@ export class EventsManager {
|
|
|
97
94
|
const waiters = this.completionWaiters.get(jobKey);
|
|
98
95
|
if (waiters) {
|
|
99
96
|
this.completionWaiters.delete(jobKey);
|
|
100
|
-
|
|
101
|
-
|
|
97
|
+
// Only notify non-cancelled waiters
|
|
98
|
+
for (const waiter of waiters) {
|
|
99
|
+
if (!waiter.cancelled) {
|
|
100
|
+
waiter.resolve();
|
|
101
|
+
}
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
104
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eventsManager.js","sourceRoot":"","sources":["../../src/application/eventsManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"eventsManager.js","sourceRoot":"","sources":["../../src/application/eventsManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAW9C,2BAA2B;AAC3B,MAAM,OAAO,aAAa;IAMK;IAL7B,uEAAuE;IACtD,WAAW,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC1D,kFAAkF;IACjE,iBAAiB,GAAG,IAAI,GAAG,EAA8B,CAAC;IAE3E,YAA6B,cAA8B;QAA9B,mBAAc,GAAd,cAAc,CAAgB;IAAG,CAAC;IAE/D,oDAAoD;IACpD,SAAS,CAAC,QAAyB;QACjC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC/B,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,uBAAuB;QAC5D,CAAC,CAAC;IACJ,CAAC;IAED,2CAA2C;IAC3C,KAAK;QACH,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,oBAAoB;QACpB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YACtD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;oBACtB,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,oBAAoB,CAAC,KAAY,EAAE,SAAiB;QAClD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAE7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,MAAM,GAAqB;gBAC/B,OAAO,EAAE,GAAG,EAAE;oBACZ,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,OAAO,CAAC,IAAI,CAAC,CAAC;gBAChB,CAAC;gBACD,SAAS,EAAE,KAAK;aACjB,CAAC;YAEF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,qDAAqD;gBACrD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;gBACxB,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,EAAE,SAAS,CAAC,CAAC;YAEd,iBAAiB;YACjB,IAAI,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC9C,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,qEAAqE;IACrE,cAAc;QACZ,OAAO,CACL,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC;YACzB,IAAI,CAAC,cAAc,CAAC,kBAAkB,EAAE;YACxC,IAAI,CAAC,iBAAiB,CAAC,IAAI,GAAG,CAAC,CAChC,CAAC;IACJ,CAAC;IAED,oFAAoF;IACpF,SAAS,CACP,KAMC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,kBAAkB,EAAE,CAAC;QAC7D,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS,0CAAwB,CAAC;QAC7D,MAAM,UAAU,GAAG,YAAY,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,GAAG,CAAC,CAAC;QAEnE,+BAA+B;QAC/B,IAAI,CAAC,cAAc,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU,EAAE,CAAC;YACnD,OAAO;QACT,CAAC;QAED,qBAAqB;QACrB,IAAI,cAAc,EAAE,CAAC;YACnB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACnC,IAAI,CAAC;oBACH,GAAG,CAAC,KAAiB,CAAC,CAAC;gBACzB,CAAC;gBAAC,MAAM,CAAC;oBACP,2BAA2B;gBAC7B,CAAC;YACH,CAAC;QACH,CAAC;QAED,sDAAsD;QACtD,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACtC,oCAAoC;gBACpC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;wBACtB,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,wDAAwD;QACxD,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC7D,IAAI,YAAY,EAAE,CAAC;gBACjB,IAAI,CAAC,cAAc;qBAChB,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,KAAK,EAAE;oBACvD,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;iBACnB,CAAC;qBACD,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;oBACtB,UAAU,CAAC,KAAK,CAAC,wBAAwB,EAAE;wBACzC,KAAK,EAAE,YAAY;wBACnB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;wBAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;qBACnB,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC;YACP,CAAC;QACH,CAAC;IACH,CAAC;IAED,+CAA+C;IACvC,iBAAiB,CAAC,SAAoB;QAC5C,QAAQ,SAAS,EAAE,CAAC;YAClB;gBACE,OAAO,YAAY,CAAC;YACtB;gBACE,OAAO,aAAa,CAAC;YACvB;gBACE,OAAO,eAAe,CAAC;YACzB;gBACE,OAAO,YAAY,CAAC;YACtB,yCAAwB;YACxB;gBACE,2CAA2C;gBAC3C,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pull.d.ts","sourceRoot":"","sources":["../../../src/application/operations/pull.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,KAAK,EAAsB,MAAM,wBAAwB,CAAC;AAClF,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,yCAAyC,CAAC;AAC7E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAIhD,6BAA6B;AAC7B,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,gBAAgB,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;IACpC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAClC,WAAW,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/B,SAAS,EAAE,CAAC,KAAK,EAAE;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,KAAK,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,KAAK,IAAI,CAAC;CACZ;AAED;;GAEG;AACH,wBAAsB,OAAO,CAC3B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,WAAW,GACf,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAwCrB;AAkED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,WAAW,GACf,OAAO,CAAC,GAAG,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"pull.d.ts","sourceRoot":"","sources":["../../../src/application/operations/pull.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,KAAK,EAAsB,MAAM,wBAAwB,CAAC;AAClF,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,yCAAyC,CAAC;AAC7E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAIhD,6BAA6B;AAC7B,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,gBAAgB,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;IACpC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAClC,WAAW,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/B,SAAS,EAAE,CAAC,KAAK,EAAE;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,KAAK,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,KAAK,IAAI,CAAC;CACZ;AAED;;GAEG;AACH,wBAAsB,OAAO,CAC3B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,WAAW,GACf,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAwCrB;AAkED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,WAAW,GACf,OAAO,CAAC,GAAG,EAAE,CAAC,CA8DhB"}
|
|
@@ -119,15 +119,17 @@ export async function pullJobBatch(queue, count, timeoutMs, ctx) {
|
|
|
119
119
|
}
|
|
120
120
|
shardJobs.push(job);
|
|
121
121
|
}
|
|
122
|
-
// Add to processing shards in parallel
|
|
122
|
+
// Add to processing shards in parallel - avoid Array.from allocation
|
|
123
123
|
const now = Date.now();
|
|
124
|
-
|
|
125
|
-
|
|
124
|
+
const lockPromises = [];
|
|
125
|
+
for (const [procIdx, shardJobs] of byProcShard) {
|
|
126
|
+
lockPromises.push(withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
126
127
|
for (const job of shardJobs) {
|
|
127
128
|
ctx.processingShards[procIdx].set(job.id, job);
|
|
128
129
|
}
|
|
129
|
-
});
|
|
130
|
-
}
|
|
130
|
+
}));
|
|
131
|
+
}
|
|
132
|
+
await Promise.all(lockPromises);
|
|
131
133
|
// Update indexes and metrics
|
|
132
134
|
for (const job of jobs) {
|
|
133
135
|
const procIdx = processingShardIndex(job.id);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pull.js","sourceRoot":"","sources":["../../../src/application/operations/pull.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAwB,SAAS,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAKlF,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAmBrE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAa,EACb,SAAiB,EACjB,GAAgB;IAEhB,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAE9B,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAEpD,IAAI,GAAG,EAAE,CAAC;YACR,qBAAqB;YACrB,MAAM,OAAO,GAAG,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7C,MAAM,aAAa,CAAC,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE;gBACrD,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACjD,CAAC,CAAC,CAAC;YACH,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;YAEpE,uBAAuB;YACvB,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YAC9C,GAAG,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAE3C,iBAAiB;YACjB,GAAG,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YACxB,GAAG,CAAC,SAAS,CAAC;gBACZ,SAAS,EAAE,QAAqB;gBAChC,KAAK;gBACL,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC,CAAC;YAEH,OAAO,GAAG,CAAC;QACb,CAAC;QAED,kCAAkC;QAClC,IAAI,QAAQ,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,8DAA8D;QAC9D,MAAM,SAAS,GAAG,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,MAAM,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,KAAa,EAAE,GAAW,EAAE,GAAgB;IAC1E,OAAO,MAAM,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE;QACnD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAEpC,kBAAkB;QAClB,IAAI,KAAK,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAE9B,mBAAmB;QACnB,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEnD,oBAAoB;QACpB,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAErD,MAAM,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,iBAAiB;QACjB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG;gBAAE,MAAM;YAEhB,oBAAoB;YACpB,IAAI,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;gBACxB,CAAC,CAAC,GAAG,EAAE,CAAC;gBACR,yCAAyC;gBACzC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC9B,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC5B,SAAS;YACX,CAAC;YAED,iBAAiB;YACjB,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC;gBAAE,MAAM;YAE9B,mBAAmB;YACnB,IAAI,GAAG,CAAC,OAAO,IAAI,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3D,mBAAmB;gBACnB,MAAM;YACR,CAAC;YAED,UAAU;YACV,CAAC,CAAC,GAAG,EAAE,CAAC;YACR,yCAAyC;YACzC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAE9B,oBAAoB;YACpB,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBAChB,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YAED,aAAa;YACb,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC;YACpB,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC;YAExB,OAAO,GAAG,CAAC;QACb,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAa,EACb,KAAa,EACb,SAAiB,EACjB,GAAgB;IAEhB,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAE9B,OAAO,IAAI,EAAE,CAAC;QACZ,6CAA6C;QAC7C,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,yEAAyE;YACzE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAiB,CAAC;YAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,OAAO,GAAG,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7C,IAAI,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACzC,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,SAAS,GAAG,EAAE,CAAC;oBACf,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;gBACtC,CAAC;gBACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC;YAED,
|
|
1
|
+
{"version":3,"file":"pull.js","sourceRoot":"","sources":["../../../src/application/operations/pull.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAwB,SAAS,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAKlF,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAmBrE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAa,EACb,SAAiB,EACjB,GAAgB;IAEhB,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAE9B,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAEpD,IAAI,GAAG,EAAE,CAAC;YACR,qBAAqB;YACrB,MAAM,OAAO,GAAG,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7C,MAAM,aAAa,CAAC,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE;gBACrD,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACjD,CAAC,CAAC,CAAC;YACH,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;YAEpE,uBAAuB;YACvB,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YAC9C,GAAG,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAE3C,iBAAiB;YACjB,GAAG,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YACxB,GAAG,CAAC,SAAS,CAAC;gBACZ,SAAS,EAAE,QAAqB;gBAChC,KAAK;gBACL,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC,CAAC;YAEH,OAAO,GAAG,CAAC;QACb,CAAC;QAED,kCAAkC;QAClC,IAAI,QAAQ,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,8DAA8D;QAC9D,MAAM,SAAS,GAAG,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,MAAM,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,KAAa,EAAE,GAAW,EAAE,GAAgB;IAC1E,OAAO,MAAM,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE;QACnD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAEpC,kBAAkB;QAClB,IAAI,KAAK,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAE9B,mBAAmB;QACnB,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEnD,oBAAoB;QACpB,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAErD,MAAM,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,iBAAiB;QACjB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG;gBAAE,MAAM;YAEhB,oBAAoB;YACpB,IAAI,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;gBACxB,CAAC,CAAC,GAAG,EAAE,CAAC;gBACR,yCAAyC;gBACzC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC9B,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC5B,SAAS;YACX,CAAC;YAED,iBAAiB;YACjB,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC;gBAAE,MAAM;YAE9B,mBAAmB;YACnB,IAAI,GAAG,CAAC,OAAO,IAAI,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3D,mBAAmB;gBACnB,MAAM;YACR,CAAC;YAED,UAAU;YACV,CAAC,CAAC,GAAG,EAAE,CAAC;YACR,yCAAyC;YACzC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAE9B,oBAAoB;YACpB,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBAChB,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YAED,aAAa;YACb,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC;YACpB,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC;YAExB,OAAO,GAAG,CAAC;QACb,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAa,EACb,KAAa,EACb,SAAiB,EACjB,GAAgB;IAEhB,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAE9B,OAAO,IAAI,EAAE,CAAC;QACZ,6CAA6C;QAC7C,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,yEAAyE;YACzE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAiB,CAAC;YAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,OAAO,GAAG,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7C,IAAI,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACzC,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,SAAS,GAAG,EAAE,CAAC;oBACf,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;gBACtC,CAAC;gBACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC;YAED,qEAAqE;YACrE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,YAAY,GAAoB,EAAE,CAAC;YACzC,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,WAAW,EAAE,CAAC;gBAC/C,YAAY,CAAC,IAAI,CACf,aAAa,CAAC,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE;oBAC/C,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;wBAC5B,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;oBACjD,CAAC;gBACH,CAAC,CAAC,CACH,CAAC;YACJ,CAAC;YACD,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YAEhC,6BAA6B;YAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,OAAO,GAAG,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7C,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;gBACpE,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC;gBACvC,GAAG,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;gBAC3C,GAAG,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;gBACxB,GAAG,CAAC,SAAS,CAAC;oBACZ,SAAS,EAAE,QAAqB;oBAChC,KAAK;oBACL,KAAK,EAAE,GAAG,CAAC,EAAE;oBACb,SAAS,EAAE,GAAG;iBACf,CAAC,CAAC;YACL,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAED,mCAAmC;QACnC,IAAI,QAAQ,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,mCAAmC;QACnC,MAAM,SAAS,GAAG,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,MAAM,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,qBAAqB,CAClC,KAAa,EACb,GAAW,EACX,KAAa,EACb,GAAgB;IAEhB,OAAO,MAAM,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE;QACnD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,IAAI,GAAU,EAAE,CAAC;QAEvB,kBAAkB;QAClB,IAAI,KAAK,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAE9B,MAAM,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,wBAAwB;QACxB,OAAO,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;YAC3B,gCAAgC;YAChC,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,KAAK,CAAC;gBAAE,MAAM;YAE7C,iCAAiC;YACjC,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,KAAK,CAAC;gBAAE,MAAM;YAE/C,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG;gBAAE,MAAM;YAEhB,oBAAoB;YACpB,IAAI,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;gBACxB,CAAC,CAAC,GAAG,EAAE,CAAC;gBACR,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC9B,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC5B,SAAS;YACX,CAAC;YAED,iBAAiB;YACjB,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC;gBAAE,MAAM;YAE9B,mBAAmB;YACnB,IAAI,GAAG,CAAC,OAAO,IAAI,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3D,MAAM;YACR,CAAC;YAED,UAAU;YACV,CAAC,CAAC,GAAG,EAAE,CAAC;YACR,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAE9B,oBAAoB;YACpB,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBAChB,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YAED,aAAa;YACb,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC;YACpB,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC;YAExB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/client/index.d.ts
CHANGED
|
@@ -20,10 +20,12 @@
|
|
|
20
20
|
*/
|
|
21
21
|
export { Queue } from './queue';
|
|
22
22
|
export { Worker } from './worker';
|
|
23
|
+
export { SandboxedWorker } from './sandboxedWorker';
|
|
23
24
|
export { QueueEvents } from './events';
|
|
24
25
|
export { QueueGroup } from './queueGroup';
|
|
25
26
|
export { FlowProducer } from './flow';
|
|
26
27
|
export { shutdownManager } from './manager';
|
|
27
28
|
export type { Job, JobOptions, QueueOptions, WorkerOptions, Processor, StallConfig, DlqConfig, DlqEntry, DlqStats, DlqFilter, FailureReason, } from './types';
|
|
28
29
|
export type { FlowStep, FlowResult } from './flow';
|
|
30
|
+
export type { SandboxedWorkerOptions } from './sandboxedWorker';
|
|
29
31
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,YAAY,EACV,GAAG,EACH,UAAU,EACV,YAAY,EACZ,aAAa,EACb,SAAS,EACT,WAAW,EACX,SAAS,EACT,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,aAAa,GACd,MAAM,SAAS,CAAC;AACjB,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,YAAY,EACV,GAAG,EACH,UAAU,EACV,YAAY,EACZ,aAAa,EACb,SAAS,EACT,WAAW,EACX,SAAS,EACT,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,aAAa,GACd,MAAM,SAAS,CAAC;AACjB,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACnD,YAAY,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC"}
|
package/dist/client/index.js
CHANGED
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
*/
|
|
21
21
|
export { Queue } from './queue';
|
|
22
22
|
export { Worker } from './worker';
|
|
23
|
+
export { SandboxedWorker } from './sandboxedWorker';
|
|
23
24
|
export { QueueEvents } from './events';
|
|
24
25
|
export { QueueGroup } from './queueGroup';
|
|
25
26
|
export { FlowProducer } from './flow';
|
package/dist/client/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC"}
|
package/dist/client/manager.d.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* Shared QueueManager singleton
|
|
3
3
|
*/
|
|
4
4
|
import { QueueManager } from '../application/queueManager';
|
|
5
|
+
/** Shared manager type export */
|
|
6
|
+
export type SharedManager = QueueManager;
|
|
5
7
|
/** Get shared QueueManager instance */
|
|
6
8
|
export declare function getSharedManager(): QueueManager;
|
|
7
9
|
/** Shutdown shared manager */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manager.d.ts","sourceRoot":"","sources":["../../src/client/manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"manager.d.ts","sourceRoot":"","sources":["../../src/client/manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,iCAAiC;AACjC,MAAM,MAAM,aAAa,GAAG,YAAY,CAAC;AAIzC,uCAAuC;AACvC,wBAAgB,gBAAgB,IAAI,YAAY,CAG/C;AAED,8BAA8B;AAC9B,wBAAgB,eAAe,IAAI,IAAI,CAKtC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../src/client/manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../src/client/manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAK3D,IAAI,QAAQ,GAAwB,IAAI,CAAC;AAEzC,uCAAuC;AACvC,MAAM,UAAU,gBAAgB;IAC9B,QAAQ,KAAK,IAAI,YAAY,EAAE,CAAC;IAChC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8BAA8B;AAC9B,MAAM,UAAU,eAAe;IAC7B,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACpB,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sandboxed Worker
|
|
3
|
+
* Runs job processors in isolated Bun Worker processes
|
|
4
|
+
*/
|
|
5
|
+
import { type SharedManager } from './manager';
|
|
6
|
+
/** Sandboxed worker configuration */
|
|
7
|
+
export interface SandboxedWorkerOptions {
|
|
8
|
+
/** Path to processor file (must export default async function) */
|
|
9
|
+
processor: string;
|
|
10
|
+
/** Number of worker processes (default: 1) */
|
|
11
|
+
concurrency?: number;
|
|
12
|
+
/** Max memory per worker in MB - uses smol mode if <= 64 (default: 256) */
|
|
13
|
+
maxMemory?: number;
|
|
14
|
+
/** Job timeout in ms (default: 30000) */
|
|
15
|
+
timeout?: number;
|
|
16
|
+
/** Auto-restart crashed workers (default: true) */
|
|
17
|
+
autoRestart?: boolean;
|
|
18
|
+
/** Max restarts before giving up (default: 10) */
|
|
19
|
+
maxRestarts?: number;
|
|
20
|
+
/** Poll interval when no workers are idle (default: 10ms) */
|
|
21
|
+
pollInterval?: number;
|
|
22
|
+
/** Custom QueueManager (for testing, defaults to shared manager) */
|
|
23
|
+
manager?: SharedManager;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Sandboxed Worker - runs processors in isolated Bun Worker processes
|
|
27
|
+
*
|
|
28
|
+
* @example
|
|
29
|
+
* ```typescript
|
|
30
|
+
* import { Queue, SandboxedWorker } from 'bunqueue/client';
|
|
31
|
+
*
|
|
32
|
+
* const queue = new Queue('cpu-intensive');
|
|
33
|
+
*
|
|
34
|
+
* // Create processor file: processor.ts
|
|
35
|
+
* // export default async (job) => {
|
|
36
|
+
* // const result = heavyComputation(job.data);
|
|
37
|
+
* // return result;
|
|
38
|
+
* // };
|
|
39
|
+
*
|
|
40
|
+
* const worker = new SandboxedWorker('cpu-intensive', {
|
|
41
|
+
* processor: './processor.ts',
|
|
42
|
+
* concurrency: 4,
|
|
43
|
+
* timeout: 60000,
|
|
44
|
+
* });
|
|
45
|
+
*
|
|
46
|
+
* await worker.start();
|
|
47
|
+
* // Workers process jobs in isolated processes
|
|
48
|
+
* // If one crashes, others continue working
|
|
49
|
+
* ```
|
|
50
|
+
*/
|
|
51
|
+
export declare class SandboxedWorker {
|
|
52
|
+
private readonly queueName;
|
|
53
|
+
private readonly options;
|
|
54
|
+
private readonly workers;
|
|
55
|
+
private running;
|
|
56
|
+
private pullPromise;
|
|
57
|
+
private wrapperPath;
|
|
58
|
+
private readonly manager;
|
|
59
|
+
constructor(queueName: string, options: SandboxedWorkerOptions);
|
|
60
|
+
/** Start the sandboxed worker pool */
|
|
61
|
+
start(): void;
|
|
62
|
+
/** Stop all workers gracefully */
|
|
63
|
+
stop(): Promise<void>;
|
|
64
|
+
/** Get worker pool stats */
|
|
65
|
+
getStats(): {
|
|
66
|
+
total: number;
|
|
67
|
+
busy: number;
|
|
68
|
+
idle: number;
|
|
69
|
+
restarts: number;
|
|
70
|
+
};
|
|
71
|
+
/** Create wrapper script file that loads the processor */
|
|
72
|
+
private createWrapperScript;
|
|
73
|
+
/** Spawn a single worker process */
|
|
74
|
+
private spawnWorker;
|
|
75
|
+
/** Main loop - pull jobs and dispatch to workers */
|
|
76
|
+
private pullLoop;
|
|
77
|
+
/** Dispatch job to a worker process */
|
|
78
|
+
private dispatchJob;
|
|
79
|
+
/** Handle message from worker */
|
|
80
|
+
private handleWorkerMessage;
|
|
81
|
+
/** Complete a job successfully */
|
|
82
|
+
private completeJob;
|
|
83
|
+
/** Fail a job */
|
|
84
|
+
private failJob;
|
|
85
|
+
/** Handle job timeout */
|
|
86
|
+
private handleJobTimeout;
|
|
87
|
+
/** Handle worker crash and potentially restart */
|
|
88
|
+
private handleWorkerCrash;
|
|
89
|
+
}
|
|
90
|
+
//# sourceMappingURL=sandboxedWorker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sandboxedWorker.d.ts","sourceRoot":"","sources":["../../src/client/sandboxedWorker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAoB,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAMjE,qCAAqC;AACrC,MAAM,WAAW,sBAAsB;IACrC,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6DAA6D;IAC7D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AA+BD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoD;IAC5E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;IAC/C,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgB;gBAE5B,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,sBAAsB;IAc9D,sCAAsC;IACtC,KAAK,IAAI,IAAI;IAgBb,kCAAkC;IAC5B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA+B3B,4BAA4B;IAC5B,QAAQ,IAAI;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;IAW3E,0DAA0D;IAC1D,OAAO,CAAC,mBAAmB;IA+C3B,oCAAoC;IACpC,OAAO,CAAC,WAAW;IAiCnB,oDAAoD;YACtC,QAAQ;IAkBtB,uCAAuC;IACvC,OAAO,CAAC,WAAW;IAsBnB,iCAAiC;IACjC,OAAO,CAAC,mBAAmB;IAkB3B,kCAAkC;IAClC,OAAO,CAAC,WAAW;IAiBnB,iBAAiB;IACjB,OAAO,CAAC,OAAO;IAiBf,yBAAyB;IACzB,OAAO,CAAC,gBAAgB;IAgBxB,kDAAkD;IAClD,OAAO,CAAC,iBAAiB;CAoB1B"}
|