@zintrust/workers 0.1.29 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -1
- package/dist/AnomalyDetection.d.ts +4 -0
- package/dist/AnomalyDetection.js +8 -0
- package/dist/BroadcastWorker.d.ts +2 -0
- package/dist/CanaryController.js +49 -5
- package/dist/ChaosEngineering.js +13 -0
- package/dist/ClusterLock.js +21 -10
- package/dist/DeadLetterQueue.js +12 -8
- package/dist/MultiQueueWorker.d.ts +1 -1
- package/dist/MultiQueueWorker.js +12 -7
- package/dist/NotificationWorker.d.ts +2 -0
- package/dist/PriorityQueue.d.ts +2 -2
- package/dist/PriorityQueue.js +20 -21
- package/dist/ResourceMonitor.js +65 -38
- package/dist/WorkerFactory.d.ts +23 -3
- package/dist/WorkerFactory.js +420 -40
- package/dist/WorkerInit.js +8 -3
- package/dist/WorkerMetrics.d.ts +2 -1
- package/dist/WorkerMetrics.js +152 -93
- package/dist/WorkerRegistry.d.ts +6 -0
- package/dist/WorkerRegistry.js +70 -1
- package/dist/WorkerShutdown.d.ts +21 -0
- package/dist/WorkerShutdown.js +82 -9
- package/dist/WorkerShutdownDurableObject.d.ts +12 -0
- package/dist/WorkerShutdownDurableObject.js +41 -0
- package/dist/build-manifest.json +171 -99
- package/dist/createQueueWorker.d.ts +2 -0
- package/dist/createQueueWorker.js +42 -27
- package/dist/dashboard/types.d.ts +5 -0
- package/dist/dashboard/workers-api.js +136 -43
- package/dist/http/WorkerApiController.js +1 -0
- package/dist/http/WorkerController.js +133 -85
- package/dist/http/WorkerMonitoringService.d.ts +11 -0
- package/dist/http/WorkerMonitoringService.js +62 -0
- package/dist/http/middleware/CustomValidation.js +1 -1
- package/dist/http/middleware/EditWorkerValidation.d.ts +1 -1
- package/dist/http/middleware/EditWorkerValidation.js +7 -6
- package/dist/http/middleware/ProcessorPathSanitizer.js +101 -35
- package/dist/http/middleware/WorkerValidationChain.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -0
- package/dist/routes/workers.js +48 -6
- package/dist/storage/WorkerStore.d.ts +4 -1
- package/dist/storage/WorkerStore.js +55 -7
- package/dist/telemetry/api/TelemetryAPI.d.ts +46 -0
- package/dist/telemetry/api/TelemetryAPI.js +219 -0
- package/dist/telemetry/api/TelemetryMonitoringService.d.ts +17 -0
- package/dist/telemetry/api/TelemetryMonitoringService.js +113 -0
- package/dist/telemetry/components/AlertPanel.d.ts +1 -0
- package/dist/telemetry/components/AlertPanel.js +13 -0
- package/dist/telemetry/components/CostTracking.d.ts +1 -0
- package/dist/telemetry/components/CostTracking.js +14 -0
- package/dist/telemetry/components/ResourceUsageChart.d.ts +1 -0
- package/dist/telemetry/components/ResourceUsageChart.js +11 -0
- package/dist/telemetry/components/WorkerHealthChart.d.ts +1 -0
- package/dist/telemetry/components/WorkerHealthChart.js +11 -0
- package/dist/telemetry/index.d.ts +15 -0
- package/dist/telemetry/index.js +60 -0
- package/dist/telemetry/routes/dashboard.d.ts +6 -0
- package/dist/telemetry/routes/dashboard.js +608 -0
- package/dist/ui/router/EmbeddedAssets.d.ts +4 -0
- package/dist/ui/router/EmbeddedAssets.js +13 -0
- package/dist/ui/router/ui.js +100 -4
- package/package.json +10 -6
- package/src/AnomalyDetection.ts +9 -0
- package/src/CanaryController.ts +41 -5
- package/src/ChaosEngineering.ts +14 -0
- package/src/ClusterLock.ts +22 -9
- package/src/DeadLetterQueue.ts +13 -8
- package/src/MultiQueueWorker.ts +15 -8
- package/src/PriorityQueue.ts +21 -22
- package/src/ResourceMonitor.ts +72 -40
- package/src/WorkerFactory.ts +545 -49
- package/src/WorkerInit.ts +8 -3
- package/src/WorkerMetrics.ts +183 -105
- package/src/WorkerRegistry.ts +80 -1
- package/src/WorkerShutdown.ts +115 -9
- package/src/WorkerShutdownDurableObject.ts +64 -0
- package/src/createQueueWorker.ts +73 -30
- package/src/dashboard/types.ts +5 -0
- package/src/dashboard/workers-api.ts +165 -52
- package/src/http/WorkerApiController.ts +1 -0
- package/src/http/WorkerController.ts +167 -90
- package/src/http/WorkerMonitoringService.ts +77 -0
- package/src/http/middleware/CustomValidation.ts +1 -1
- package/src/http/middleware/EditWorkerValidation.ts +7 -6
- package/src/http/middleware/ProcessorPathSanitizer.ts +123 -36
- package/src/http/middleware/WorkerValidationChain.ts +1 -0
- package/src/index.ts +6 -1
- package/src/routes/workers.ts +66 -9
- package/src/storage/WorkerStore.ts +59 -9
- package/src/telemetry/api/TelemetryAPI.ts +292 -0
- package/src/telemetry/api/TelemetryMonitoringService.ts +149 -0
- package/src/telemetry/components/AlertPanel.ts +13 -0
- package/src/telemetry/components/CostTracking.ts +14 -0
- package/src/telemetry/components/ResourceUsageChart.ts +11 -0
- package/src/telemetry/components/WorkerHealthChart.ts +11 -0
- package/src/telemetry/index.ts +121 -0
- package/src/telemetry/public/assets/zintrust-logo.svg +15 -0
- package/src/telemetry/routes/dashboard.ts +638 -0
- package/src/telemetry/styles/tailwind.css +1 -0
- package/src/telemetry/styles/zintrust-theme.css +8 -0
- package/src/ui/router/EmbeddedAssets.ts +13 -0
- package/src/ui/router/ui.ts +112 -5
- package/src/ui/workers/index.html +2 -2
- package/src/ui/workers/main.js +232 -61
- package/src/ui/workers/zintrust.svg +30 -0
- package/dist/dashboard/workers-dashboard-ui.d.ts +0 -3
- package/dist/dashboard/workers-dashboard-ui.js +0 -1026
- package/dist/dashboard/workers-dashboard.d.ts +0 -4
- package/dist/dashboard/workers-dashboard.js +0 -904
package/dist/ui/router/ui.js
CHANGED
|
@@ -1,8 +1,59 @@
|
|
|
1
|
-
import { Logger, MIME_TYPES, NodeSingletons, Router } from '@zintrust/core';
|
|
1
|
+
import { Cloudflare, Logger, MIME_TYPES, NodeSingletons, Router, detectRuntime, } from '@zintrust/core';
|
|
2
|
+
import { INDEX_HTML, MAIN_JS, STYLES_CSS, ZINTRUST_SVG } from './EmbeddedAssets';
|
|
3
|
+
const isCloudflare = detectRuntime().isCloudflare;
|
|
4
|
+
const safeFileUrlToPath = (url) => {
|
|
5
|
+
if (typeof url !== 'string' || url.trim() === '')
|
|
6
|
+
return '';
|
|
7
|
+
try {
|
|
8
|
+
return NodeSingletons.url.fileURLToPath(url);
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return '';
|
|
12
|
+
}
|
|
13
|
+
};
|
|
14
|
+
const safeCwd = () => {
|
|
15
|
+
try {
|
|
16
|
+
const cwd = NodeSingletons.process?.cwd?.();
|
|
17
|
+
if (typeof cwd === 'string' && cwd.trim() !== '')
|
|
18
|
+
return cwd;
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
// ignore
|
|
22
|
+
}
|
|
23
|
+
return '';
|
|
24
|
+
};
|
|
25
|
+
const getAssetsBinding = () => Cloudflare.getAssetsBinding();
|
|
26
|
+
const fetchAssetText = async (assetPath) => {
|
|
27
|
+
const assets = getAssetsBinding();
|
|
28
|
+
if (!assets)
|
|
29
|
+
return '';
|
|
30
|
+
const url = new URL(assetPath, 'http://assets');
|
|
31
|
+
const response = await assets.fetch(url);
|
|
32
|
+
if (!response.ok)
|
|
33
|
+
return '';
|
|
34
|
+
return response.text();
|
|
35
|
+
};
|
|
36
|
+
const fetchAssetBytes = async (assetPath) => {
|
|
37
|
+
const assets = getAssetsBinding();
|
|
38
|
+
if (!assets)
|
|
39
|
+
return null;
|
|
40
|
+
const url = new URL(assetPath, 'http://assets');
|
|
41
|
+
const response = await assets.fetch(url);
|
|
42
|
+
if (!response.ok)
|
|
43
|
+
return null;
|
|
44
|
+
const buffer = await response.arrayBuffer();
|
|
45
|
+
return new Uint8Array(buffer);
|
|
46
|
+
};
|
|
2
47
|
export const uiResolver = async (uiBasePath) => {
|
|
3
48
|
// Resolve base path for UI assets
|
|
4
49
|
// const __filename = NodeSingletons.url.fileURLToPath(import.meta.url);
|
|
5
50
|
// const __dirname = NodeSingletons.path.dirname(__filename);
|
|
51
|
+
const assetHtml = await fetchAssetText('/workers/index.html');
|
|
52
|
+
if (assetHtml !== '')
|
|
53
|
+
return assetHtml;
|
|
54
|
+
if (isCloudflare) {
|
|
55
|
+
return Buffer.from(INDEX_HTML, 'base64').toString('utf-8');
|
|
56
|
+
}
|
|
6
57
|
const uiPath = NodeSingletons.path.resolve(uiBasePath, 'workers/index.html');
|
|
7
58
|
const html = await NodeSingletons.fs.readFile(uiPath, 'utf8');
|
|
8
59
|
return html;
|
|
@@ -30,14 +81,58 @@ const getUiBase = () => {
|
|
|
30
81
|
// Resolve base path for UI assets
|
|
31
82
|
if (uiBasePath.length > 0)
|
|
32
83
|
return uiBasePath;
|
|
33
|
-
const __filename =
|
|
34
|
-
|
|
35
|
-
|
|
84
|
+
const __filename = safeFileUrlToPath(import.meta.url);
|
|
85
|
+
if (__filename !== '') {
|
|
86
|
+
const __dirname = NodeSingletons.path.dirname(__filename);
|
|
87
|
+
uiBasePath = NodeSingletons.path.resolve(__dirname, '../');
|
|
88
|
+
return uiBasePath;
|
|
89
|
+
}
|
|
90
|
+
const cwd = safeCwd();
|
|
91
|
+
if (cwd !== '') {
|
|
92
|
+
uiBasePath = NodeSingletons.path.resolve(cwd, 'packages', 'workers', 'src', 'ui');
|
|
93
|
+
return uiBasePath;
|
|
94
|
+
}
|
|
95
|
+
uiBasePath = '';
|
|
36
96
|
return uiBasePath;
|
|
37
97
|
};
|
|
38
98
|
const serveStaticFile = async (req, res) => {
|
|
39
99
|
try {
|
|
40
100
|
const filePath = req.getPath();
|
|
101
|
+
const assetBytes = await fetchAssetBytes(filePath);
|
|
102
|
+
if (assetBytes) {
|
|
103
|
+
const mimeType = getMimeType(filePath);
|
|
104
|
+
res.setHeader('Content-Type', mimeType);
|
|
105
|
+
res.setHeader('Cache-Control', 'public, max-age=3600'); // 1 hour cache
|
|
106
|
+
res.send(Buffer.from(assetBytes));
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
if (isCloudflare) {
|
|
110
|
+
const normalizedPath = filePath.replace(/^\//, '');
|
|
111
|
+
if (normalizedPath === 'workers/styles.css') {
|
|
112
|
+
const mimeType = MIME_TYPES.CSS;
|
|
113
|
+
res.setHeader('Content-Type', mimeType);
|
|
114
|
+
res.setHeader('Cache-Control', 'public, max-age=3600');
|
|
115
|
+
res.send(Buffer.from(STYLES_CSS, 'base64'));
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
if (normalizedPath === 'workers/main.js') {
|
|
119
|
+
const mimeType = MIME_TYPES.JS;
|
|
120
|
+
res.setHeader('Content-Type', mimeType);
|
|
121
|
+
res.setHeader('Cache-Control', 'public, max-age=3600');
|
|
122
|
+
res.send(Buffer.from(MAIN_JS, 'base64'));
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
if (normalizedPath === 'workers/zintrust.svg') {
|
|
126
|
+
const mimeType = MIME_TYPES.SVG;
|
|
127
|
+
res.setHeader('Content-Type', mimeType);
|
|
128
|
+
res.setHeader('Cache-Control', 'public, max-age=3600');
|
|
129
|
+
res.send(Buffer.from(ZINTRUST_SVG, 'base64'));
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
res.setStatus(404);
|
|
133
|
+
res.send(Buffer.from('Not Found'));
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
41
136
|
const fullPath = NodeSingletons.path.resolve(getUiBase(), filePath.replace(/^\//, ''));
|
|
42
137
|
// Security check - prevent directory traversal
|
|
43
138
|
if (!fullPath.startsWith(uiBasePath)) {
|
|
@@ -74,6 +169,7 @@ export const registerStaticAssets = (router, middleware) => {
|
|
|
74
169
|
// Serve workers CSS and JS files
|
|
75
170
|
Router.get(r, '/styles.css', serveStaticFile);
|
|
76
171
|
Router.get(r, '/main.js', serveStaticFile);
|
|
172
|
+
Router.get(r, '/zintrust.svg', serveStaticFile);
|
|
77
173
|
Router.get(r, '/:filename', serveStaticFile);
|
|
78
174
|
Router.get(r, '/integration/:filename', serveStaticFile);
|
|
79
175
|
// Serve components CSS files
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zintrust/workers",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.30",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -22,22 +22,26 @@
|
|
|
22
22
|
"types": "./dist/ResourceMonitor.d.ts",
|
|
23
23
|
"default": "./dist/ResourceMonitor.js"
|
|
24
24
|
},
|
|
25
|
-
"./
|
|
26
|
-
"types": "./dist/
|
|
27
|
-
"default": "./dist/
|
|
25
|
+
"./telemetry": {
|
|
26
|
+
"types": "./telemetry/dist/index.d.ts",
|
|
27
|
+
"default": "./telemetry/dist/index.js"
|
|
28
|
+
},
|
|
29
|
+
"./telemetry/*": {
|
|
30
|
+
"types": "./telemetry/dist/*.d.ts",
|
|
31
|
+
"default": "./telemetry/dist/*.js"
|
|
28
32
|
}
|
|
29
33
|
},
|
|
30
34
|
"engines": {
|
|
31
35
|
"node": ">=20.0.0"
|
|
32
36
|
},
|
|
33
37
|
"peerDependencies": {
|
|
34
|
-
"@zintrust/core": "
|
|
38
|
+
"@zintrust/core": "file:../../dist"
|
|
35
39
|
},
|
|
36
40
|
"publishConfig": {
|
|
37
41
|
"access": "public"
|
|
38
42
|
},
|
|
39
43
|
"scripts": {
|
|
40
|
-
"build": "
|
|
44
|
+
"build": "tsc -p tsconfig.json",
|
|
41
45
|
"prepublishOnly": "npm run build"
|
|
42
46
|
},
|
|
43
47
|
"dependencies": {
|
package/src/AnomalyDetection.ts
CHANGED
|
@@ -194,6 +194,15 @@ export const AnomalyDetection = Object.freeze({
|
|
|
194
194
|
Logger.info(`Anomaly detection configured for ${config.workerName}`);
|
|
195
195
|
},
|
|
196
196
|
|
|
197
|
+
/**
|
|
198
|
+
* Cleanup anomaly models for a worker
|
|
199
|
+
*/
|
|
200
|
+
cleanup(workerName: string): void {
|
|
201
|
+
configs.delete(workerName);
|
|
202
|
+
models.delete(workerName);
|
|
203
|
+
Logger.debug(`Anomaly detection cleanup completed for ${workerName}`);
|
|
204
|
+
},
|
|
205
|
+
|
|
197
206
|
/**
|
|
198
207
|
* Train baseline model
|
|
199
208
|
*/
|
package/src/CanaryController.ts
CHANGED
|
@@ -174,7 +174,16 @@ const incrementTraffic = (workerName: string): void => {
|
|
|
174
174
|
|
|
175
175
|
// eslint-disable-next-line no-restricted-syntax
|
|
176
176
|
const timer = setTimeout(() => {
|
|
177
|
-
|
|
177
|
+
try {
|
|
178
|
+
CanaryController.complete(workerName);
|
|
179
|
+
} catch (error) {
|
|
180
|
+
Logger.error('Error during canary completion callback', error);
|
|
181
|
+
} finally {
|
|
182
|
+
const current = canaryTimers.get(`${workerName}:complete`);
|
|
183
|
+
if (current === timer) {
|
|
184
|
+
canaryTimers.delete(`${workerName}:complete`);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
178
187
|
}, config.monitoringDuration * 1000);
|
|
179
188
|
|
|
180
189
|
canaryTimers.set(`${workerName}:complete`, timer);
|
|
@@ -188,7 +197,16 @@ const incrementTraffic = (workerName: string): void => {
|
|
|
188
197
|
|
|
189
198
|
// eslint-disable-next-line no-restricted-syntax
|
|
190
199
|
const timer = setTimeout(() => {
|
|
191
|
-
|
|
200
|
+
try {
|
|
201
|
+
incrementTraffic(workerName);
|
|
202
|
+
} catch (error) {
|
|
203
|
+
Logger.error('Error during canary increment callback', error);
|
|
204
|
+
} finally {
|
|
205
|
+
const current = canaryTimers.get(workerName);
|
|
206
|
+
if (current === timer) {
|
|
207
|
+
canaryTimers.delete(workerName);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
192
210
|
}, config.incrementInterval * 1000);
|
|
193
211
|
|
|
194
212
|
canaryTimers.set(workerName, timer);
|
|
@@ -201,7 +219,7 @@ const appendHistory = (
|
|
|
201
219
|
): void => {
|
|
202
220
|
deployment.history.push(entry);
|
|
203
221
|
if (deployment.history.length > MAX_HISTORY) {
|
|
204
|
-
deployment.history.
|
|
222
|
+
deployment.history.splice(0, deployment.history.length - MAX_HISTORY);
|
|
205
223
|
}
|
|
206
224
|
};
|
|
207
225
|
|
|
@@ -269,7 +287,16 @@ export const CanaryController = Object.freeze({
|
|
|
269
287
|
|
|
270
288
|
// eslint-disable-next-line no-restricted-syntax
|
|
271
289
|
const timer = setTimeout(() => {
|
|
272
|
-
|
|
290
|
+
try {
|
|
291
|
+
incrementTraffic(workerName);
|
|
292
|
+
} catch (error) {
|
|
293
|
+
Logger.error('Error during canary start callback', error);
|
|
294
|
+
} finally {
|
|
295
|
+
const current = canaryTimers.get(workerName);
|
|
296
|
+
if (current === timer) {
|
|
297
|
+
canaryTimers.delete(workerName);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
273
300
|
}, config.monitoringDuration * 1000);
|
|
274
301
|
|
|
275
302
|
canaryTimers.set(workerName, timer);
|
|
@@ -314,7 +341,16 @@ export const CanaryController = Object.freeze({
|
|
|
314
341
|
|
|
315
342
|
// eslint-disable-next-line no-restricted-syntax
|
|
316
343
|
const timer = setTimeout(() => {
|
|
317
|
-
|
|
344
|
+
try {
|
|
345
|
+
incrementTraffic(workerName);
|
|
346
|
+
} catch (error) {
|
|
347
|
+
Logger.error('Error during canary resume callback', error);
|
|
348
|
+
} finally {
|
|
349
|
+
const current = canaryTimers.get(workerName);
|
|
350
|
+
if (current === timer) {
|
|
351
|
+
canaryTimers.delete(workerName);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
318
354
|
}, deployment.config.incrementInterval * 1000);
|
|
319
355
|
|
|
320
356
|
canaryTimers.set(workerName, timer);
|
package/src/ChaosEngineering.ts
CHANGED
|
@@ -61,6 +61,18 @@ type ExperimentRecord = {
|
|
|
61
61
|
};
|
|
62
62
|
|
|
63
63
|
const experiments = new Map<string, ExperimentRecord>();
|
|
64
|
+
const EXPERIMENT_RETENTION_MS = 24 * 60 * 60 * 1000;
|
|
65
|
+
|
|
66
|
+
const cleanupExpiredExperiments = (): void => {
|
|
67
|
+
const cutoff = Date.now() - EXPERIMENT_RETENTION_MS;
|
|
68
|
+
for (const [id, record] of experiments.entries()) {
|
|
69
|
+
if (record.status.state !== 'completed') continue;
|
|
70
|
+
const endedAt = record.status.endedAt?.getTime() ?? 0;
|
|
71
|
+
if (endedAt > 0 && endedAt < cutoff) {
|
|
72
|
+
experiments.delete(id);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
};
|
|
64
76
|
|
|
65
77
|
const getTargetWorkers = (config: IChaosExperiment): string[] => {
|
|
66
78
|
const candidates = config.target.workers ?? WorkerRegistry.listRunning();
|
|
@@ -224,6 +236,8 @@ export const ChaosEngineering = Object.freeze({
|
|
|
224
236
|
id: experimentId,
|
|
225
237
|
duration: record.config.duration,
|
|
226
238
|
});
|
|
239
|
+
|
|
240
|
+
cleanupExpiredExperiments();
|
|
227
241
|
},
|
|
228
242
|
|
|
229
243
|
/**
|
package/src/ClusterLock.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import {
|
|
8
|
+
Cloudflare,
|
|
8
9
|
ErrorFactory,
|
|
9
10
|
Logger,
|
|
10
11
|
createRedisConnection,
|
|
@@ -39,8 +40,20 @@ export type AuditLogEntry = {
|
|
|
39
40
|
success: boolean;
|
|
40
41
|
};
|
|
41
42
|
|
|
42
|
-
|
|
43
|
-
|
|
43
|
+
let INSTANCE_ID = '';
|
|
44
|
+
|
|
45
|
+
const createInstanceId = (): string => {
|
|
46
|
+
const workers = Cloudflare.getWorkersEnv() !== null;
|
|
47
|
+
const pid = typeof process !== 'undefined' && typeof process.pid === 'number' ? process.pid : 0;
|
|
48
|
+
const prefix = workers ? 'worker-cf' : 'worker';
|
|
49
|
+
return `${prefix}-${pid}-${Date.now()}-${generateUuid()}`;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const getInstanceId = (): string => {
|
|
53
|
+
if (INSTANCE_ID !== '') return INSTANCE_ID;
|
|
54
|
+
INSTANCE_ID = createInstanceId();
|
|
55
|
+
return INSTANCE_ID;
|
|
56
|
+
};
|
|
44
57
|
|
|
45
58
|
// Redis key prefixes
|
|
46
59
|
const LOCK_PREFIX = 'worker:lock:';
|
|
@@ -94,7 +107,7 @@ const extendLockTTL = async (client: IORedis, lockKey: string, ttl: number): Pro
|
|
|
94
107
|
const redisKey = getLockKey(lockKey);
|
|
95
108
|
const value = await client.get(redisKey);
|
|
96
109
|
|
|
97
|
-
if (value === null || value !==
|
|
110
|
+
if (value === null || value !== getInstanceId()) {
|
|
98
111
|
return false; // Lock not held by this instance
|
|
99
112
|
}
|
|
100
113
|
|
|
@@ -132,7 +145,7 @@ const startHeartbeat = (client: IORedis): void => {
|
|
|
132
145
|
timestamp: now,
|
|
133
146
|
operation: 'extend',
|
|
134
147
|
lockKey,
|
|
135
|
-
instanceId:
|
|
148
|
+
instanceId: getInstanceId(),
|
|
136
149
|
success: true,
|
|
137
150
|
});
|
|
138
151
|
} else {
|
|
@@ -178,7 +191,7 @@ export const ClusterLock = Object.freeze({
|
|
|
178
191
|
redisClient = createRedisConnection(config);
|
|
179
192
|
startHeartbeat(redisClient);
|
|
180
193
|
|
|
181
|
-
Logger.info('ClusterLock initialized', { instanceId:
|
|
194
|
+
Logger.info('ClusterLock initialized', { instanceId: getInstanceId() });
|
|
182
195
|
},
|
|
183
196
|
|
|
184
197
|
/**
|
|
@@ -197,14 +210,14 @@ export const ClusterLock = Object.freeze({
|
|
|
197
210
|
|
|
198
211
|
try {
|
|
199
212
|
// Try to acquire lock using SET NX EX (set if not exists with expiry)
|
|
200
|
-
const result = await redisClient.set(redisKey,
|
|
213
|
+
const result = await redisClient.set(redisKey, getInstanceId(), 'EX', ttl, 'NX');
|
|
201
214
|
|
|
202
215
|
const success = result === 'OK';
|
|
203
216
|
|
|
204
217
|
if (success) {
|
|
205
218
|
const lockInfo: LockInfo = {
|
|
206
219
|
lockKey,
|
|
207
|
-
instanceId:
|
|
220
|
+
instanceId: getInstanceId(),
|
|
208
221
|
acquiredAt: now,
|
|
209
222
|
expiresAt: new Date(now.getTime() + ttl * 1000),
|
|
210
223
|
region,
|
|
@@ -224,7 +237,7 @@ export const ClusterLock = Object.freeze({
|
|
|
224
237
|
timestamp: now,
|
|
225
238
|
operation: 'acquire',
|
|
226
239
|
lockKey,
|
|
227
|
-
instanceId:
|
|
240
|
+
instanceId: getInstanceId(),
|
|
228
241
|
userId,
|
|
229
242
|
success: true,
|
|
230
243
|
});
|
|
@@ -235,7 +248,7 @@ export const ClusterLock = Object.freeze({
|
|
|
235
248
|
timestamp: now,
|
|
236
249
|
operation: 'acquire',
|
|
237
250
|
lockKey,
|
|
238
|
-
instanceId:
|
|
251
|
+
instanceId: getInstanceId(),
|
|
239
252
|
userId,
|
|
240
253
|
success: false,
|
|
241
254
|
});
|
package/src/DeadLetterQueue.ts
CHANGED
|
@@ -70,9 +70,14 @@ export type DLQStats = {
|
|
|
70
70
|
retentionViolations: number;
|
|
71
71
|
};
|
|
72
72
|
|
|
73
|
-
// Redis key prefixes
|
|
74
|
-
const
|
|
75
|
-
|
|
73
|
+
// Redis key prefixes - using workers package prefix system
|
|
74
|
+
const getDLQPrefix = (): string => {
|
|
75
|
+
return 'worker:dlq:';
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const getAuditPrefix = (): string => {
|
|
79
|
+
return 'worker:dlq:audit:';
|
|
80
|
+
};
|
|
76
81
|
|
|
77
82
|
// Internal state
|
|
78
83
|
let redisClient: IORedis | null = null;
|
|
@@ -83,14 +88,14 @@ let cleanupInterval: NodeJS.Timeout | null = null;
|
|
|
83
88
|
* Helper: Get DLQ key
|
|
84
89
|
*/
|
|
85
90
|
const getDLQKey = (queueName: string): string => {
|
|
86
|
-
return `${
|
|
91
|
+
return `${getDLQPrefix()}${queueName}`;
|
|
87
92
|
};
|
|
88
93
|
|
|
89
94
|
/**
|
|
90
95
|
* Helper: Get audit key
|
|
91
96
|
*/
|
|
92
97
|
const getAuditKey = (failedJobId: string): string => {
|
|
93
|
-
return `${
|
|
98
|
+
return `${getAuditPrefix()}${failedJobId}`;
|
|
94
99
|
};
|
|
95
100
|
|
|
96
101
|
/**
|
|
@@ -184,7 +189,7 @@ const cleanupOldEntries = async (): Promise<number> => {
|
|
|
184
189
|
try {
|
|
185
190
|
const cutoffTimestamp = Date.now() - policy.autoDeleteAfterDays * 24 * 60 * 60 * 1000;
|
|
186
191
|
// Find all DLQ keys
|
|
187
|
-
const pattern = `${
|
|
192
|
+
const pattern = `${getDLQPrefix()}*`;
|
|
188
193
|
const keys = await client.keys(pattern);
|
|
189
194
|
|
|
190
195
|
const cleanedCounts = await Promise.all(
|
|
@@ -587,12 +592,12 @@ export const DeadLetterQueue = Object.freeze({
|
|
|
587
592
|
|
|
588
593
|
try {
|
|
589
594
|
const client = redisClient;
|
|
590
|
-
const pattern = `${
|
|
595
|
+
const pattern = `${getDLQPrefix()}*`;
|
|
591
596
|
const keys = await client.keys(pattern);
|
|
592
597
|
|
|
593
598
|
const entriesByQueue = await Promise.all(
|
|
594
599
|
keys.map(async (key) => {
|
|
595
|
-
const queueName = key.replace(
|
|
600
|
+
const queueName = key.replace(getDLQPrefix(), '');
|
|
596
601
|
const entries = await client.zrange(key, 0, -1);
|
|
597
602
|
return {
|
|
598
603
|
queueName,
|
package/src/MultiQueueWorker.ts
CHANGED
|
@@ -50,12 +50,12 @@ const multiQueueWorkers = new Map<
|
|
|
50
50
|
/**
|
|
51
51
|
* Helper: Create worker for a queue
|
|
52
52
|
*/
|
|
53
|
-
const createQueueWorker = (
|
|
53
|
+
const createQueueWorker = async (
|
|
54
54
|
workerName: string,
|
|
55
55
|
queueConfig: QueueConfig,
|
|
56
56
|
processor: MultiQueueWorkerConfig['processor']
|
|
57
|
-
): Worker => {
|
|
58
|
-
const queue = PriorityQueue.getQueueInstance(queueConfig.name);
|
|
57
|
+
): Promise<Worker> => {
|
|
58
|
+
const queue = await PriorityQueue.getQueueInstance(queueConfig.name);
|
|
59
59
|
const connection = queue.opts.connection;
|
|
60
60
|
|
|
61
61
|
const workerOptions: WorkerOptions = {
|
|
@@ -138,7 +138,7 @@ export const MultiQueueWorker = Object.freeze({
|
|
|
138
138
|
/**
|
|
139
139
|
* Create multi-queue worker
|
|
140
140
|
*/
|
|
141
|
-
create(config: MultiQueueWorkerConfig): void {
|
|
141
|
+
async create(config: MultiQueueWorkerConfig): Promise<void> {
|
|
142
142
|
if (multiQueueWorkers.has(config.workerName)) {
|
|
143
143
|
throw ErrorFactory.createWorkerError(
|
|
144
144
|
`Multi-queue worker "${config.workerName}" already exists`
|
|
@@ -151,9 +151,16 @@ export const MultiQueueWorker = Object.freeze({
|
|
|
151
151
|
// Sort queues by priority (higher first)
|
|
152
152
|
const sortedQueues = [...config.queues].sort((a, b) => b.priority - a.priority);
|
|
153
153
|
|
|
154
|
-
// Create workers for each queue
|
|
155
|
-
|
|
156
|
-
const worker = createQueueWorker(config.workerName, queueConfig, config.processor);
|
|
154
|
+
// Create workers for each queue in parallel
|
|
155
|
+
const workerPromises = sortedQueues.map(async (queueConfig) => {
|
|
156
|
+
const worker = await createQueueWorker(config.workerName, queueConfig, config.processor);
|
|
157
|
+
return { queueConfig, worker };
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
const workerResults = await Promise.all(workerPromises);
|
|
161
|
+
|
|
162
|
+
// Store workers and stats
|
|
163
|
+
for (const { queueConfig, worker } of workerResults) {
|
|
157
164
|
workers.set(queueConfig.name, worker);
|
|
158
165
|
stats.set(queueConfig.name, initializeQueueStats(queueConfig.name, queueConfig.enabled));
|
|
159
166
|
}
|
|
@@ -322,7 +329,7 @@ export const MultiQueueWorker = Object.freeze({
|
|
|
322
329
|
// Update worker concurrency (requires restart in BullMQ)
|
|
323
330
|
await worker.close();
|
|
324
331
|
|
|
325
|
-
const newWorker = createQueueWorker(workerName, queueConfig, mqw.config.processor);
|
|
332
|
+
const newWorker = await createQueueWorker(workerName, queueConfig, mqw.config.processor);
|
|
326
333
|
mqw.workers.set(queueName, newWorker);
|
|
327
334
|
|
|
328
335
|
Logger.info(`Queue concurrency updated: ${queueName}`, { workerName, concurrency });
|
package/src/PriorityQueue.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Sealed namespace for immutability
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import { ErrorFactory, Logger,
|
|
7
|
+
import { ErrorFactory, Logger, type RedisConfig } from '@zintrust/core';
|
|
8
8
|
import type { Queue } from 'bullmq';
|
|
9
9
|
|
|
10
10
|
export type PriorityLevel = 'critical' | 'high' | 'normal' | 'low';
|
|
@@ -55,15 +55,14 @@ const PRIORITY_VALUES: Record<PriorityLevel, number> = {
|
|
|
55
55
|
|
|
56
56
|
type QueueRedisModule = typeof import('@zintrust/queue-redis');
|
|
57
57
|
|
|
58
|
-
const require = NodeSingletons.module.createRequire(import.meta.url);
|
|
59
58
|
let queueRedisModule: QueueRedisModule | undefined;
|
|
60
59
|
let hasWarnedMissingQueueRedis = false;
|
|
61
60
|
|
|
62
|
-
const loadQueueRedisModule = (): QueueRedisModule | undefined => {
|
|
61
|
+
const loadQueueRedisModule = async (): Promise<QueueRedisModule | undefined> => {
|
|
63
62
|
if (queueRedisModule) return queueRedisModule;
|
|
64
63
|
|
|
65
64
|
try {
|
|
66
|
-
queueRedisModule =
|
|
65
|
+
queueRedisModule = (await import('@zintrust/queue-redis')) as QueueRedisModule;
|
|
67
66
|
return queueRedisModule;
|
|
68
67
|
} catch (error) {
|
|
69
68
|
if (!hasWarnedMissingQueueRedis) {
|
|
@@ -80,8 +79,8 @@ const loadQueueRedisModule = (): QueueRedisModule | undefined => {
|
|
|
80
79
|
/**
|
|
81
80
|
* Helper: Get or create queue via shared driver
|
|
82
81
|
*/
|
|
83
|
-
const getQueue = (queueName: string): Queue => {
|
|
84
|
-
const queueRedis = loadQueueRedisModule();
|
|
82
|
+
const getQueue = async (queueName: string): Promise<Queue> => {
|
|
83
|
+
const queueRedis = await loadQueueRedisModule();
|
|
85
84
|
if (!queueRedis) {
|
|
86
85
|
throw ErrorFactory.createWorkerError(
|
|
87
86
|
'Optional package "@zintrust/queue-redis" is required for PriorityQueue. Install it to use queue features.'
|
|
@@ -170,7 +169,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
170
169
|
data: T,
|
|
171
170
|
options: PriorityJobOptions
|
|
172
171
|
): Promise<string> {
|
|
173
|
-
const queue = getQueue(queueName);
|
|
172
|
+
const queue = await getQueue(queueName);
|
|
174
173
|
const jobOptions = buildJobOptions(options);
|
|
175
174
|
|
|
176
175
|
try {
|
|
@@ -206,7 +205,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
206
205
|
options: PriorityJobOptions;
|
|
207
206
|
}>
|
|
208
207
|
): Promise<string[]> {
|
|
209
|
-
const queue = getQueue(queueName);
|
|
208
|
+
const queue = await getQueue(queueName);
|
|
210
209
|
|
|
211
210
|
try {
|
|
212
211
|
const bulkJobs = jobs.map((job) => ({
|
|
@@ -237,7 +236,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
237
236
|
* Get job by ID
|
|
238
237
|
*/
|
|
239
238
|
async getJob(queueName: string, jobId: string) {
|
|
240
|
-
const queue = getQueue(queueName);
|
|
239
|
+
const queue = await getQueue(queueName);
|
|
241
240
|
return queue.getJob(jobId);
|
|
242
241
|
},
|
|
243
242
|
|
|
@@ -245,7 +244,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
245
244
|
* Remove a job
|
|
246
245
|
*/
|
|
247
246
|
async removeJob(queueName: string, jobId: string): Promise<void> {
|
|
248
|
-
const queue = getQueue(queueName);
|
|
247
|
+
const queue = await getQueue(queueName);
|
|
249
248
|
const job = await queue.getJob(jobId);
|
|
250
249
|
|
|
251
250
|
if (job) {
|
|
@@ -258,7 +257,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
258
257
|
* Pause a queue
|
|
259
258
|
*/
|
|
260
259
|
async pause(queueName: string): Promise<void> {
|
|
261
|
-
const queue = getQueue(queueName);
|
|
260
|
+
const queue = await getQueue(queueName);
|
|
262
261
|
await queue.pause();
|
|
263
262
|
Logger.info(`Paused queue "${queueName}"`);
|
|
264
263
|
},
|
|
@@ -267,7 +266,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
267
266
|
* Resume a queue
|
|
268
267
|
*/
|
|
269
268
|
async resume(queueName: string): Promise<void> {
|
|
270
|
-
const queue = getQueue(queueName);
|
|
269
|
+
const queue = await getQueue(queueName);
|
|
271
270
|
await queue.resume();
|
|
272
271
|
Logger.info(`Resumed queue "${queueName}"`);
|
|
273
272
|
},
|
|
@@ -276,7 +275,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
276
275
|
* Get queue information
|
|
277
276
|
*/
|
|
278
277
|
async getQueueInfo(queueName: string): Promise<QueueInfo> {
|
|
279
|
-
const queue = getQueue(queueName);
|
|
278
|
+
const queue = await getQueue(queueName);
|
|
280
279
|
const isPaused = await queue.isPaused();
|
|
281
280
|
const jobCounts = await queue.getJobCounts();
|
|
282
281
|
|
|
@@ -297,8 +296,8 @@ export const PriorityQueue = Object.freeze({
|
|
|
297
296
|
/**
|
|
298
297
|
* Get all queue names
|
|
299
298
|
*/
|
|
300
|
-
getQueueNames(): string[] {
|
|
301
|
-
const queueRedis = loadQueueRedisModule();
|
|
299
|
+
async getQueueNames(): Promise<string[]> {
|
|
300
|
+
const queueRedis = await loadQueueRedisModule();
|
|
302
301
|
if (!queueRedis) return [];
|
|
303
302
|
return queueRedis.BullMQRedisQueue.getQueueNames();
|
|
304
303
|
},
|
|
@@ -307,7 +306,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
307
306
|
* Drain queue (remove all jobs)
|
|
308
307
|
*/
|
|
309
308
|
async drain(queueName: string, delayed = false): Promise<void> {
|
|
310
|
-
const queue = getQueue(queueName);
|
|
309
|
+
const queue = await getQueue(queueName);
|
|
311
310
|
await queue.drain(delayed);
|
|
312
311
|
Logger.info(`Drained queue "${queueName}"`, { delayed });
|
|
313
312
|
},
|
|
@@ -321,7 +320,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
321
320
|
limit: number,
|
|
322
321
|
type: 'completed' | 'failed' | 'delayed' | 'wait' | 'active' | 'paused' = 'completed'
|
|
323
322
|
): Promise<string[]> {
|
|
324
|
-
const queue = getQueue(queueName);
|
|
323
|
+
const queue = await getQueue(queueName);
|
|
325
324
|
const jobs = await queue.clean(grace, limit, type);
|
|
326
325
|
|
|
327
326
|
Logger.info(`Cleaned ${jobs.length} ${type} jobs from queue "${queueName}"`);
|
|
@@ -333,9 +332,9 @@ export const PriorityQueue = Object.freeze({
|
|
|
333
332
|
* Obliterate queue (remove all data including queue itself)
|
|
334
333
|
*/
|
|
335
334
|
async obliterate(queueName: string, force = false): Promise<void> {
|
|
336
|
-
const queue = getQueue(queueName);
|
|
335
|
+
const queue = await getQueue(queueName);
|
|
337
336
|
await queue.obliterate({ force });
|
|
338
|
-
const queueRedis = loadQueueRedisModule();
|
|
337
|
+
const queueRedis = await loadQueueRedisModule();
|
|
339
338
|
if (queueRedis) {
|
|
340
339
|
await queueRedis.BullMQRedisQueue.closeQueue(queueName);
|
|
341
340
|
}
|
|
@@ -360,7 +359,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
360
359
|
/**
|
|
361
360
|
* Get queue instance (internal use)
|
|
362
361
|
*/
|
|
363
|
-
getQueueInstance(queueName: string): Queue {
|
|
362
|
+
getQueueInstance(queueName: string): Promise<Queue> {
|
|
364
363
|
return getQueue(queueName);
|
|
365
364
|
},
|
|
366
365
|
|
|
@@ -368,7 +367,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
368
367
|
* Close a queue
|
|
369
368
|
*/
|
|
370
369
|
async closeQueue(queueName: string): Promise<void> {
|
|
371
|
-
const queueRedis = loadQueueRedisModule();
|
|
370
|
+
const queueRedis = await loadQueueRedisModule();
|
|
372
371
|
if (!queueRedis) return;
|
|
373
372
|
await queueRedis.BullMQRedisQueue.closeQueue(queueName);
|
|
374
373
|
Logger.info(`Closed queue "${queueName}"`);
|
|
@@ -379,7 +378,7 @@ export const PriorityQueue = Object.freeze({
|
|
|
379
378
|
*/
|
|
380
379
|
async shutdown(): Promise<void> {
|
|
381
380
|
Logger.info('PriorityQueue shutting down via BullMQRedisQueue...');
|
|
382
|
-
const queueRedis = loadQueueRedisModule();
|
|
381
|
+
const queueRedis = await loadQueueRedisModule();
|
|
383
382
|
if (!queueRedis) return;
|
|
384
383
|
await queueRedis.BullMQRedisQueue.shutdown();
|
|
385
384
|
Logger.info('PriorityQueue shutdown complete');
|