@zintrust/workers 0.1.27 → 0.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/PriorityQueue.js +40 -7
- package/dist/WorkerFactory.d.ts +2 -2
- package/dist/WorkerFactory.js +39 -15
- package/dist/WorkerMetrics.js +8 -10
- package/dist/WorkerShutdown.js +5 -9
- package/dist/build-manifest.json +17 -19
- package/package.json +6 -4
- package/src/PriorityQueue.ts +45 -7
- package/src/WorkerFactory.ts +43 -17
- package/src/WorkerMetrics.ts +8 -12
- package/src/WorkerShutdown.ts +5 -8
package/dist/PriorityQueue.js
CHANGED
|
@@ -3,8 +3,7 @@
|
|
|
3
3
|
* BullMQ priority levels with datacenter affinity
|
|
4
4
|
* Sealed namespace for immutability
|
|
5
5
|
*/
|
|
6
|
-
import { ErrorFactory, Logger } from '@zintrust/core';
|
|
7
|
-
import { BullMQRedisQueue } from '@zintrust/queue-redis';
|
|
6
|
+
import { ErrorFactory, Logger, NodeSingletons } from '@zintrust/core';
|
|
8
7
|
// Priority mappings
|
|
9
8
|
const PRIORITY_VALUES = {
|
|
10
9
|
critical: 10,
|
|
@@ -12,11 +11,33 @@ const PRIORITY_VALUES = {
|
|
|
12
11
|
normal: 1,
|
|
13
12
|
low: 0,
|
|
14
13
|
};
|
|
14
|
+
const require = NodeSingletons.module.createRequire(import.meta.url);
|
|
15
|
+
let queueRedisModule;
|
|
16
|
+
let hasWarnedMissingQueueRedis = false;
|
|
17
|
+
const loadQueueRedisModule = () => {
|
|
18
|
+
if (queueRedisModule)
|
|
19
|
+
return queueRedisModule;
|
|
20
|
+
try {
|
|
21
|
+
queueRedisModule = require('@zintrust/queue-redis');
|
|
22
|
+
return queueRedisModule;
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
if (!hasWarnedMissingQueueRedis) {
|
|
26
|
+
hasWarnedMissingQueueRedis = true;
|
|
27
|
+
Logger.warn('Optional package "@zintrust/queue-redis" is not installed. PriorityQueue features are disabled until it is installed.', error);
|
|
28
|
+
}
|
|
29
|
+
return undefined;
|
|
30
|
+
}
|
|
31
|
+
};
|
|
15
32
|
/**
|
|
16
33
|
* Helper: Get or create queue via shared driver
|
|
17
34
|
*/
|
|
18
35
|
const getQueue = (queueName) => {
|
|
19
|
-
|
|
36
|
+
const queueRedis = loadQueueRedisModule();
|
|
37
|
+
if (!queueRedis) {
|
|
38
|
+
throw ErrorFactory.createWorkerError('Optional package "@zintrust/queue-redis" is required for PriorityQueue. Install it to use queue features.');
|
|
39
|
+
}
|
|
40
|
+
return queueRedis.BullMQRedisQueue.getQueue(queueName);
|
|
20
41
|
};
|
|
21
42
|
/**
|
|
22
43
|
* Helper: Build job options with priority
|
|
@@ -179,7 +200,10 @@ export const PriorityQueue = Object.freeze({
|
|
|
179
200
|
* Get all queue names
|
|
180
201
|
*/
|
|
181
202
|
getQueueNames() {
|
|
182
|
-
|
|
203
|
+
const queueRedis = loadQueueRedisModule();
|
|
204
|
+
if (!queueRedis)
|
|
205
|
+
return [];
|
|
206
|
+
return queueRedis.BullMQRedisQueue.getQueueNames();
|
|
183
207
|
},
|
|
184
208
|
/**
|
|
185
209
|
* Drain queue (remove all jobs)
|
|
@@ -204,7 +228,10 @@ export const PriorityQueue = Object.freeze({
|
|
|
204
228
|
async obliterate(queueName, force = false) {
|
|
205
229
|
const queue = getQueue(queueName);
|
|
206
230
|
await queue.obliterate({ force });
|
|
207
|
-
|
|
231
|
+
const queueRedis = loadQueueRedisModule();
|
|
232
|
+
if (queueRedis) {
|
|
233
|
+
await queueRedis.BullMQRedisQueue.closeQueue(queueName);
|
|
234
|
+
}
|
|
208
235
|
Logger.warn(`Obliterated queue "${queueName}"`);
|
|
209
236
|
},
|
|
210
237
|
/**
|
|
@@ -229,7 +256,10 @@ export const PriorityQueue = Object.freeze({
|
|
|
229
256
|
* Close a queue
|
|
230
257
|
*/
|
|
231
258
|
async closeQueue(queueName) {
|
|
232
|
-
|
|
259
|
+
const queueRedis = loadQueueRedisModule();
|
|
260
|
+
if (!queueRedis)
|
|
261
|
+
return;
|
|
262
|
+
await queueRedis.BullMQRedisQueue.closeQueue(queueName);
|
|
233
263
|
Logger.info(`Closed queue "${queueName}"`);
|
|
234
264
|
},
|
|
235
265
|
/**
|
|
@@ -237,7 +267,10 @@ export const PriorityQueue = Object.freeze({
|
|
|
237
267
|
*/
|
|
238
268
|
async shutdown() {
|
|
239
269
|
Logger.info('PriorityQueue shutting down via BullMQRedisQueue...');
|
|
240
|
-
|
|
270
|
+
const queueRedis = loadQueueRedisModule();
|
|
271
|
+
if (!queueRedis)
|
|
272
|
+
return;
|
|
273
|
+
await queueRedis.BullMQRedisQueue.shutdown();
|
|
241
274
|
Logger.info('PriorityQueue shutdown complete');
|
|
242
275
|
},
|
|
243
276
|
});
|
package/dist/WorkerFactory.d.ts
CHANGED
|
@@ -73,9 +73,9 @@ export type WorkerInstance = {
|
|
|
73
73
|
connectionState?: 'disconnected' | 'connecting' | 'connected' | 'error';
|
|
74
74
|
};
|
|
75
75
|
type RedisEnvConfig = {
|
|
76
|
-
env
|
|
76
|
+
env?: true;
|
|
77
77
|
host?: string;
|
|
78
|
-
port?:
|
|
78
|
+
port?: number;
|
|
79
79
|
password?: string;
|
|
80
80
|
db?: string;
|
|
81
81
|
};
|
package/dist/WorkerFactory.js
CHANGED
|
@@ -502,8 +502,8 @@ const resolveRedisFallbacks = () => {
|
|
|
502
502
|
const resolveRedisConfigFromEnv = (config, context) => {
|
|
503
503
|
const fallback = resolveRedisFallbacks();
|
|
504
504
|
const host = requireRedisHost(resolveEnvString(config.host ?? 'REDIS_HOST', fallback.host), context);
|
|
505
|
-
const port = resolveEnvInt(config.port ?? 'REDIS_PORT', fallback.port);
|
|
506
|
-
const db =
|
|
505
|
+
const port = resolveEnvInt(String(config.port ?? 'REDIS_PORT'), fallback.port);
|
|
506
|
+
const db = config.db ? Number(config.db) : Env.getInt('REDIS_DB', fallback.db);
|
|
507
507
|
const password = resolveEnvString(config.password ?? 'REDIS_PASSWORD', fallback.password);
|
|
508
508
|
return {
|
|
509
509
|
host,
|
|
@@ -944,25 +944,49 @@ const initializeDatacenter = (config) => {
|
|
|
944
944
|
};
|
|
945
945
|
const setupWorkerEventListeners = (worker, workerName, workerVersion, features) => {
|
|
946
946
|
worker.on('completed', (job) => {
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
worker
|
|
951
|
-
|
|
952
|
-
|
|
947
|
+
try {
|
|
948
|
+
Logger.debug(`Job completed: ${workerName}`, { jobId: job.id });
|
|
949
|
+
if (features?.observability === true) {
|
|
950
|
+
Observability.incrementCounter('worker.jobs.completed', 1, {
|
|
951
|
+
worker: workerName,
|
|
952
|
+
version: workerVersion,
|
|
953
|
+
});
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
catch (error) {
|
|
957
|
+
// Isolate error - don't let it bubble up
|
|
958
|
+
Logger.error(`Error in worker completed event handler: ${workerName}`, error, 'workers');
|
|
953
959
|
}
|
|
954
960
|
});
|
|
955
961
|
worker.on('failed', (job, error) => {
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
worker
|
|
960
|
-
|
|
961
|
-
|
|
962
|
+
try {
|
|
963
|
+
Logger.error(`Job failed: ${workerName}`, { error, jobId: job?.id }, 'workers');
|
|
964
|
+
if (features?.observability === true) {
|
|
965
|
+
Observability.incrementCounter('worker.jobs.failed', 1, {
|
|
966
|
+
worker: workerName,
|
|
967
|
+
version: workerVersion,
|
|
968
|
+
});
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
catch (handlerError) {
|
|
972
|
+
// Isolate error - don't let it bubble up
|
|
973
|
+
Logger.error(`Error in worker failed event handler: ${workerName}`, handlerError, 'workers');
|
|
962
974
|
}
|
|
963
975
|
});
|
|
964
976
|
worker.on('error', (error) => {
|
|
965
|
-
|
|
977
|
+
try {
|
|
978
|
+
Logger.error(`Worker error: ${workerName}`, error);
|
|
979
|
+
// Check if this is a Redis connection error that should be handled gracefully
|
|
980
|
+
if (error.message.includes('ERR value is not an integer') ||
|
|
981
|
+
error.message.includes('NOAUTH') ||
|
|
982
|
+
error.message.includes('ECONNREFUSED')) {
|
|
983
|
+
Logger.warn(`Worker ${workerName} encountered Redis configuration error - worker will remain failed but server will continue running`);
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
catch (handlerError) {
|
|
987
|
+
// Isolate error - don't let it bubble up
|
|
988
|
+
Logger.error(`Error in worker error event handler: ${workerName}`, handlerError, 'workers');
|
|
989
|
+
}
|
|
966
990
|
});
|
|
967
991
|
};
|
|
968
992
|
const registerWorkerInstance = (params) => {
|
package/dist/WorkerMetrics.js
CHANGED
|
@@ -3,11 +3,7 @@
|
|
|
3
3
|
* Time-series metrics persistence with Redis Sorted Sets
|
|
4
4
|
* Sealed namespace for immutability
|
|
5
5
|
*/
|
|
6
|
-
import { ErrorFactory, Logger,
|
|
7
|
-
const PREFIX = appConfig.prefix;
|
|
8
|
-
// Redis key prefixes
|
|
9
|
-
const METRICS_PREFIX = `${PREFIX}:worker:metrics:`;
|
|
10
|
-
const HEALTH_PREFIX = `${PREFIX}:worker:health:`;
|
|
6
|
+
import { ErrorFactory, Logger, RedisKeys, createRedisConnection, } from '@zintrust/core';
|
|
11
7
|
// Retention periods (in seconds)
|
|
12
8
|
const RETENTION = {
|
|
13
9
|
hourly: 7 * 24 * 60 * 60, // 7 days
|
|
@@ -18,15 +14,17 @@ const RETENTION = {
|
|
|
18
14
|
let redisClient = null;
|
|
19
15
|
/**
|
|
20
16
|
* Helper: Get Redis key for metrics
|
|
17
|
+
* Uses singleton RedisKeys for consistent key management
|
|
21
18
|
*/
|
|
22
19
|
const getMetricsKey = (workerName, metricType, granularity) => {
|
|
23
|
-
return
|
|
20
|
+
return RedisKeys.createMetricsKey(workerName, metricType, granularity);
|
|
24
21
|
};
|
|
25
22
|
/**
|
|
26
23
|
* Helper: Get Redis key for health scores
|
|
24
|
+
* Uses singleton RedisKeys for consistent key management
|
|
27
25
|
*/
|
|
28
26
|
const getHealthKey = (workerName) => {
|
|
29
|
-
return
|
|
27
|
+
return RedisKeys.createHealthKey(workerName);
|
|
30
28
|
};
|
|
31
29
|
/**
|
|
32
30
|
* Helper: Round timestamp to granularity
|
|
@@ -428,9 +426,9 @@ export const WorkerMetrics = Object.freeze({
|
|
|
428
426
|
}
|
|
429
427
|
try {
|
|
430
428
|
// Find all unique worker names from health keys
|
|
431
|
-
const pattern = `${
|
|
429
|
+
const pattern = `${RedisKeys.healthPrefix}*`;
|
|
432
430
|
const keys = await redisClient.keys(pattern);
|
|
433
|
-
const workerNames = keys.map((key) => key.replace(
|
|
431
|
+
const workerNames = keys.map((key) => key.replace(RedisKeys.healthPrefix, ''));
|
|
434
432
|
const summaries = await Promise.all(workerNames.map(async (workerName) => {
|
|
435
433
|
const now = new Date();
|
|
436
434
|
const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
|
|
@@ -478,7 +476,7 @@ export const WorkerMetrics = Object.freeze({
|
|
|
478
476
|
throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
|
|
479
477
|
}
|
|
480
478
|
try {
|
|
481
|
-
const pattern = `${
|
|
479
|
+
const pattern = `${RedisKeys.metricsPrefix}${workerName}:*`;
|
|
482
480
|
const keys = await redisClient.keys(pattern);
|
|
483
481
|
if (keys.length > 0) {
|
|
484
482
|
await redisClient.del(...keys);
|
package/dist/WorkerShutdown.js
CHANGED
|
@@ -111,15 +111,11 @@ function registerShutdownHandlers() {
|
|
|
111
111
|
}
|
|
112
112
|
process.exit(1);
|
|
113
113
|
});
|
|
114
|
-
process.on('unhandledRejection',
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
catch {
|
|
120
|
-
// Ignore errors during emergency shutdown
|
|
121
|
-
}
|
|
122
|
-
process.exit(1);
|
|
114
|
+
process.on('unhandledRejection', (reason) => {
|
|
115
|
+
// Only log the error - don't shut down the entire application
|
|
116
|
+
Logger.error('💥 Unhandled promise rejection detected', reason);
|
|
117
|
+
Logger.warn('⚠️ This error has been logged but will not shut down the server');
|
|
118
|
+
Logger.warn('⚠️ Check the error context and fix the underlying issue');
|
|
123
119
|
});
|
|
124
120
|
shutdownHandlersRegistered = true;
|
|
125
121
|
Logger.debug('Worker management system shutdown handlers registered');
|
package/dist/build-manifest.json
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zintrust/workers",
|
|
3
|
-
"version": "0.1.
|
|
4
|
-
"buildDate": "2026-01-
|
|
3
|
+
"version": "0.1.27",
|
|
4
|
+
"buildDate": "2026-01-29T10:20:40.563Z",
|
|
5
5
|
"buildEnvironment": {
|
|
6
6
|
"node": "v22.20.0",
|
|
7
7
|
"platform": "darwin",
|
|
8
8
|
"arch": "arm64"
|
|
9
9
|
},
|
|
10
10
|
"git": {
|
|
11
|
-
"commit": "
|
|
11
|
+
"commit": "15551a6c",
|
|
12
12
|
"branch": "dev"
|
|
13
13
|
},
|
|
14
14
|
"package": {
|
|
@@ -17,8 +17,6 @@
|
|
|
17
17
|
},
|
|
18
18
|
"dependencies": [
|
|
19
19
|
"@opentelemetry/api",
|
|
20
|
-
"@zintrust/queue-redis",
|
|
21
|
-
"@zintrust/queue-monitor",
|
|
22
20
|
"hot-shots",
|
|
23
21
|
"ioredis",
|
|
24
22
|
"ml.js",
|
|
@@ -155,8 +153,8 @@
|
|
|
155
153
|
"sha256": "c7610ac74b5e1ce9cae991d15b8eeecadc1a7732fcf42a8a4a95ecbea29f19de"
|
|
156
154
|
},
|
|
157
155
|
"PriorityQueue.js": {
|
|
158
|
-
"size":
|
|
159
|
-
"sha256": "
|
|
156
|
+
"size": 8689,
|
|
157
|
+
"sha256": "348cce03da64d4b2b6131dfb37903c81e9005781951485f4a6afaf925fe8f2ce"
|
|
160
158
|
},
|
|
161
159
|
"ResourceMonitor.d.ts": {
|
|
162
160
|
"size": 4019,
|
|
@@ -175,12 +173,12 @@
|
|
|
175
173
|
"sha256": "3785712c1cc30f4cfbdaebfc738f5e2b4dc0dd80134f78c42e24287909803b2c"
|
|
176
174
|
},
|
|
177
175
|
"WorkerFactory.d.ts": {
|
|
178
|
-
"size":
|
|
179
|
-
"sha256": "
|
|
176
|
+
"size": 6405,
|
|
177
|
+
"sha256": "8cfd3b1743f21b11c16a1f8f6285354626ecbe957e3e5b79801165a42289b716"
|
|
180
178
|
},
|
|
181
179
|
"WorkerFactory.js": {
|
|
182
|
-
"size":
|
|
183
|
-
"sha256": "
|
|
180
|
+
"size": 60631,
|
|
181
|
+
"sha256": "a285c6d9f3405d805cc90245994bbef55d041f18b7e86b6bde00dbf8dd5035bd"
|
|
184
182
|
},
|
|
185
183
|
"WorkerInit.d.ts": {
|
|
186
184
|
"size": 2391,
|
|
@@ -195,8 +193,8 @@
|
|
|
195
193
|
"sha256": "e68d3abf7e83bb3c0317c518325653ef3468a2a989a888ff997ce56ccc64e50f"
|
|
196
194
|
},
|
|
197
195
|
"WorkerMetrics.js": {
|
|
198
|
-
"size":
|
|
199
|
-
"sha256": "
|
|
196
|
+
"size": 18875,
|
|
197
|
+
"sha256": "a95f130989a9ffccbd77a8955568c0b603ef89a04c8f166b14fac8814407086f"
|
|
200
198
|
},
|
|
201
199
|
"WorkerRegistry.d.ts": {
|
|
202
200
|
"size": 3753,
|
|
@@ -211,8 +209,8 @@
|
|
|
211
209
|
"sha256": "f7c77d325f325729976206e1666d6d084ae486e08e047e5cea6ce8bfafbb3359"
|
|
212
210
|
},
|
|
213
211
|
"WorkerShutdown.js": {
|
|
214
|
-
"size":
|
|
215
|
-
"sha256": "
|
|
212
|
+
"size": 5371,
|
|
213
|
+
"sha256": "6c25fe18b27b881acfd2b424d31bbbcbb57ae2bd3c0d86d4ee54b8a28e3e9cb9"
|
|
216
214
|
},
|
|
217
215
|
"WorkerVersioning.d.ts": {
|
|
218
216
|
"size": 2881,
|
|
@@ -223,8 +221,8 @@
|
|
|
223
221
|
"sha256": "8af20d462270e7044c6ea983821f5b6e6ce8a5caf39b6e8fefff07c9a0bf071e"
|
|
224
222
|
},
|
|
225
223
|
"build-manifest.json": {
|
|
226
|
-
"size":
|
|
227
|
-
"sha256": "
|
|
224
|
+
"size": 16012,
|
|
225
|
+
"sha256": "a0f6ef230375ffbef2eac3f2275ff5c34327b10806a4a2c057f8bbad566f33ed"
|
|
228
226
|
},
|
|
229
227
|
"config/workerConfig.d.ts": {
|
|
230
228
|
"size": 86,
|
|
@@ -415,8 +413,8 @@
|
|
|
415
413
|
"sha256": "333d1433cf26d6e4d0ab1a5b0da4400846fd924aacce8e7d8e5305b4253290c6"
|
|
416
414
|
},
|
|
417
415
|
"index.js": {
|
|
418
|
-
"size":
|
|
419
|
-
"sha256": "
|
|
416
|
+
"size": 2142,
|
|
417
|
+
"sha256": "afe0657df6f04542ca71e7a8d827f122075f22a073b76e17b7c2a2b1bd007913"
|
|
420
418
|
},
|
|
421
419
|
"routes/workers.d.ts": {
|
|
422
420
|
"size": 498,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zintrust/workers",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.29",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"node": ">=20.0.0"
|
|
32
32
|
},
|
|
33
33
|
"peerDependencies": {
|
|
34
|
-
"@zintrust/core": "^0.1.
|
|
34
|
+
"@zintrust/core": "^0.1.34"
|
|
35
35
|
},
|
|
36
36
|
"publishConfig": {
|
|
37
37
|
"access": "public"
|
|
@@ -42,12 +42,14 @@
|
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
44
|
"@opentelemetry/api": "^1.9.0",
|
|
45
|
-
"@zintrust/queue-redis": "file:../queue-redis",
|
|
46
|
-
"@zintrust/queue-monitor": "file:../queue-monitor",
|
|
47
45
|
"hot-shots": "^10.2.1",
|
|
48
46
|
"ioredis": "^5.9.2",
|
|
49
47
|
"ml.js": "^0.0.1",
|
|
50
48
|
"prom-client": "^15.1.3",
|
|
51
49
|
"simple-statistics": "^7.8.8"
|
|
50
|
+
},
|
|
51
|
+
"optionalDependencies": {
|
|
52
|
+
"@zintrust/queue-redis": "^0.1.27",
|
|
53
|
+
"@zintrust/queue-monitor": "^0.1.27"
|
|
52
54
|
}
|
|
53
55
|
}
|
package/src/PriorityQueue.ts
CHANGED
|
@@ -4,8 +4,7 @@
|
|
|
4
4
|
* Sealed namespace for immutability
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import { ErrorFactory, Logger, type RedisConfig } from '@zintrust/core';
|
|
8
|
-
import { BullMQRedisQueue } from '@zintrust/queue-redis';
|
|
7
|
+
import { ErrorFactory, Logger, NodeSingletons, type RedisConfig } from '@zintrust/core';
|
|
9
8
|
import type { Queue } from 'bullmq';
|
|
10
9
|
|
|
11
10
|
export type PriorityLevel = 'critical' | 'high' | 'normal' | 'low';
|
|
@@ -54,11 +53,41 @@ const PRIORITY_VALUES: Record<PriorityLevel, number> = {
|
|
|
54
53
|
low: 0,
|
|
55
54
|
};
|
|
56
55
|
|
|
56
|
+
type QueueRedisModule = typeof import('@zintrust/queue-redis');
|
|
57
|
+
|
|
58
|
+
const require = NodeSingletons.module.createRequire(import.meta.url);
|
|
59
|
+
let queueRedisModule: QueueRedisModule | undefined;
|
|
60
|
+
let hasWarnedMissingQueueRedis = false;
|
|
61
|
+
|
|
62
|
+
const loadQueueRedisModule = (): QueueRedisModule | undefined => {
|
|
63
|
+
if (queueRedisModule) return queueRedisModule;
|
|
64
|
+
|
|
65
|
+
try {
|
|
66
|
+
queueRedisModule = require('@zintrust/queue-redis') as QueueRedisModule;
|
|
67
|
+
return queueRedisModule;
|
|
68
|
+
} catch (error) {
|
|
69
|
+
if (!hasWarnedMissingQueueRedis) {
|
|
70
|
+
hasWarnedMissingQueueRedis = true;
|
|
71
|
+
Logger.warn(
|
|
72
|
+
'Optional package "@zintrust/queue-redis" is not installed. PriorityQueue features are disabled until it is installed.',
|
|
73
|
+
error as Error
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
|
|
57
80
|
/**
|
|
58
81
|
* Helper: Get or create queue via shared driver
|
|
59
82
|
*/
|
|
60
83
|
const getQueue = (queueName: string): Queue => {
|
|
61
|
-
|
|
84
|
+
const queueRedis = loadQueueRedisModule();
|
|
85
|
+
if (!queueRedis) {
|
|
86
|
+
throw ErrorFactory.createWorkerError(
|
|
87
|
+
'Optional package "@zintrust/queue-redis" is required for PriorityQueue. Install it to use queue features.'
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
return queueRedis.BullMQRedisQueue.getQueue(queueName) as Queue;
|
|
62
91
|
};
|
|
63
92
|
|
|
64
93
|
/**
|
|
@@ -269,7 +298,9 @@ export const PriorityQueue = Object.freeze({
|
|
|
269
298
|
* Get all queue names
|
|
270
299
|
*/
|
|
271
300
|
getQueueNames(): string[] {
|
|
272
|
-
|
|
301
|
+
const queueRedis = loadQueueRedisModule();
|
|
302
|
+
if (!queueRedis) return [];
|
|
303
|
+
return queueRedis.BullMQRedisQueue.getQueueNames();
|
|
273
304
|
},
|
|
274
305
|
|
|
275
306
|
/**
|
|
@@ -304,7 +335,10 @@ export const PriorityQueue = Object.freeze({
|
|
|
304
335
|
async obliterate(queueName: string, force = false): Promise<void> {
|
|
305
336
|
const queue = getQueue(queueName);
|
|
306
337
|
await queue.obliterate({ force });
|
|
307
|
-
|
|
338
|
+
const queueRedis = loadQueueRedisModule();
|
|
339
|
+
if (queueRedis) {
|
|
340
|
+
await queueRedis.BullMQRedisQueue.closeQueue(queueName);
|
|
341
|
+
}
|
|
308
342
|
|
|
309
343
|
Logger.warn(`Obliterated queue "${queueName}"`);
|
|
310
344
|
},
|
|
@@ -334,7 +368,9 @@ export const PriorityQueue = Object.freeze({
|
|
|
334
368
|
* Close a queue
|
|
335
369
|
*/
|
|
336
370
|
async closeQueue(queueName: string): Promise<void> {
|
|
337
|
-
|
|
371
|
+
const queueRedis = loadQueueRedisModule();
|
|
372
|
+
if (!queueRedis) return;
|
|
373
|
+
await queueRedis.BullMQRedisQueue.closeQueue(queueName);
|
|
338
374
|
Logger.info(`Closed queue "${queueName}"`);
|
|
339
375
|
},
|
|
340
376
|
|
|
@@ -343,7 +379,9 @@ export const PriorityQueue = Object.freeze({
|
|
|
343
379
|
*/
|
|
344
380
|
async shutdown(): Promise<void> {
|
|
345
381
|
Logger.info('PriorityQueue shutting down via BullMQRedisQueue...');
|
|
346
|
-
|
|
382
|
+
const queueRedis = loadQueueRedisModule();
|
|
383
|
+
if (!queueRedis) return;
|
|
384
|
+
await queueRedis.BullMQRedisQueue.shutdown();
|
|
347
385
|
Logger.info('PriorityQueue shutdown complete');
|
|
348
386
|
},
|
|
349
387
|
});
|
package/src/WorkerFactory.ts
CHANGED
|
@@ -161,9 +161,9 @@ export type WorkerInstance = {
|
|
|
161
161
|
};
|
|
162
162
|
|
|
163
163
|
type RedisEnvConfig = {
|
|
164
|
-
env
|
|
164
|
+
env?: true;
|
|
165
165
|
host?: string;
|
|
166
|
-
port?:
|
|
166
|
+
port?: number;
|
|
167
167
|
password?: string;
|
|
168
168
|
db?: string;
|
|
169
169
|
};
|
|
@@ -836,8 +836,8 @@ const resolveRedisConfigFromEnv = (config: RedisEnvConfig, context: string): Red
|
|
|
836
836
|
resolveEnvString(config.host ?? 'REDIS_HOST', fallback.host),
|
|
837
837
|
context
|
|
838
838
|
);
|
|
839
|
-
const port = resolveEnvInt(config.port ?? 'REDIS_PORT', fallback.port);
|
|
840
|
-
const db =
|
|
839
|
+
const port = resolveEnvInt(String(config.port ?? 'REDIS_PORT'), fallback.port);
|
|
840
|
+
const db = config.db ? Number(config.db) : Env.getInt('REDIS_DB', fallback.db);
|
|
841
841
|
const password = resolveEnvString(config.password ?? 'REDIS_PASSWORD', fallback.password);
|
|
842
842
|
|
|
843
843
|
return {
|
|
@@ -1408,29 +1408,55 @@ const setupWorkerEventListeners = (
|
|
|
1408
1408
|
features?: WorkerFactoryConfig['features']
|
|
1409
1409
|
): void => {
|
|
1410
1410
|
worker.on('completed', (job: Job) => {
|
|
1411
|
-
|
|
1411
|
+
try {
|
|
1412
|
+
Logger.debug(`Job completed: ${workerName}`, { jobId: job.id });
|
|
1412
1413
|
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1414
|
+
if (features?.observability === true) {
|
|
1415
|
+
Observability.incrementCounter('worker.jobs.completed', 1, {
|
|
1416
|
+
worker: workerName,
|
|
1417
|
+
version: workerVersion,
|
|
1418
|
+
});
|
|
1419
|
+
}
|
|
1420
|
+
} catch (error) {
|
|
1421
|
+
// Isolate error - don't let it bubble up
|
|
1422
|
+
Logger.error(`Error in worker completed event handler: ${workerName}`, error, 'workers');
|
|
1418
1423
|
}
|
|
1419
1424
|
});
|
|
1420
1425
|
|
|
1421
1426
|
worker.on('failed', (job: Job | undefined, error: Error) => {
|
|
1422
|
-
|
|
1427
|
+
try {
|
|
1428
|
+
Logger.error(`Job failed: ${workerName}`, { error, jobId: job?.id }, 'workers');
|
|
1423
1429
|
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1430
|
+
if (features?.observability === true) {
|
|
1431
|
+
Observability.incrementCounter('worker.jobs.failed', 1, {
|
|
1432
|
+
worker: workerName,
|
|
1433
|
+
version: workerVersion,
|
|
1434
|
+
});
|
|
1435
|
+
}
|
|
1436
|
+
} catch (handlerError) {
|
|
1437
|
+
// Isolate error - don't let it bubble up
|
|
1438
|
+
Logger.error(`Error in worker failed event handler: ${workerName}`, handlerError, 'workers');
|
|
1429
1439
|
}
|
|
1430
1440
|
});
|
|
1431
1441
|
|
|
1432
1442
|
worker.on('error', (error: Error) => {
|
|
1433
|
-
|
|
1443
|
+
try {
|
|
1444
|
+
Logger.error(`Worker error: ${workerName}`, error);
|
|
1445
|
+
|
|
1446
|
+
// Check if this is a Redis connection error that should be handled gracefully
|
|
1447
|
+
if (
|
|
1448
|
+
error.message.includes('ERR value is not an integer') ||
|
|
1449
|
+
error.message.includes('NOAUTH') ||
|
|
1450
|
+
error.message.includes('ECONNREFUSED')
|
|
1451
|
+
) {
|
|
1452
|
+
Logger.warn(
|
|
1453
|
+
`Worker ${workerName} encountered Redis configuration error - worker will remain failed but server will continue running`
|
|
1454
|
+
);
|
|
1455
|
+
}
|
|
1456
|
+
} catch (handlerError) {
|
|
1457
|
+
// Isolate error - don't let it bubble up
|
|
1458
|
+
Logger.error(`Error in worker error event handler: ${workerName}`, handlerError, 'workers');
|
|
1459
|
+
}
|
|
1434
1460
|
});
|
|
1435
1461
|
};
|
|
1436
1462
|
|
package/src/WorkerMetrics.ts
CHANGED
|
@@ -7,14 +7,12 @@
|
|
|
7
7
|
import {
|
|
8
8
|
ErrorFactory,
|
|
9
9
|
Logger,
|
|
10
|
-
|
|
10
|
+
RedisKeys,
|
|
11
11
|
createRedisConnection,
|
|
12
12
|
type RedisConfig,
|
|
13
13
|
} from '@zintrust/core';
|
|
14
14
|
import type IORedis from 'ioredis';
|
|
15
15
|
|
|
16
|
-
const PREFIX = appConfig.prefix;
|
|
17
|
-
|
|
18
16
|
export type MetricType =
|
|
19
17
|
| 'processed'
|
|
20
18
|
| 'errors'
|
|
@@ -76,10 +74,6 @@ export type WorkerHealthScore = {
|
|
|
76
74
|
status: 'healthy' | 'degraded' | 'unhealthy';
|
|
77
75
|
};
|
|
78
76
|
|
|
79
|
-
// Redis key prefixes
|
|
80
|
-
const METRICS_PREFIX = `${PREFIX}:worker:metrics:`;
|
|
81
|
-
const HEALTH_PREFIX = `${PREFIX}:worker:health:`;
|
|
82
|
-
|
|
83
77
|
// Retention periods (in seconds)
|
|
84
78
|
const RETENTION = {
|
|
85
79
|
hourly: 7 * 24 * 60 * 60, // 7 days
|
|
@@ -92,20 +86,22 @@ let redisClient: IORedis | null = null;
|
|
|
92
86
|
|
|
93
87
|
/**
|
|
94
88
|
* Helper: Get Redis key for metrics
|
|
89
|
+
* Uses singleton RedisKeys for consistent key management
|
|
95
90
|
*/
|
|
96
91
|
const getMetricsKey = (
|
|
97
92
|
workerName: string,
|
|
98
93
|
metricType: MetricType,
|
|
99
94
|
granularity: MetricGranularity
|
|
100
95
|
): string => {
|
|
101
|
-
return
|
|
96
|
+
return RedisKeys.createMetricsKey(workerName, metricType, granularity);
|
|
102
97
|
};
|
|
103
98
|
|
|
104
99
|
/**
|
|
105
100
|
* Helper: Get Redis key for health scores
|
|
101
|
+
* Uses singleton RedisKeys for consistent key management
|
|
106
102
|
*/
|
|
107
103
|
const getHealthKey = (workerName: string): string => {
|
|
108
|
-
return
|
|
104
|
+
return RedisKeys.createHealthKey(workerName);
|
|
109
105
|
};
|
|
110
106
|
|
|
111
107
|
/**
|
|
@@ -613,9 +609,9 @@ export const WorkerMetrics = Object.freeze({
|
|
|
613
609
|
|
|
614
610
|
try {
|
|
615
611
|
// Find all unique worker names from health keys
|
|
616
|
-
const pattern = `${
|
|
612
|
+
const pattern = `${RedisKeys.healthPrefix}*`;
|
|
617
613
|
const keys = await redisClient.keys(pattern);
|
|
618
|
-
const workerNames = keys.map((key) => key.replace(
|
|
614
|
+
const workerNames = keys.map((key) => key.replace(RedisKeys.healthPrefix, ''));
|
|
619
615
|
|
|
620
616
|
const summaries = await Promise.all(
|
|
621
617
|
workerNames.map(async (workerName) => {
|
|
@@ -671,7 +667,7 @@ export const WorkerMetrics = Object.freeze({
|
|
|
671
667
|
}
|
|
672
668
|
|
|
673
669
|
try {
|
|
674
|
-
const pattern = `${
|
|
670
|
+
const pattern = `${RedisKeys.metricsPrefix}${workerName}:*`;
|
|
675
671
|
const keys = await redisClient.keys(pattern);
|
|
676
672
|
|
|
677
673
|
if (keys.length > 0) {
|
package/src/WorkerShutdown.ts
CHANGED
|
@@ -155,14 +155,11 @@ function registerShutdownHandlers(): void {
|
|
|
155
155
|
process.exit(1);
|
|
156
156
|
});
|
|
157
157
|
|
|
158
|
-
process.on('unhandledRejection',
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
// Ignore errors during emergency shutdown
|
|
164
|
-
}
|
|
165
|
-
process.exit(1);
|
|
158
|
+
process.on('unhandledRejection', (reason: unknown) => {
|
|
159
|
+
// Only log the error - don't shut down the entire application
|
|
160
|
+
Logger.error('💥 Unhandled promise rejection detected', reason);
|
|
161
|
+
Logger.warn('⚠️ This error has been logged but will not shut down the server');
|
|
162
|
+
Logger.warn('⚠️ Check the error context and fix the underlying issue');
|
|
166
163
|
});
|
|
167
164
|
|
|
168
165
|
shutdownHandlersRegistered = true;
|