@zintrust/workers 0.1.31 → 0.1.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ClusterLock.js +3 -2
- package/dist/DeadLetterQueue.js +3 -2
- package/dist/HealthMonitor.js +24 -13
- package/dist/Observability.js +8 -0
- package/dist/WorkerFactory.d.ts +4 -0
- package/dist/WorkerFactory.js +384 -42
- package/dist/WorkerInit.js +122 -43
- package/dist/WorkerMetrics.js +5 -1
- package/dist/WorkerRegistry.js +8 -0
- package/dist/WorkerShutdown.d.ts +0 -13
- package/dist/WorkerShutdown.js +1 -44
- package/dist/build-manifest.json +99 -83
- package/dist/config/workerConfig.d.ts +1 -0
- package/dist/config/workerConfig.js +7 -1
- package/dist/createQueueWorker.js +281 -42
- package/dist/dashboard/workers-api.js +8 -1
- package/dist/http/WorkerController.js +90 -35
- package/dist/http/WorkerMonitoringService.js +29 -2
- package/dist/index.d.ts +1 -2
- package/dist/index.js +0 -1
- package/dist/routes/workers.js +10 -7
- package/dist/storage/WorkerStore.d.ts +6 -3
- package/dist/storage/WorkerStore.js +16 -0
- package/dist/telemetry/api/TelemetryMonitoringService.js +29 -2
- package/dist/ui/router/ui.js +58 -29
- package/dist/ui/workers/index.html +202 -0
- package/dist/ui/workers/main.js +1952 -0
- package/dist/ui/workers/styles.css +1350 -0
- package/dist/ui/workers/zintrust.svg +30 -0
- package/package.json +5 -5
- package/src/ClusterLock.ts +13 -7
- package/src/ComplianceManager.ts +3 -2
- package/src/DeadLetterQueue.ts +6 -4
- package/src/HealthMonitor.ts +33 -17
- package/src/Observability.ts +11 -0
- package/src/WorkerFactory.ts +446 -43
- package/src/WorkerInit.ts +167 -48
- package/src/WorkerMetrics.ts +14 -8
- package/src/WorkerRegistry.ts +11 -0
- package/src/WorkerShutdown.ts +1 -69
- package/src/config/workerConfig.ts +9 -1
- package/src/createQueueWorker.ts +428 -43
- package/src/dashboard/workers-api.ts +8 -1
- package/src/http/WorkerController.ts +111 -36
- package/src/http/WorkerMonitoringService.ts +35 -2
- package/src/index.ts +2 -3
- package/src/routes/workers.ts +10 -8
- package/src/storage/WorkerStore.ts +21 -3
- package/src/telemetry/api/TelemetryMonitoringService.ts +35 -2
- package/src/types/queue-monitor.d.ts +2 -1
- package/src/ui/router/EmbeddedAssets.ts +3 -0
- package/src/ui/router/ui.ts +57 -39
- package/src/WorkerShutdownDurableObject.ts +0 -64
|
@@ -1,55 +1,294 @@
|
|
|
1
|
-
import
|
|
1
|
+
import * as Core from '@zintrust/core';
|
|
2
|
+
import { Env, Logger, Queue } from '@zintrust/core';
|
|
3
|
+
const RETRY_BASE_DELAY_MS = 1000;
|
|
4
|
+
const RETRY_MAX_DELAY_MS = 30000;
|
|
5
|
+
const getJobStateTracker = () => {
|
|
6
|
+
try {
|
|
7
|
+
return Core['JobStateTracker'];
|
|
8
|
+
}
|
|
9
|
+
catch {
|
|
10
|
+
return undefined;
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
const getJobHeartbeatStore = () => {
|
|
14
|
+
try {
|
|
15
|
+
return Core['JobHeartbeatStore'];
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
const getTimeoutManager = () => {
|
|
22
|
+
try {
|
|
23
|
+
return Core['TimeoutManager'];
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
const getEnvInt = (key, fallback) => {
|
|
30
|
+
const getter = Env.getInt;
|
|
31
|
+
if (typeof getter === 'function') {
|
|
32
|
+
return getter(key, fallback);
|
|
33
|
+
}
|
|
34
|
+
const raw = Env[key];
|
|
35
|
+
if (typeof raw === 'number' && Number.isFinite(raw))
|
|
36
|
+
return Math.floor(raw);
|
|
37
|
+
if (typeof raw === 'string' && raw.trim() !== '') {
|
|
38
|
+
const parsed = Number(raw);
|
|
39
|
+
if (Number.isFinite(parsed))
|
|
40
|
+
return Math.floor(parsed);
|
|
41
|
+
}
|
|
42
|
+
return fallback;
|
|
43
|
+
};
|
|
44
|
+
const resolveQueueJobTimeoutMs = () => {
|
|
45
|
+
const timeoutManager = getTimeoutManager();
|
|
46
|
+
const tm = (timeoutManager ?? {});
|
|
47
|
+
if (typeof tm.getQueueJobTimeoutMs === 'function') {
|
|
48
|
+
return tm.getQueueJobTimeoutMs();
|
|
49
|
+
}
|
|
50
|
+
return Math.max(1000, getEnvInt('QUEUE_JOB_TIMEOUT', 60) * 1000);
|
|
51
|
+
};
|
|
52
|
+
const runWithTimeout = async (operation, timeoutMs, operationName) => {
|
|
53
|
+
const timeoutManager = getTimeoutManager();
|
|
54
|
+
const tm = (timeoutManager ?? {});
|
|
55
|
+
if (typeof tm.withTimeout === 'function') {
|
|
56
|
+
return tm.withTimeout(operation, timeoutMs, operationName);
|
|
57
|
+
}
|
|
58
|
+
return operation();
|
|
59
|
+
};
|
|
60
|
+
const isTimeoutError = (error) => {
|
|
61
|
+
const timeoutManager = getTimeoutManager();
|
|
62
|
+
const tm = (timeoutManager ?? {});
|
|
63
|
+
if (typeof tm.isTimeoutError === 'function') {
|
|
64
|
+
return tm.isTimeoutError(error);
|
|
65
|
+
}
|
|
66
|
+
if (error instanceof Error) {
|
|
67
|
+
return error.message.toLowerCase().includes('timed out');
|
|
68
|
+
}
|
|
69
|
+
return false;
|
|
70
|
+
};
|
|
71
|
+
const normalizeAttempts = (value) => {
|
|
72
|
+
if (typeof value === 'number' && Number.isFinite(value) && value >= 0) {
|
|
73
|
+
return Math.floor(value);
|
|
74
|
+
}
|
|
75
|
+
return 0;
|
|
76
|
+
};
|
|
77
|
+
const getAttemptsFromMessage = (message) => {
|
|
78
|
+
const payloadAttempts = typeof message.payload === 'object' && message.payload !== null
|
|
79
|
+
? normalizeAttempts(message.payload['attempts'])
|
|
80
|
+
: 0;
|
|
81
|
+
const messageAttempts = normalizeAttempts(message.attempts);
|
|
82
|
+
return Math.max(payloadAttempts, messageAttempts);
|
|
83
|
+
};
|
|
84
|
+
const getRetryDelayMs = (nextAttempts) => {
|
|
85
|
+
const exponentialDelay = Math.min(RETRY_BASE_DELAY_MS * 2 ** Math.max(0, nextAttempts - 1), RETRY_MAX_DELAY_MS);
|
|
86
|
+
const jitterMs = Math.floor(Math.random() * 250); // NOSONAR
|
|
87
|
+
return exponentialDelay + jitterMs;
|
|
88
|
+
};
|
|
2
89
|
const buildBaseLogFields = (message, getLogFields) => {
|
|
3
90
|
return {
|
|
4
91
|
messageId: message.id,
|
|
5
92
|
...getLogFields(message.payload),
|
|
6
93
|
};
|
|
7
94
|
};
|
|
8
|
-
const
|
|
9
|
-
return
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
95
|
+
const getWorkerInstanceId = () => {
|
|
96
|
+
return typeof Env['WORKER_INSTANCE_ID'] === 'string'
|
|
97
|
+
? String(Env['WORKER_INSTANCE_ID'])
|
|
98
|
+
: undefined;
|
|
99
|
+
};
|
|
100
|
+
const getTrackerApi = () => {
|
|
101
|
+
return (getJobStateTracker() ?? {}) ?? {};
|
|
102
|
+
};
|
|
103
|
+
const getHeartbeatStoreApi = () => {
|
|
104
|
+
return (getJobHeartbeatStore() ?? {}) ?? {};
|
|
105
|
+
};
|
|
106
|
+
const removeHeartbeatIfSupported = async (queueName, jobId) => {
|
|
107
|
+
const heartbeatStore = getHeartbeatStoreApi();
|
|
108
|
+
if (typeof heartbeatStore.remove === 'function') {
|
|
109
|
+
await heartbeatStore.remove(queueName, jobId);
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
const scheduleHeartbeatLoop = (trackerApi, queueName, jobId, workerInstanceId, heartbeatIntervalMs) => {
|
|
113
|
+
return setInterval(() => {
|
|
114
|
+
if (typeof trackerApi.heartbeat === 'function') {
|
|
115
|
+
void trackerApi.heartbeat({
|
|
116
|
+
queueName,
|
|
117
|
+
jobId,
|
|
118
|
+
workerInstanceId,
|
|
22
119
|
});
|
|
23
|
-
// Re-queue original payload
|
|
24
|
-
await Queue.enqueue(queueName, message.payload, driverName);
|
|
25
|
-
await Queue.ack(queueName, message.id, driverName);
|
|
26
|
-
return false;
|
|
27
|
-
}
|
|
28
|
-
try {
|
|
29
|
-
Logger.info(`Processing queued ${options.kindLabel}`, baseLogFields);
|
|
30
|
-
await options.handle(message.payload);
|
|
31
|
-
await Queue.ack(queueName, message.id, driverName);
|
|
32
|
-
Logger.info(`${options.kindLabel} processed successfully`, baseLogFields);
|
|
33
|
-
return true;
|
|
34
120
|
}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
121
|
+
const heartbeatStore = getHeartbeatStoreApi();
|
|
122
|
+
if (typeof heartbeatStore.heartbeat === 'function') {
|
|
123
|
+
void heartbeatStore.heartbeat({
|
|
124
|
+
queueName,
|
|
125
|
+
jobId,
|
|
126
|
+
workerInstanceId,
|
|
127
|
+
intervalMs: heartbeatIntervalMs,
|
|
41
128
|
});
|
|
42
|
-
if (attempts < options.maxAttempts) {
|
|
43
|
-
await Queue.enqueue(queueName, message.payload, driverName);
|
|
44
|
-
Logger.info(`${options.kindLabel} re-queued for retry`, {
|
|
45
|
-
...baseLogFields,
|
|
46
|
-
attempts: attempts + 1,
|
|
47
|
-
});
|
|
48
|
-
}
|
|
49
|
-
await Queue.ack(queueName, message.id, driverName);
|
|
50
|
-
// We processed the message (even if it failed), so return true to continue processing
|
|
51
|
-
return true;
|
|
52
129
|
}
|
|
130
|
+
}, heartbeatIntervalMs);
|
|
131
|
+
};
|
|
132
|
+
const checkAndRequeueIfNotDue = async (options, queueName, driverName, message, baseLogFields) => {
|
|
133
|
+
const payload = message.payload;
|
|
134
|
+
const rawTimestamp = 'timestamp' in payload ? payload['timestamp'] : 0;
|
|
135
|
+
const timestamp = typeof rawTimestamp === 'number' ? rawTimestamp : 0;
|
|
136
|
+
if (timestamp <= Date.now())
|
|
137
|
+
return false;
|
|
138
|
+
Logger.info(`${options.kindLabel} not due yet, re-queueing`, {
|
|
139
|
+
...baseLogFields,
|
|
140
|
+
dueAt: new Date(timestamp).toISOString(),
|
|
141
|
+
});
|
|
142
|
+
await Queue.enqueue(queueName, message.payload, driverName);
|
|
143
|
+
await Queue.ack(queueName, message.id, driverName);
|
|
144
|
+
return true;
|
|
145
|
+
};
|
|
146
|
+
const onProcessSuccess = async (input) => {
|
|
147
|
+
await Queue.ack(input.queueName, input.message.id, input.driverName);
|
|
148
|
+
if (typeof input.trackerApi.completed === 'function') {
|
|
149
|
+
await input.trackerApi.completed({
|
|
150
|
+
queueName: input.queueName,
|
|
151
|
+
jobId: input.message.id,
|
|
152
|
+
processingTimeMs: Date.now() - input.startedAtMs,
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
await removeHeartbeatIfSupported(input.queueName, input.message.id);
|
|
156
|
+
Logger.info(`${input.options.kindLabel} processed successfully`, input.baseLogFields);
|
|
157
|
+
return true;
|
|
158
|
+
};
|
|
159
|
+
const onProcessFailure = async (input) => {
|
|
160
|
+
const attempts = getAttemptsFromMessage(input.message);
|
|
161
|
+
const nextAttempts = attempts + 1;
|
|
162
|
+
const isFinal = nextAttempts >= input.options.maxAttempts;
|
|
163
|
+
let retryAt;
|
|
164
|
+
Logger.error(`Failed to process ${input.options.kindLabel}`, {
|
|
165
|
+
...input.baseLogFields,
|
|
166
|
+
error: input.error,
|
|
167
|
+
attempts: nextAttempts,
|
|
168
|
+
});
|
|
169
|
+
if (isTimeoutError(input.error) && typeof input.trackerApi.timedOut === 'function') {
|
|
170
|
+
await input.trackerApi.timedOut({
|
|
171
|
+
queueName: input.queueName,
|
|
172
|
+
jobId: input.message.id,
|
|
173
|
+
reason: `Worker processing exceeded timeout for ${input.options.kindLabel}`,
|
|
174
|
+
error: input.error,
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
if (nextAttempts < input.options.maxAttempts) {
|
|
178
|
+
const retryDelayMs = getRetryDelayMs(nextAttempts);
|
|
179
|
+
retryAt = new Date(Date.now() + retryDelayMs).toISOString();
|
|
180
|
+
const currentPayload = typeof input.message.payload === 'object' && input.message.payload !== null
|
|
181
|
+
? input.message.payload
|
|
182
|
+
: { payload: input.message.payload };
|
|
183
|
+
const payloadForRetry = {
|
|
184
|
+
...currentPayload,
|
|
185
|
+
attempts: nextAttempts,
|
|
186
|
+
timestamp: Date.now() + retryDelayMs,
|
|
187
|
+
};
|
|
188
|
+
await Queue.enqueue(input.queueName, payloadForRetry, input.driverName);
|
|
189
|
+
Logger.info(`${input.options.kindLabel} re-queued for retry`, {
|
|
190
|
+
...input.baseLogFields,
|
|
191
|
+
attempts: nextAttempts,
|
|
192
|
+
retryDelayMs,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
await Queue.ack(input.queueName, input.message.id, input.driverName);
|
|
196
|
+
await removeHeartbeatIfSupported(input.queueName, input.message.id);
|
|
197
|
+
if (typeof input.trackerApi.failed === 'function') {
|
|
198
|
+
await input.trackerApi.failed({
|
|
199
|
+
queueName: input.queueName,
|
|
200
|
+
jobId: input.message.id,
|
|
201
|
+
attempts: nextAttempts,
|
|
202
|
+
isFinal,
|
|
203
|
+
retryAt,
|
|
204
|
+
error: input.error,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
return true;
|
|
208
|
+
};
|
|
209
|
+
const startTrackingAndHeartbeat = async (input) => {
|
|
210
|
+
const startedAtMs = Date.now();
|
|
211
|
+
const timeoutMs = resolveQueueJobTimeoutMs();
|
|
212
|
+
const heartbeatIntervalMs = Math.max(1000, getEnvInt('JOB_HEARTBEAT_INTERVAL_MS', 10000));
|
|
213
|
+
const attempts = getAttemptsFromMessage(input.message);
|
|
214
|
+
const workerInstanceId = getWorkerInstanceId();
|
|
215
|
+
if (typeof input.trackerApi.started === 'function') {
|
|
216
|
+
await input.trackerApi.started({
|
|
217
|
+
queueName: input.queueName,
|
|
218
|
+
jobId: input.message.id,
|
|
219
|
+
attempts: attempts + 1,
|
|
220
|
+
timeoutMs,
|
|
221
|
+
workerName: input.options.kindLabel,
|
|
222
|
+
workerInstanceId,
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
const heartbeatStore = getHeartbeatStoreApi();
|
|
226
|
+
if (typeof heartbeatStore.heartbeat === 'function') {
|
|
227
|
+
await heartbeatStore.heartbeat({
|
|
228
|
+
queueName: input.queueName,
|
|
229
|
+
jobId: input.message.id,
|
|
230
|
+
workerInstanceId,
|
|
231
|
+
intervalMs: heartbeatIntervalMs,
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
const heartbeatTimer = scheduleHeartbeatLoop(input.trackerApi, input.queueName, input.message.id, workerInstanceId, heartbeatIntervalMs);
|
|
235
|
+
return { startedAtMs, heartbeatTimer };
|
|
236
|
+
};
|
|
237
|
+
const processQueueMessage = async (options, queueName, driverName) => {
|
|
238
|
+
const message = await Queue.dequeue(queueName, driverName);
|
|
239
|
+
if (!message)
|
|
240
|
+
return false;
|
|
241
|
+
const baseLogFields = buildBaseLogFields(message, options.getLogFields);
|
|
242
|
+
const isRequeued = await checkAndRequeueIfNotDue(options, queueName, driverName, message, baseLogFields);
|
|
243
|
+
if (isRequeued)
|
|
244
|
+
return false;
|
|
245
|
+
let heartbeatTimer;
|
|
246
|
+
const trackerApi = getTrackerApi();
|
|
247
|
+
const timeoutMs = resolveQueueJobTimeoutMs();
|
|
248
|
+
let startedAtMs;
|
|
249
|
+
try {
|
|
250
|
+
const tracking = await startTrackingAndHeartbeat({
|
|
251
|
+
options,
|
|
252
|
+
trackerApi,
|
|
253
|
+
queueName,
|
|
254
|
+
message,
|
|
255
|
+
});
|
|
256
|
+
startedAtMs = tracking.startedAtMs;
|
|
257
|
+
heartbeatTimer = tracking.heartbeatTimer;
|
|
258
|
+
Logger.info(`Processing queued ${options.kindLabel}`, baseLogFields);
|
|
259
|
+
await runWithTimeout(async () => {
|
|
260
|
+
await options.handle(message.payload);
|
|
261
|
+
}, timeoutMs, `${options.kindLabel}:${queueName}:${message.id}`);
|
|
262
|
+
return onProcessSuccess({
|
|
263
|
+
options,
|
|
264
|
+
trackerApi,
|
|
265
|
+
queueName,
|
|
266
|
+
driverName,
|
|
267
|
+
message,
|
|
268
|
+
startedAtMs,
|
|
269
|
+
baseLogFields,
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
catch (error) {
|
|
273
|
+
return onProcessFailure({
|
|
274
|
+
options,
|
|
275
|
+
trackerApi,
|
|
276
|
+
queueName,
|
|
277
|
+
driverName,
|
|
278
|
+
message,
|
|
279
|
+
baseLogFields,
|
|
280
|
+
error,
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
finally {
|
|
284
|
+
if (heartbeatTimer !== undefined) {
|
|
285
|
+
clearInterval(heartbeatTimer);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
};
|
|
289
|
+
const createProcessOne = (options) => {
|
|
290
|
+
return async (queueName = options.defaultQueueName, driverName) => {
|
|
291
|
+
return processQueueMessage(options, queueName, driverName);
|
|
53
292
|
};
|
|
54
293
|
};
|
|
55
294
|
const createProcessAll = (defaultQueueName, processOne) => {
|
|
@@ -425,7 +425,14 @@ async function getRedisQueueData() {
|
|
|
425
425
|
if (redisConfig?.driver !== 'redis') {
|
|
426
426
|
throw ErrorFactory.createConfigError('Redis driver not configured');
|
|
427
427
|
}
|
|
428
|
-
const monitor = QueueMonitor.create({
|
|
428
|
+
const monitor = QueueMonitor.create({
|
|
429
|
+
redis: {
|
|
430
|
+
host: redisConfig.host || 'localhost',
|
|
431
|
+
port: redisConfig.port || 6379,
|
|
432
|
+
db: redisConfig.database || 1,
|
|
433
|
+
password: redisConfig.password,
|
|
434
|
+
},
|
|
435
|
+
});
|
|
429
436
|
const snapshot = await monitor.getSnapshot();
|
|
430
437
|
let totalJobs = 0;
|
|
431
438
|
let processingJobs = 0;
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Worker Controller
|
|
4
4
|
* HTTP handlers for worker management API
|
|
5
5
|
*/
|
|
6
|
-
import { Env, Logger, getValidatedBody } from '@zintrust/core';
|
|
6
|
+
import { Cloudflare, Env, Logger, getValidatedBody, } from '@zintrust/core';
|
|
7
7
|
import { CanaryController } from '../CanaryController';
|
|
8
8
|
import { HealthMonitor } from '../HealthMonitor';
|
|
9
9
|
import { getParam } from '../helper';
|
|
@@ -24,39 +24,41 @@ const getBody = (req) => {
|
|
|
24
24
|
{});
|
|
25
25
|
};
|
|
26
26
|
// ==================== Core Worker Operations ====================
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
async
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
27
|
+
const isCloudflareEnv = () => Cloudflare.getWorkersEnv() !== null;
|
|
28
|
+
const validateCreatePayload = (body, res) => {
|
|
29
|
+
if (!body.name || !body.queueName || !body.processor || !body.version) {
|
|
30
|
+
res.setStatus(400).json({
|
|
31
|
+
error: 'Missing required fields',
|
|
32
|
+
message: 'name, queueName, processor, and version are required',
|
|
33
|
+
code: 'MISSING_REQUIRED_FIELDS',
|
|
34
|
+
});
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
return true;
|
|
38
|
+
};
|
|
39
|
+
const respondIfWorkerExists = async (name, persistenceOverride, res) => {
|
|
40
|
+
const existing = await WorkerFactory.getPersisted(name, persistenceOverride);
|
|
41
|
+
if (!existing)
|
|
42
|
+
return false;
|
|
43
|
+
res.status(409).json({
|
|
44
|
+
ok: false,
|
|
45
|
+
error: `Worker ${name} already exists`,
|
|
46
|
+
code: 'WORKER_EXISTS',
|
|
47
|
+
worker: existing,
|
|
48
|
+
});
|
|
49
|
+
return true;
|
|
50
|
+
};
|
|
51
|
+
const resolveCreateProcessor = async (body, res) => {
|
|
52
|
+
const rawProcessor = body.processor;
|
|
53
|
+
let processor = rawProcessor;
|
|
54
|
+
let processorSpec;
|
|
55
|
+
if (!isCloudflareEnv()) {
|
|
54
56
|
if (typeof rawProcessor === 'string') {
|
|
55
57
|
processorSpec = rawProcessor;
|
|
56
58
|
const resolved = await WorkerFactory.resolveProcessorSpec(rawProcessor);
|
|
57
59
|
if (!resolved) {
|
|
58
60
|
res.setStatus(400).json({ error: 'Processor spec could not be resolved' });
|
|
59
|
-
return;
|
|
61
|
+
return null;
|
|
60
62
|
}
|
|
61
63
|
processor = resolved;
|
|
62
64
|
}
|
|
@@ -65,13 +67,22 @@ async function create(req, res) {
|
|
|
65
67
|
}
|
|
66
68
|
if (typeof processor !== 'function') {
|
|
67
69
|
res.setStatus(400).json({ error: 'Processor must be a function or resolvable path' });
|
|
68
|
-
return;
|
|
70
|
+
return null;
|
|
69
71
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
return { processor, processorSpec };
|
|
73
|
+
}
|
|
74
|
+
// Cloudflare environment: treat string as spec, otherwise accept as-is
|
|
75
|
+
if (typeof rawProcessor === 'string') {
|
|
76
|
+
processorSpec = rawProcessor;
|
|
77
|
+
processor = async () => { };
|
|
78
|
+
}
|
|
79
|
+
return { processor, processorSpec };
|
|
80
|
+
};
|
|
81
|
+
const finalizeWorkerCreate = async (config, res) => {
|
|
82
|
+
const globalAutoStart = Env.getBool('WORKER_AUTO_START', false);
|
|
83
|
+
const workerAutoStart = config.autoStart ?? globalAutoStart;
|
|
84
|
+
const isCloudflare = isCloudflareEnv();
|
|
85
|
+
if (!isCloudflare && globalAutoStart && workerAutoStart) {
|
|
75
86
|
await WorkerFactory.create(config);
|
|
76
87
|
res.json({
|
|
77
88
|
ok: true,
|
|
@@ -79,6 +90,50 @@ async function create(req, res) {
|
|
|
79
90
|
status: 'creating',
|
|
80
91
|
message: 'Worker creation started. Check status endpoint for progress.',
|
|
81
92
|
});
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
await WorkerFactory.register(config);
|
|
96
|
+
res.json({
|
|
97
|
+
ok: true,
|
|
98
|
+
workerName: config.name,
|
|
99
|
+
status: 'registered',
|
|
100
|
+
message: isCloudflare
|
|
101
|
+
? 'Worker registered. Cloudflare environment detected; worker will be picked up by external processor.'
|
|
102
|
+
: 'Worker registered successfully.',
|
|
103
|
+
});
|
|
104
|
+
};
|
|
105
|
+
/**
|
|
106
|
+
* Create a new worker instance
|
|
107
|
+
* @param req.body.name - Worker name (required)
|
|
108
|
+
* @param req.body.queueName - Queue name (required)
|
|
109
|
+
* @param req.body.processor - Job processor function (required; internal only)
|
|
110
|
+
* @param req.body.version - Worker version (optional)
|
|
111
|
+
* @param req.body.options - BullMQ worker options (optional)
|
|
112
|
+
* @param req.body.infrastructure - Infrastructure config (optional)
|
|
113
|
+
* @param req.body.features - Feature flags (optional)
|
|
114
|
+
* @param req.body.datacenter - Datacenter placement config (optional)
|
|
115
|
+
* @returns Success response with worker name
|
|
116
|
+
*/
|
|
117
|
+
async function create(req, res) {
|
|
118
|
+
Logger.info('WorkerController.create called');
|
|
119
|
+
try {
|
|
120
|
+
const body = req.data();
|
|
121
|
+
if (!validateCreatePayload(body, res))
|
|
122
|
+
return;
|
|
123
|
+
const name = body.name;
|
|
124
|
+
const persistenceOverride = resolvePersistenceOverride(req);
|
|
125
|
+
const exists = await respondIfWorkerExists(name, persistenceOverride, res);
|
|
126
|
+
if (exists)
|
|
127
|
+
return;
|
|
128
|
+
const resolvedProcessor = await resolveCreateProcessor(body, res);
|
|
129
|
+
if (!resolvedProcessor)
|
|
130
|
+
return;
|
|
131
|
+
const config = {
|
|
132
|
+
...body,
|
|
133
|
+
processor: resolvedProcessor.processor,
|
|
134
|
+
processorSpec: resolvedProcessor.processorSpec,
|
|
135
|
+
};
|
|
136
|
+
await finalizeWorkerCreate(config, res);
|
|
82
137
|
}
|
|
83
138
|
catch (error) {
|
|
84
139
|
Logger.error('WorkerController.create failed', error);
|
|
@@ -1,9 +1,36 @@
|
|
|
1
1
|
import { Logger, NodeSingletons, workersConfig } from '@zintrust/core';
|
|
2
2
|
import { HealthMonitor } from '../HealthMonitor';
|
|
3
3
|
import { getWorkers } from '../dashboard/workers-api';
|
|
4
|
+
const createFallbackEmitter = () => {
|
|
5
|
+
const listeners = new Map();
|
|
6
|
+
return {
|
|
7
|
+
on(event, listener) {
|
|
8
|
+
const set = listeners.get(event) ?? new Set();
|
|
9
|
+
set.add(listener);
|
|
10
|
+
listeners.set(event, set);
|
|
11
|
+
},
|
|
12
|
+
off(event, listener) {
|
|
13
|
+
const set = listeners.get(event);
|
|
14
|
+
if (!set)
|
|
15
|
+
return;
|
|
16
|
+
set.delete(listener);
|
|
17
|
+
if (set.size === 0)
|
|
18
|
+
listeners.delete(event);
|
|
19
|
+
},
|
|
20
|
+
emit(event, payload) {
|
|
21
|
+
const set = listeners.get(event);
|
|
22
|
+
if (!set)
|
|
23
|
+
return false;
|
|
24
|
+
for (const listener of set)
|
|
25
|
+
listener(payload);
|
|
26
|
+
return true;
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
};
|
|
4
30
|
// Internal state
|
|
5
|
-
const
|
|
6
|
-
emitter
|
|
31
|
+
const EventEmitterCtor = NodeSingletons?.EventEmitter;
|
|
32
|
+
const emitter = typeof EventEmitterCtor === 'function' ? new EventEmitterCtor() : createFallbackEmitter();
|
|
33
|
+
emitter.setMaxListeners?.(Infinity);
|
|
7
34
|
let interval = null;
|
|
8
35
|
let subscribers = 0;
|
|
9
36
|
const INTERVAL_MS = workersConfig?.intervalMs || 5000;
|
package/dist/index.d.ts
CHANGED
|
@@ -26,14 +26,13 @@ export { WorkerFactory } from './WorkerFactory';
|
|
|
26
26
|
export type { ProcessorResolver, WorkerFactoryConfig, WorkerPersistenceConfig, } from './WorkerFactory';
|
|
27
27
|
export { WorkerInit } from './WorkerInit';
|
|
28
28
|
export { WorkerShutdown } from './WorkerShutdown';
|
|
29
|
-
export { ZinTrustWorkerShutdownDurableObject } from './WorkerShutdownDurableObject';
|
|
30
29
|
export { WorkerController } from './http/WorkerController';
|
|
31
30
|
export { registerWorkerRoutes } from './routes/workers';
|
|
32
31
|
export { BroadcastWorker } from './BroadcastWorker';
|
|
33
32
|
export { createQueueWorker } from './createQueueWorker';
|
|
34
33
|
export type { CreateQueueWorkerOptions } from './createQueueWorker';
|
|
35
34
|
export { NotificationWorker } from './NotificationWorker';
|
|
36
|
-
export type { RedisConfig, WorkerAutoScalingConfig, WorkerComplianceConfig, WorkerConfig, WorkerCostConfig, WorkerObservabilityConfig,
|
|
35
|
+
export type { RedisConfig, WorkerAutoScalingConfig, WorkerComplianceConfig, WorkerConfig, WorkerCostConfig, WorkerObservabilityConfig, WorkerStatus, WorkerVersioningConfig, WorkersConfigOverrides, WorkersGlobalConfig, } from '@zintrust/core';
|
|
37
36
|
export type { Job, Worker, WorkerOptions } from 'bullmq';
|
|
38
37
|
export type { IAnomaly, IAnomalyConfig, IForecast, IMetric, IPrediction, IRecommendation, IRootCauseAnalysis, } from './AnomalyDetection';
|
|
39
38
|
export type { IChaosComparison, IChaosExperiment, IChaosReport, IChaosStatus, } from './ChaosEngineering';
|
package/dist/index.js
CHANGED
|
@@ -33,7 +33,6 @@ export { WorkerVersioning } from './WorkerVersioning';
|
|
|
33
33
|
export { WorkerFactory } from './WorkerFactory';
|
|
34
34
|
export { WorkerInit } from './WorkerInit';
|
|
35
35
|
export { WorkerShutdown } from './WorkerShutdown';
|
|
36
|
-
export { ZinTrustWorkerShutdownDurableObject } from './WorkerShutdownDurableObject';
|
|
37
36
|
// HTTP Controllers & Routes
|
|
38
37
|
export { WorkerController } from './http/WorkerController';
|
|
39
38
|
export { registerWorkerRoutes } from './routes/workers';
|
package/dist/routes/workers.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Worker Management Routes
|
|
3
3
|
* HTTP API for managing workers with dashboard functionality
|
|
4
4
|
*/
|
|
5
|
-
import { Logger, Router } from '@zintrust/core';
|
|
5
|
+
import { Cloudflare, Env, Logger, Router } from '@zintrust/core';
|
|
6
6
|
import { HealthMonitor } from '../HealthMonitor';
|
|
7
7
|
import { ValidationSchemas, withCustomValidation } from '../http/middleware/CustomValidation';
|
|
8
8
|
import { withEditWorkerValidation } from '../http/middleware/EditWorkerValidation';
|
|
@@ -70,11 +70,13 @@ function registerUtilityRoutes(r) {
|
|
|
70
70
|
function registerWorkerLifecycleRoutes(router, middleware) {
|
|
71
71
|
Router.group(router, '/api/workers', (r) => {
|
|
72
72
|
Logger.info('Registering Worker Management Routes');
|
|
73
|
-
registerMonitoringRoutes(r); // ← Move FIRST - has /events
|
|
74
73
|
registerCoreWorkerRoutes(r);
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
74
|
+
if (Cloudflare.getWorkersEnv() === null) {
|
|
75
|
+
registerMonitoringRoutes(r); // ← Move FIRST - has /events
|
|
76
|
+
registerWorkerQueryRoutes(r);
|
|
77
|
+
registerVersioningRoutes(r);
|
|
78
|
+
registerUtilityRoutes(r);
|
|
79
|
+
}
|
|
78
80
|
}, { middleware: middleware });
|
|
79
81
|
}
|
|
80
82
|
function registerWorkerTelemetryRoutes(router, middleware) {
|
|
@@ -117,7 +119,8 @@ export function registerWorkerRoutes(router, _options, routeOptions) {
|
|
|
117
119
|
basePath: '/telemetry',
|
|
118
120
|
});
|
|
119
121
|
dashboard.registerRoutes(router);
|
|
120
|
-
|
|
121
|
-
Logger.info(
|
|
122
|
+
const port = Env.get('PORT', '7777');
|
|
123
|
+
Logger.info(`Worker routes registered at http://127.0.0.1:${port}/workers`);
|
|
124
|
+
Logger.info(`Telemetry dashboard registered at http://127.0.0.1:${port}/telemetry`);
|
|
122
125
|
}
|
|
123
126
|
export default registerWorkerRoutes;
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
* Worker Store
|
|
3
3
|
* Persistence layer for workers (memory, redis, db)
|
|
4
4
|
*/
|
|
5
|
-
import type {
|
|
6
|
-
import type
|
|
5
|
+
import type { createRedisConnection } from '@zintrust/core';
|
|
6
|
+
import { type IDatabase } from '@zintrust/core';
|
|
7
|
+
type RedisConnection = ReturnType<typeof createRedisConnection>;
|
|
7
8
|
export type WorkerRecord = {
|
|
8
9
|
name: string;
|
|
9
10
|
queueName: string;
|
|
@@ -36,13 +37,15 @@ export type WorkerStore = {
|
|
|
36
37
|
update(name: string, patch: Partial<WorkerRecord>): Promise<void>;
|
|
37
38
|
updateMany?: (names: string[], patch: Partial<WorkerRecord>) => Promise<void>;
|
|
38
39
|
remove(name: string): Promise<void>;
|
|
40
|
+
close?(): Promise<void>;
|
|
39
41
|
};
|
|
40
42
|
export declare const InMemoryWorkerStore: Readonly<{
|
|
41
43
|
create(): WorkerStore;
|
|
42
44
|
}>;
|
|
43
45
|
export declare const RedisWorkerStore: Readonly<{
|
|
44
|
-
create(client:
|
|
46
|
+
create(client: RedisConnection, keyPrefix?: string): WorkerStore;
|
|
45
47
|
}>;
|
|
46
48
|
export declare const DbWorkerStore: Readonly<{
|
|
47
49
|
create(db: IDatabase, table?: string): WorkerStore;
|
|
48
50
|
}>;
|
|
51
|
+
export {};
|