@zintrust/workers 0.1.29 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -1
- package/dist/AnomalyDetection.d.ts +4 -0
- package/dist/AnomalyDetection.js +8 -0
- package/dist/BroadcastWorker.d.ts +2 -0
- package/dist/CanaryController.js +49 -5
- package/dist/ChaosEngineering.js +13 -0
- package/dist/ClusterLock.js +21 -10
- package/dist/DeadLetterQueue.js +12 -8
- package/dist/MultiQueueWorker.d.ts +1 -1
- package/dist/MultiQueueWorker.js +12 -7
- package/dist/NotificationWorker.d.ts +2 -0
- package/dist/PriorityQueue.d.ts +2 -2
- package/dist/PriorityQueue.js +20 -21
- package/dist/ResourceMonitor.js +65 -38
- package/dist/WorkerFactory.d.ts +23 -3
- package/dist/WorkerFactory.js +420 -40
- package/dist/WorkerInit.js +8 -3
- package/dist/WorkerMetrics.d.ts +2 -1
- package/dist/WorkerMetrics.js +152 -93
- package/dist/WorkerRegistry.d.ts +6 -0
- package/dist/WorkerRegistry.js +70 -1
- package/dist/WorkerShutdown.d.ts +21 -0
- package/dist/WorkerShutdown.js +82 -9
- package/dist/WorkerShutdownDurableObject.d.ts +12 -0
- package/dist/WorkerShutdownDurableObject.js +41 -0
- package/dist/build-manifest.json +171 -99
- package/dist/createQueueWorker.d.ts +2 -0
- package/dist/createQueueWorker.js +42 -27
- package/dist/dashboard/types.d.ts +5 -0
- package/dist/dashboard/workers-api.js +136 -43
- package/dist/http/WorkerApiController.js +1 -0
- package/dist/http/WorkerController.js +133 -85
- package/dist/http/WorkerMonitoringService.d.ts +11 -0
- package/dist/http/WorkerMonitoringService.js +62 -0
- package/dist/http/middleware/CustomValidation.js +1 -1
- package/dist/http/middleware/EditWorkerValidation.d.ts +1 -1
- package/dist/http/middleware/EditWorkerValidation.js +7 -6
- package/dist/http/middleware/ProcessorPathSanitizer.js +101 -35
- package/dist/http/middleware/WorkerValidationChain.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -0
- package/dist/routes/workers.js +48 -6
- package/dist/storage/WorkerStore.d.ts +4 -1
- package/dist/storage/WorkerStore.js +55 -7
- package/dist/telemetry/api/TelemetryAPI.d.ts +46 -0
- package/dist/telemetry/api/TelemetryAPI.js +219 -0
- package/dist/telemetry/api/TelemetryMonitoringService.d.ts +17 -0
- package/dist/telemetry/api/TelemetryMonitoringService.js +113 -0
- package/dist/telemetry/components/AlertPanel.d.ts +1 -0
- package/dist/telemetry/components/AlertPanel.js +13 -0
- package/dist/telemetry/components/CostTracking.d.ts +1 -0
- package/dist/telemetry/components/CostTracking.js +14 -0
- package/dist/telemetry/components/ResourceUsageChart.d.ts +1 -0
- package/dist/telemetry/components/ResourceUsageChart.js +11 -0
- package/dist/telemetry/components/WorkerHealthChart.d.ts +1 -0
- package/dist/telemetry/components/WorkerHealthChart.js +11 -0
- package/dist/telemetry/index.d.ts +15 -0
- package/dist/telemetry/index.js +60 -0
- package/dist/telemetry/routes/dashboard.d.ts +6 -0
- package/dist/telemetry/routes/dashboard.js +608 -0
- package/dist/ui/router/EmbeddedAssets.d.ts +4 -0
- package/dist/ui/router/EmbeddedAssets.js +13 -0
- package/dist/ui/router/ui.js +100 -4
- package/package.json +10 -6
- package/src/AnomalyDetection.ts +9 -0
- package/src/CanaryController.ts +41 -5
- package/src/ChaosEngineering.ts +14 -0
- package/src/ClusterLock.ts +22 -9
- package/src/DeadLetterQueue.ts +13 -8
- package/src/MultiQueueWorker.ts +15 -8
- package/src/PriorityQueue.ts +21 -22
- package/src/ResourceMonitor.ts +72 -40
- package/src/WorkerFactory.ts +545 -49
- package/src/WorkerInit.ts +8 -3
- package/src/WorkerMetrics.ts +183 -105
- package/src/WorkerRegistry.ts +80 -1
- package/src/WorkerShutdown.ts +115 -9
- package/src/WorkerShutdownDurableObject.ts +64 -0
- package/src/createQueueWorker.ts +73 -30
- package/src/dashboard/types.ts +5 -0
- package/src/dashboard/workers-api.ts +165 -52
- package/src/http/WorkerApiController.ts +1 -0
- package/src/http/WorkerController.ts +167 -90
- package/src/http/WorkerMonitoringService.ts +77 -0
- package/src/http/middleware/CustomValidation.ts +1 -1
- package/src/http/middleware/EditWorkerValidation.ts +7 -6
- package/src/http/middleware/ProcessorPathSanitizer.ts +123 -36
- package/src/http/middleware/WorkerValidationChain.ts +1 -0
- package/src/index.ts +6 -1
- package/src/routes/workers.ts +66 -9
- package/src/storage/WorkerStore.ts +59 -9
- package/src/telemetry/api/TelemetryAPI.ts +292 -0
- package/src/telemetry/api/TelemetryMonitoringService.ts +149 -0
- package/src/telemetry/components/AlertPanel.ts +13 -0
- package/src/telemetry/components/CostTracking.ts +14 -0
- package/src/telemetry/components/ResourceUsageChart.ts +11 -0
- package/src/telemetry/components/WorkerHealthChart.ts +11 -0
- package/src/telemetry/index.ts +121 -0
- package/src/telemetry/public/assets/zintrust-logo.svg +15 -0
- package/src/telemetry/routes/dashboard.ts +638 -0
- package/src/telemetry/styles/tailwind.css +1 -0
- package/src/telemetry/styles/zintrust-theme.css +8 -0
- package/src/ui/router/EmbeddedAssets.ts +13 -0
- package/src/ui/router/ui.ts +112 -5
- package/src/ui/workers/index.html +2 -2
- package/src/ui/workers/main.js +232 -61
- package/src/ui/workers/zintrust.svg +30 -0
- package/dist/dashboard/workers-dashboard-ui.d.ts +0 -3
- package/dist/dashboard/workers-dashboard-ui.js +0 -1026
- package/dist/dashboard/workers-dashboard.d.ts +0 -4
- package/dist/dashboard/workers-dashboard.js +0 -904
package/src/WorkerFactory.ts
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
registerDatabasesFromRuntimeConfig,
|
|
18
18
|
useEnsureDbConnected,
|
|
19
19
|
workersConfig,
|
|
20
|
+
ZintrustLang,
|
|
20
21
|
type IDatabase,
|
|
21
22
|
type RedisConfig,
|
|
22
23
|
type WorkerStatus,
|
|
@@ -110,9 +111,10 @@ export type WorkerFactoryConfig = {
|
|
|
110
111
|
version?: string;
|
|
111
112
|
queueName: string;
|
|
112
113
|
processor: (job: Job) => Promise<unknown>;
|
|
113
|
-
|
|
114
|
+
processorSpec?: string;
|
|
114
115
|
options?: WorkerOptions;
|
|
115
116
|
autoStart?: boolean;
|
|
117
|
+
activeStatus?: boolean;
|
|
116
118
|
infrastructure?: {
|
|
117
119
|
redis?: RedisConfigInput;
|
|
118
120
|
persistence?: WorkerPersistenceConfig;
|
|
@@ -189,7 +191,8 @@ const workers = new Map<string, WorkerInstance>();
|
|
|
189
191
|
let workerStore: WorkerStore = InMemoryWorkerStore.create();
|
|
190
192
|
let workerStoreConfigured = false;
|
|
191
193
|
let workerStoreConfig: WorkerPersistenceConfig | null = null;
|
|
192
|
-
|
|
194
|
+
|
|
195
|
+
export type ProcessorResolver = (
|
|
193
196
|
name: string
|
|
194
197
|
) =>
|
|
195
198
|
| WorkerFactoryConfig['processor']
|
|
@@ -199,6 +202,20 @@ type ProcessorResolver = (
|
|
|
199
202
|
const processorRegistry = new Map<string, WorkerFactoryConfig['processor']>();
|
|
200
203
|
const processorPathRegistry = new Map<string, string>();
|
|
201
204
|
const processorResolvers: ProcessorResolver[] = [];
|
|
205
|
+
const processorSpecRegistry = new Map<string, WorkerFactoryConfig['processor']>();
|
|
206
|
+
|
|
207
|
+
type CachedProcessor = {
|
|
208
|
+
code: string;
|
|
209
|
+
processor: WorkerFactoryConfig['processor'];
|
|
210
|
+
etag?: string;
|
|
211
|
+
cachedAt: number;
|
|
212
|
+
expiresAt: number;
|
|
213
|
+
size: number;
|
|
214
|
+
lastAccess: number;
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
const processorCache = new Map<string, CachedProcessor>();
|
|
218
|
+
let processorCacheSize = 0;
|
|
202
219
|
|
|
203
220
|
const buildPersistenceBootstrapConfig = (): WorkerFactoryConfig => {
|
|
204
221
|
const driver = Env.get('WORKER_PERSISTENCE_DRIVER', 'memory') as 'memory' | 'redis' | 'database';
|
|
@@ -249,12 +266,110 @@ const registerProcessorResolver = (resolver: ProcessorResolver): void => {
|
|
|
249
266
|
processorResolvers.push(resolver);
|
|
250
267
|
};
|
|
251
268
|
|
|
269
|
+
const registerProcessorSpec = (spec: string, processor: WorkerFactoryConfig['processor']): void => {
|
|
270
|
+
if (!spec || typeof processor !== 'function') return;
|
|
271
|
+
processorSpecRegistry.set(normalizeProcessorSpec(spec), processor);
|
|
272
|
+
};
|
|
273
|
+
|
|
252
274
|
const decodeProcessorPathEntities = (value: string): string =>
|
|
253
275
|
value
|
|
254
276
|
.replaceAll(///gi, '/')
|
|
255
277
|
.replaceAll('/', '/')
|
|
256
278
|
.replaceAll(///gi, '/');
|
|
257
279
|
|
|
280
|
+
const isUrlSpec = (spec: string): boolean => {
|
|
281
|
+
if (spec.startsWith('url:')) return true;
|
|
282
|
+
return spec.includes('://');
|
|
283
|
+
};
|
|
284
|
+
|
|
285
|
+
const normalizeProcessorSpec = (spec: string): string =>
|
|
286
|
+
spec.startsWith('url:') ? spec.slice(4) : spec;
|
|
287
|
+
|
|
288
|
+
const parseCacheControl = (value: string | null): { maxAge?: number } => {
|
|
289
|
+
if (!value) return {};
|
|
290
|
+
const parts = value.split(',').map((part) => part.trim().toLowerCase());
|
|
291
|
+
const maxAge = parts.find((part) => part.startsWith('max-age='));
|
|
292
|
+
if (!maxAge) return {};
|
|
293
|
+
const raw = maxAge.split('=')[1];
|
|
294
|
+
const parsed = Number.parseInt(raw ?? '', 10);
|
|
295
|
+
return Number.isFinite(parsed) ? { maxAge: parsed } : {};
|
|
296
|
+
};
|
|
297
|
+
|
|
298
|
+
const getProcessorSpecConfig = (): typeof workersConfig.processorSpec =>
|
|
299
|
+
workersConfig.processorSpec;
|
|
300
|
+
|
|
301
|
+
const computeSha256 = async (value: string): Promise<string> => {
|
|
302
|
+
if (typeof globalThis !== 'undefined' && globalThis.crypto?.subtle) {
|
|
303
|
+
const data = new TextEncoder().encode(value);
|
|
304
|
+
const digest = await globalThis.crypto.subtle.digest('SHA-256', data);
|
|
305
|
+
return Array.from(new Uint8Array(digest))
|
|
306
|
+
.map((b) => b.toString(16).padStart(2, '0'))
|
|
307
|
+
.join('');
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
if (typeof NodeSingletons.createHash === 'function') {
|
|
311
|
+
return NodeSingletons.createHash('sha256').update(value).digest('hex');
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
return String(Math.random()).slice(2);
|
|
315
|
+
};
|
|
316
|
+
|
|
317
|
+
const toBase64 = (value: string): string => {
|
|
318
|
+
if (typeof Buffer !== 'undefined') {
|
|
319
|
+
return Buffer.from(value, 'utf-8').toString('base64');
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
if (typeof globalThis !== 'undefined' && typeof globalThis.btoa === 'function') {
|
|
323
|
+
const bytes = new TextEncoder().encode(value);
|
|
324
|
+
let binary = '';
|
|
325
|
+
bytes.forEach((byte) => {
|
|
326
|
+
binary += String.fromCodePoint(byte);
|
|
327
|
+
});
|
|
328
|
+
return globalThis.btoa(binary);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
return value;
|
|
332
|
+
};
|
|
333
|
+
|
|
334
|
+
const getCachedProcessor = (key: string): CachedProcessor | null => {
|
|
335
|
+
const entry = processorCache.get(key);
|
|
336
|
+
if (!entry) return null;
|
|
337
|
+
const now = Date.now();
|
|
338
|
+
if (entry.expiresAt <= now) {
|
|
339
|
+
processorCache.delete(key);
|
|
340
|
+
processorCacheSize -= entry.size;
|
|
341
|
+
return null;
|
|
342
|
+
}
|
|
343
|
+
entry.lastAccess = now;
|
|
344
|
+
return entry;
|
|
345
|
+
};
|
|
346
|
+
|
|
347
|
+
const evictCacheIfNeeded = (maxSize: number): void => {
|
|
348
|
+
if (processorCacheSize <= maxSize) return;
|
|
349
|
+
const entries = Array.from(processorCache.entries());
|
|
350
|
+
entries.sort((a, b) => a[1].lastAccess - b[1].lastAccess);
|
|
351
|
+
for (const [key, entry] of entries) {
|
|
352
|
+
if (processorCacheSize <= maxSize) break;
|
|
353
|
+
processorCache.delete(key);
|
|
354
|
+
processorCacheSize -= entry.size;
|
|
355
|
+
}
|
|
356
|
+
};
|
|
357
|
+
|
|
358
|
+
const setCachedProcessor = (key: string, entry: CachedProcessor, maxSize: number): void => {
|
|
359
|
+
const existing = processorCache.get(key);
|
|
360
|
+
if (existing) {
|
|
361
|
+
processorCacheSize -= existing.size;
|
|
362
|
+
}
|
|
363
|
+
processorCache.set(key, entry);
|
|
364
|
+
processorCacheSize += entry.size;
|
|
365
|
+
evictCacheIfNeeded(maxSize);
|
|
366
|
+
};
|
|
367
|
+
|
|
368
|
+
const isAllowedRemoteHost = (host: string): boolean => {
|
|
369
|
+
const allowlist = getProcessorSpecConfig().remoteAllowlist.map((value) => value.toLowerCase());
|
|
370
|
+
return allowlist.includes(host.toLowerCase());
|
|
371
|
+
};
|
|
372
|
+
|
|
258
373
|
const waitForWorkerConnection = async (
|
|
259
374
|
worker: Worker,
|
|
260
375
|
name: string,
|
|
@@ -329,6 +444,250 @@ const sanitizeProcessorPath = (value: string): string => {
|
|
|
329
444
|
return isAbsolutePath ? base : path.resolve(process.cwd(), relativePath);
|
|
330
445
|
};
|
|
331
446
|
|
|
447
|
+
const stripProcessorExtension = (value: string): string => value.replace(/\.(ts|js)$/i, '');
|
|
448
|
+
|
|
449
|
+
const normalizeModulePath = (value: string): string => value.replaceAll('\\', '/');
|
|
450
|
+
|
|
451
|
+
const buildProcessorModuleCandidates = (modulePath: string, resolvedPath: string): string[] => {
|
|
452
|
+
const candidates: string[] = [];
|
|
453
|
+
const normalized = normalizeModulePath(modulePath.trim());
|
|
454
|
+
const normalizedResolved = normalizeModulePath(resolvedPath);
|
|
455
|
+
|
|
456
|
+
if (normalized.startsWith('/app/')) {
|
|
457
|
+
candidates.push(`@app/${stripProcessorExtension(normalized.slice(5))}`);
|
|
458
|
+
} else if (normalized.startsWith('app/')) {
|
|
459
|
+
candidates.push(`@app/${stripProcessorExtension(normalized.slice(4))}`);
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
const appIndex = normalizedResolved.lastIndexOf('/app/');
|
|
463
|
+
if (appIndex !== -1) {
|
|
464
|
+
const relative = normalizedResolved.slice(appIndex + 5);
|
|
465
|
+
if (relative) {
|
|
466
|
+
candidates.push(`@app/${stripProcessorExtension(relative)}`);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
return Array.from(new Set(candidates));
|
|
471
|
+
};
|
|
472
|
+
|
|
473
|
+
const pickProcessorFromModule = (
|
|
474
|
+
mod: Record<string, unknown> | undefined,
|
|
475
|
+
source: string
|
|
476
|
+
): WorkerFactoryConfig['processor'] | undefined => {
|
|
477
|
+
const candidate = mod?.['default'] ?? mod?.['processor'] ?? mod?.['handler'] ?? mod?.['handle'];
|
|
478
|
+
if (typeof candidate !== 'function') {
|
|
479
|
+
const keys = mod ? Object.keys(mod) : [];
|
|
480
|
+
Logger.warn(
|
|
481
|
+
`Module imported from ${source} but no valid processor function found (exported: ${keys.join(', ')})`
|
|
482
|
+
);
|
|
483
|
+
return undefined;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
return candidate as WorkerFactoryConfig['processor'];
|
|
487
|
+
};
|
|
488
|
+
|
|
489
|
+
const extractZinTrustProcessor = (
|
|
490
|
+
mod: Record<string, unknown> | undefined,
|
|
491
|
+
source: string
|
|
492
|
+
): WorkerFactoryConfig['processor'] | undefined => {
|
|
493
|
+
const candidate = mod?.['ZinTrustProcessor'];
|
|
494
|
+
if (typeof candidate !== 'function') {
|
|
495
|
+
const keys = mod ? Object.keys(mod) : [];
|
|
496
|
+
Logger.warn(
|
|
497
|
+
`Module imported from ${source} but missing ZinTrustProcessor export (exported: ${keys.join(', ')})`
|
|
498
|
+
);
|
|
499
|
+
return undefined;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
return candidate as WorkerFactoryConfig['processor'];
|
|
503
|
+
};
|
|
504
|
+
|
|
505
|
+
const readResponseBody = async (response: Response, maxSize: number): Promise<string> => {
|
|
506
|
+
const contentLength = response.headers.get('content-length');
|
|
507
|
+
if (contentLength) {
|
|
508
|
+
const size = Number.parseInt(contentLength, 10);
|
|
509
|
+
if (Number.isFinite(size) && size > maxSize) {
|
|
510
|
+
throw ErrorFactory.createConfigError('PROCESSOR_FETCH_SIZE_EXCEEDED');
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
const buffer = await response.arrayBuffer();
|
|
515
|
+
if (buffer.byteLength > maxSize) {
|
|
516
|
+
throw ErrorFactory.createConfigError('PROCESSOR_FETCH_SIZE_EXCEEDED');
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
return new TextDecoder().decode(buffer);
|
|
520
|
+
};
|
|
521
|
+
|
|
522
|
+
const computeCacheTtlSeconds = (
|
|
523
|
+
config: ReturnType<typeof getProcessorSpecConfig>,
|
|
524
|
+
cacheControl: { maxAge?: number }
|
|
525
|
+
): number =>
|
|
526
|
+
Math.min(config.cacheMaxTtlSeconds, cacheControl.maxAge ?? config.cacheDefaultTtlSeconds);
|
|
527
|
+
|
|
528
|
+
const refreshCachedProcessor = (
|
|
529
|
+
existing: CachedProcessor,
|
|
530
|
+
config: ReturnType<typeof getProcessorSpecConfig>,
|
|
531
|
+
cacheControl: { maxAge?: number }
|
|
532
|
+
): WorkerFactoryConfig['processor'] => {
|
|
533
|
+
const ttl = computeCacheTtlSeconds(config, cacheControl);
|
|
534
|
+
const now = Date.now();
|
|
535
|
+
existing.expiresAt = now + ttl * 1000;
|
|
536
|
+
existing.lastAccess = now;
|
|
537
|
+
return existing.processor;
|
|
538
|
+
};
|
|
539
|
+
|
|
540
|
+
const cacheProcessorFromResponse = async (params: {
|
|
541
|
+
response: Response;
|
|
542
|
+
normalized: string;
|
|
543
|
+
config: ReturnType<typeof getProcessorSpecConfig>;
|
|
544
|
+
cacheKey: string;
|
|
545
|
+
}): Promise<WorkerFactoryConfig['processor']> => {
|
|
546
|
+
const { response, normalized, config, cacheKey } = params;
|
|
547
|
+
const code = await readResponseBody(response, config.fetchMaxSizeBytes);
|
|
548
|
+
const dataUrl = `data:text/javascript;base64,${toBase64(code)}`;
|
|
549
|
+
const mod = await import(dataUrl);
|
|
550
|
+
const processor = extractZinTrustProcessor(mod as Record<string, unknown>, normalized);
|
|
551
|
+
if (!processor) {
|
|
552
|
+
throw ErrorFactory.createConfigError('INVALID_PROCESSOR_URL_EXPORT');
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
const cacheControl = parseCacheControl(response.headers.get('cache-control'));
|
|
556
|
+
const ttl = computeCacheTtlSeconds(config, cacheControl);
|
|
557
|
+
const size = new TextEncoder().encode(code).byteLength;
|
|
558
|
+
const now = Date.now();
|
|
559
|
+
setCachedProcessor(
|
|
560
|
+
cacheKey,
|
|
561
|
+
{
|
|
562
|
+
code,
|
|
563
|
+
processor,
|
|
564
|
+
etag: response.headers.get('etag') ?? undefined,
|
|
565
|
+
cachedAt: now,
|
|
566
|
+
expiresAt: now + ttl * 1000,
|
|
567
|
+
size,
|
|
568
|
+
lastAccess: now,
|
|
569
|
+
},
|
|
570
|
+
config.cacheMaxSizeBytes
|
|
571
|
+
);
|
|
572
|
+
|
|
573
|
+
return processor;
|
|
574
|
+
};
|
|
575
|
+
|
|
576
|
+
const delay = (ms: number): Promise<void> =>
|
|
577
|
+
new Promise((resolve) => {
|
|
578
|
+
globalThis.setTimeout(resolve, ms);
|
|
579
|
+
});
|
|
580
|
+
|
|
581
|
+
const fetchProcessorAttempt = async (params: {
|
|
582
|
+
normalized: string;
|
|
583
|
+
config: ReturnType<typeof getProcessorSpecConfig>;
|
|
584
|
+
cacheKey: string;
|
|
585
|
+
existing: CachedProcessor | undefined;
|
|
586
|
+
attempt: number;
|
|
587
|
+
maxAttempts: number;
|
|
588
|
+
}): Promise<WorkerFactoryConfig['processor'] | undefined> => {
|
|
589
|
+
const { normalized, config, cacheKey, existing, attempt, maxAttempts } = params;
|
|
590
|
+
const controller = new AbortController();
|
|
591
|
+
const timeoutId = globalThis.setTimeout(() => controller.abort(), config.fetchTimeoutMs);
|
|
592
|
+
|
|
593
|
+
try {
|
|
594
|
+
const headers: Record<string, string> = {};
|
|
595
|
+
if (existing?.etag) headers['If-None-Match'] = existing.etag;
|
|
596
|
+
|
|
597
|
+
const response = await fetch(normalized, {
|
|
598
|
+
method: 'GET',
|
|
599
|
+
headers,
|
|
600
|
+
signal: controller.signal,
|
|
601
|
+
});
|
|
602
|
+
|
|
603
|
+
if (response.status === 304 && existing) {
|
|
604
|
+
const cacheControl = parseCacheControl(response.headers.get('cache-control'));
|
|
605
|
+
return refreshCachedProcessor(existing, config, cacheControl);
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
if (!response.ok) {
|
|
609
|
+
throw ErrorFactory.createConfigError(`PROCESSOR_FETCH_FAILED:${response.status}`);
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
return await cacheProcessorFromResponse({ response, normalized, config, cacheKey });
|
|
613
|
+
} catch (error) {
|
|
614
|
+
if (controller.signal.aborted) {
|
|
615
|
+
Logger.error('Processor URL fetch timeout', error);
|
|
616
|
+
} else {
|
|
617
|
+
Logger.error('Processor URL fetch failed', error);
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
if (attempt >= maxAttempts) {
|
|
621
|
+
return undefined;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
await delay(config.retryBackoffMs * attempt);
|
|
625
|
+
return fetchProcessorAttempt({
|
|
626
|
+
normalized,
|
|
627
|
+
config,
|
|
628
|
+
cacheKey,
|
|
629
|
+
existing,
|
|
630
|
+
attempt: attempt + 1,
|
|
631
|
+
maxAttempts,
|
|
632
|
+
});
|
|
633
|
+
} finally {
|
|
634
|
+
clearTimeout(timeoutId);
|
|
635
|
+
}
|
|
636
|
+
};
|
|
637
|
+
|
|
638
|
+
const resolveProcessorFromUrl = async (
|
|
639
|
+
spec: string
|
|
640
|
+
): Promise<WorkerFactoryConfig['processor'] | undefined> => {
|
|
641
|
+
const normalized = normalizeProcessorSpec(spec);
|
|
642
|
+
let parsed: URL;
|
|
643
|
+
try {
|
|
644
|
+
parsed = new URL(normalized);
|
|
645
|
+
} catch (error) {
|
|
646
|
+
Logger.error('Invalid processor URL spec', error);
|
|
647
|
+
return undefined;
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
if (parsed.protocol === 'file:') {
|
|
651
|
+
const filePath = decodeURIComponent(parsed.pathname);
|
|
652
|
+
return resolveProcessorFromPath(filePath);
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
if (parsed.protocol !== 'https:' && parsed.protocol !== 'file:') {
|
|
656
|
+
Logger.warn(
|
|
657
|
+
`Invalid processor URL protocol: ${parsed.protocol}. Only https:// and file:// are supported.`
|
|
658
|
+
);
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
if (!isAllowedRemoteHost(parsed.host) && parsed.protocol !== 'file:') {
|
|
662
|
+
Logger.warn(`Invalid processor URL host: ${parsed.host}. Host is not in the allowlist.`);
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
const config = getProcessorSpecConfig();
|
|
666
|
+
const cacheKey = await computeSha256(normalized);
|
|
667
|
+
const cached = getCachedProcessor(cacheKey);
|
|
668
|
+
if (cached) return cached.processor;
|
|
669
|
+
|
|
670
|
+
return fetchProcessorAttempt({
|
|
671
|
+
normalized,
|
|
672
|
+
config,
|
|
673
|
+
cacheKey,
|
|
674
|
+
existing: processorCache.get(cacheKey),
|
|
675
|
+
attempt: 1,
|
|
676
|
+
maxAttempts: Math.max(1, config.retryAttempts),
|
|
677
|
+
});
|
|
678
|
+
};
|
|
679
|
+
|
|
680
|
+
const resolveProcessorSpec = async (
|
|
681
|
+
spec: string
|
|
682
|
+
): Promise<WorkerFactoryConfig['processor'] | undefined> => {
|
|
683
|
+
if (!spec) return undefined;
|
|
684
|
+
const normalized = normalizeProcessorSpec(spec);
|
|
685
|
+
const prebuilt = processorSpecRegistry.get(normalized) ?? processorSpecRegistry.get(spec);
|
|
686
|
+
if (prebuilt) return prebuilt;
|
|
687
|
+
if (isUrlSpec(spec)) return resolveProcessorFromUrl(spec);
|
|
688
|
+
return resolveProcessorFromPath(spec);
|
|
689
|
+
};
|
|
690
|
+
|
|
332
691
|
const resolveProcessorFromPath = async (
|
|
333
692
|
modulePath: string
|
|
334
693
|
): Promise<WorkerFactoryConfig['processor'] | undefined> => {
|
|
@@ -338,23 +697,34 @@ const resolveProcessorFromPath = async (
|
|
|
338
697
|
const resolved = sanitizeProcessorPath(trimmed);
|
|
339
698
|
if (!resolved) return undefined;
|
|
340
699
|
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
);
|
|
700
|
+
const importProcessorFromCandidates = async (
|
|
701
|
+
candidates: string[]
|
|
702
|
+
): Promise<WorkerFactoryConfig['processor'] | undefined> => {
|
|
703
|
+
if (candidates.length === 0) return undefined;
|
|
704
|
+
const [candidatePath, ...rest] = candidates;
|
|
705
|
+
try {
|
|
706
|
+
const mod = await import(candidatePath);
|
|
707
|
+
const candidate = pickProcessorFromModule(mod as Record<string, unknown>, candidatePath);
|
|
708
|
+
if (candidate) return candidate;
|
|
709
|
+
} catch (candidateError) {
|
|
710
|
+
Logger.debug(`Processor module candidate import failed: ${candidatePath}`, candidateError);
|
|
349
711
|
}
|
|
350
712
|
|
|
351
|
-
return
|
|
352
|
-
|
|
353
|
-
|
|
713
|
+
return importProcessorFromCandidates(rest);
|
|
714
|
+
};
|
|
715
|
+
|
|
716
|
+
try {
|
|
717
|
+
const mod = await import(resolved);
|
|
718
|
+
const candidate = pickProcessorFromModule(mod as Record<string, unknown>, resolved);
|
|
719
|
+
if (candidate) return candidate;
|
|
354
720
|
} catch (err) {
|
|
721
|
+
const candidates = buildProcessorModuleCandidates(trimmed, resolved);
|
|
722
|
+
const resolvedCandidate = await importProcessorFromCandidates(candidates);
|
|
723
|
+
if (resolvedCandidate) return resolvedCandidate;
|
|
355
724
|
Logger.error(`Failed to import processor from path: ${resolved}`, err);
|
|
356
|
-
return undefined;
|
|
357
725
|
}
|
|
726
|
+
|
|
727
|
+
return undefined;
|
|
358
728
|
};
|
|
359
729
|
|
|
360
730
|
const resolveProcessor = async (
|
|
@@ -366,7 +736,7 @@ const resolveProcessor = async (
|
|
|
366
736
|
const pathHint = processorPathRegistry.get(name);
|
|
367
737
|
if (pathHint) {
|
|
368
738
|
try {
|
|
369
|
-
const resolved = await
|
|
739
|
+
const resolved = await resolveProcessorSpec(pathHint);
|
|
370
740
|
if (resolved) return resolved;
|
|
371
741
|
} catch (error) {
|
|
372
742
|
Logger.error(`Failed to resolve processor module for "${name}"`, error);
|
|
@@ -823,8 +1193,14 @@ const resolveRedisFallbacks = (): {
|
|
|
823
1193
|
const queueRedis = queueConfig.drivers.redis;
|
|
824
1194
|
return {
|
|
825
1195
|
host: queueRedis?.driver === 'redis' ? queueRedis.host : Env.get('REDIS_HOST', '127.0.0.1'),
|
|
826
|
-
port:
|
|
827
|
-
|
|
1196
|
+
port:
|
|
1197
|
+
queueRedis?.driver === 'redis'
|
|
1198
|
+
? queueRedis.port
|
|
1199
|
+
: Env.getInt('REDIS_PORT', ZintrustLang.REDIS_DEFAULT_PORT),
|
|
1200
|
+
db:
|
|
1201
|
+
queueRedis?.driver === 'redis'
|
|
1202
|
+
? queueRedis.database
|
|
1203
|
+
: Env.getInt('REDIS_QUEUE_DB', ZintrustLang.REDIS_DEFAULT_DB),
|
|
828
1204
|
password:
|
|
829
1205
|
queueRedis?.driver === 'redis' ? (queueRedis.password ?? '') : Env.get('REDIS_PASSWORD', ''),
|
|
830
1206
|
};
|
|
@@ -837,7 +1213,9 @@ const resolveRedisConfigFromEnv = (config: RedisEnvConfig, context: string): Red
|
|
|
837
1213
|
context
|
|
838
1214
|
);
|
|
839
1215
|
const port = resolveEnvInt(String(config.port ?? 'REDIS_PORT'), fallback.port);
|
|
840
|
-
|
|
1216
|
+
|
|
1217
|
+
const db = resolveEnvInt(config.db ?? 'REDIS_QUEUE_DB', fallback.db);
|
|
1218
|
+
|
|
841
1219
|
const password = resolveEnvString(config.password ?? 'REDIS_PASSWORD', fallback.password);
|
|
842
1220
|
|
|
843
1221
|
return {
|
|
@@ -1117,8 +1495,9 @@ const ensureWorkerStoreConfigured = async (): Promise<void> => {
|
|
|
1117
1495
|
|
|
1118
1496
|
const buildWorkerRecord = (config: WorkerFactoryConfig, status: string): WorkerRecord => {
|
|
1119
1497
|
const now = new Date();
|
|
1120
|
-
|
|
1121
|
-
|
|
1498
|
+
|
|
1499
|
+
const normalizedProcessorSpec = config.processorSpec
|
|
1500
|
+
? normalizeProcessorSpec(config.processorSpec)
|
|
1122
1501
|
: null;
|
|
1123
1502
|
return {
|
|
1124
1503
|
name: config.name,
|
|
@@ -1128,7 +1507,8 @@ const buildWorkerRecord = (config: WorkerFactoryConfig, status: string): WorkerR
|
|
|
1128
1507
|
autoStart: resolveAutoStart(config),
|
|
1129
1508
|
concurrency: config.options?.concurrency ?? 1,
|
|
1130
1509
|
region: config.datacenter?.primaryRegion ?? null,
|
|
1131
|
-
|
|
1510
|
+
processorSpec: normalizedProcessorSpec ?? null,
|
|
1511
|
+
activeStatus: config.activeStatus ?? true,
|
|
1132
1512
|
features: config.features ? { ...config.features } : null,
|
|
1133
1513
|
infrastructure: config.infrastructure ? { ...config.infrastructure } : null,
|
|
1134
1514
|
datacenter: config.datacenter ? { ...config.datacenter } : null,
|
|
@@ -1473,6 +1853,7 @@ const registerWorkerInstance = (params: {
|
|
|
1473
1853
|
WorkerRegistry.register({
|
|
1474
1854
|
name: config.name,
|
|
1475
1855
|
config: {},
|
|
1856
|
+
activeStatus: config.activeStatus ?? true,
|
|
1476
1857
|
version: workerVersion,
|
|
1477
1858
|
region: config.datacenter?.primaryRegion,
|
|
1478
1859
|
queues: [queueName],
|
|
@@ -1486,6 +1867,7 @@ const registerWorkerInstance = (params: {
|
|
|
1486
1867
|
region: config.datacenter?.primaryRegion ?? 'unknown',
|
|
1487
1868
|
queueName,
|
|
1488
1869
|
concurrency: options?.concurrency ?? 1,
|
|
1870
|
+
activeStatus: config.activeStatus ?? true,
|
|
1489
1871
|
startedAt: new Date(),
|
|
1490
1872
|
stoppedAt: null,
|
|
1491
1873
|
lastProcessedAt: null,
|
|
@@ -1548,7 +1930,9 @@ export const WorkerFactory = Object.freeze({
|
|
|
1548
1930
|
registerProcessors,
|
|
1549
1931
|
registerProcessorPaths,
|
|
1550
1932
|
registerProcessorResolver,
|
|
1933
|
+
registerProcessorSpec,
|
|
1551
1934
|
resolveProcessorPath,
|
|
1935
|
+
resolveProcessorSpec,
|
|
1552
1936
|
|
|
1553
1937
|
/**
|
|
1554
1938
|
* Create new worker with full setup
|
|
@@ -1638,13 +2022,6 @@ export const WorkerFactory = Object.freeze({
|
|
|
1638
2022
|
// Start health monitoring for the worker
|
|
1639
2023
|
startHealthMonitoring(name, worker, queueName);
|
|
1640
2024
|
|
|
1641
|
-
Logger.info(`Worker created: ${name}@${workerVersion}`, {
|
|
1642
|
-
queueName,
|
|
1643
|
-
features: Object.keys(features ?? {}).filter(
|
|
1644
|
-
(k) => features?.[k as keyof typeof features] === true
|
|
1645
|
-
),
|
|
1646
|
-
});
|
|
1647
|
-
|
|
1648
2025
|
return worker;
|
|
1649
2026
|
} catch (error) {
|
|
1650
2027
|
// Handle failure - update status to "failed"
|
|
@@ -1702,13 +2079,20 @@ export const WorkerFactory = Object.freeze({
|
|
|
1702
2079
|
/**
|
|
1703
2080
|
* Stop worker
|
|
1704
2081
|
*/
|
|
1705
|
-
async stop(
|
|
2082
|
+
async stop(
|
|
2083
|
+
name: string,
|
|
2084
|
+
persistenceOverride?: WorkerPersistenceConfig,
|
|
2085
|
+
options?: { skipPersistedUpdate?: boolean }
|
|
2086
|
+
): Promise<void> {
|
|
2087
|
+
const skipPersistedUpdate = options?.skipPersistedUpdate === true;
|
|
1706
2088
|
const instance = workers.get(name);
|
|
1707
2089
|
const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
|
|
1708
2090
|
|
|
1709
2091
|
if (!instance) {
|
|
1710
|
-
|
|
1711
|
-
|
|
2092
|
+
if (!skipPersistedUpdate) {
|
|
2093
|
+
await store.update(name, { status: 'stopped', updatedAt: new Date() });
|
|
2094
|
+
Logger.info(`Worker marked stopped (not running): ${name}`);
|
|
2095
|
+
}
|
|
1712
2096
|
return;
|
|
1713
2097
|
}
|
|
1714
2098
|
|
|
@@ -1747,14 +2131,16 @@ export const WorkerFactory = Object.freeze({
|
|
|
1747
2131
|
// Stop health monitoring for this worker
|
|
1748
2132
|
HealthMonitor.unregister(name);
|
|
1749
2133
|
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
2134
|
+
if (!skipPersistedUpdate) {
|
|
2135
|
+
try {
|
|
2136
|
+
await store.update(name, {
|
|
2137
|
+
status: WorkerCreationStatus.STOPPED,
|
|
2138
|
+
updatedAt: new Date(),
|
|
2139
|
+
});
|
|
2140
|
+
Logger.info(`Worker "${name}" status updated to stopped`);
|
|
2141
|
+
} catch (error) {
|
|
2142
|
+
Logger.error(`Failed to update worker "${name}" status`, error as Error);
|
|
2143
|
+
}
|
|
1758
2144
|
}
|
|
1759
2145
|
|
|
1760
2146
|
await WorkerRegistry.stop(name);
|
|
@@ -1870,6 +2256,47 @@ export const WorkerFactory = Object.freeze({
|
|
|
1870
2256
|
await WorkerFactory.startFromPersisted(name, persistenceOverride);
|
|
1871
2257
|
},
|
|
1872
2258
|
|
|
2259
|
+
/**
|
|
2260
|
+
* Update active status for a worker
|
|
2261
|
+
*/
|
|
2262
|
+
async setWorkerActiveStatus(
|
|
2263
|
+
name: string,
|
|
2264
|
+
activeStatus: boolean,
|
|
2265
|
+
persistenceOverride?: WorkerPersistenceConfig
|
|
2266
|
+
): Promise<void> {
|
|
2267
|
+
const instance = workers.get(name);
|
|
2268
|
+
const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
|
|
2269
|
+
|
|
2270
|
+
if (instance) {
|
|
2271
|
+
instance.config.activeStatus = activeStatus;
|
|
2272
|
+
}
|
|
2273
|
+
|
|
2274
|
+
await store.update(name, { activeStatus, updatedAt: new Date() });
|
|
2275
|
+
WorkerRegistry.setActiveStatus(name, activeStatus);
|
|
2276
|
+
|
|
2277
|
+
if (activeStatus === false && instance) {
|
|
2278
|
+
await WorkerFactory.stop(name, persistenceOverride);
|
|
2279
|
+
}
|
|
2280
|
+
},
|
|
2281
|
+
|
|
2282
|
+
/**
|
|
2283
|
+
* Get active status for a worker
|
|
2284
|
+
*/
|
|
2285
|
+
async getWorkerActiveStatus(
|
|
2286
|
+
name: string,
|
|
2287
|
+
persistenceOverride?: WorkerPersistenceConfig
|
|
2288
|
+
): Promise<boolean | null> {
|
|
2289
|
+
const instance = workers.get(name);
|
|
2290
|
+
if (instance?.config.activeStatus !== undefined) {
|
|
2291
|
+
return instance.config.activeStatus;
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
const store = await getStoreForWorker(instance?.config, persistenceOverride);
|
|
2295
|
+
const record = await store.get(name);
|
|
2296
|
+
if (!record) return null;
|
|
2297
|
+
return record.activeStatus ?? true;
|
|
2298
|
+
},
|
|
2299
|
+
|
|
1873
2300
|
/**
|
|
1874
2301
|
* Update persisted worker record and in-memory config if running.
|
|
1875
2302
|
*/
|
|
@@ -1906,6 +2333,8 @@ export const WorkerFactory = Object.freeze({
|
|
|
1906
2333
|
...cfg.options,
|
|
1907
2334
|
concurrency: merged.concurrency ?? cfg.options?.concurrency,
|
|
1908
2335
|
},
|
|
2336
|
+
processorSpec: merged.processorSpec ?? cfg.processorSpec,
|
|
2337
|
+
activeStatus: merged.activeStatus ?? cfg.activeStatus,
|
|
1909
2338
|
infrastructure: (merged.infrastructure as unknown) ?? cfg.infrastructure,
|
|
1910
2339
|
features: (merged.features as unknown) ?? cfg.features,
|
|
1911
2340
|
datacenter: (merged.datacenter as unknown) ?? cfg.datacenter,
|
|
@@ -1925,6 +2354,15 @@ export const WorkerFactory = Object.freeze({
|
|
|
1925
2354
|
throw ErrorFactory.createNotFoundError(`Worker "${name}" not found`);
|
|
1926
2355
|
}
|
|
1927
2356
|
|
|
2357
|
+
if (instance.config.activeStatus === false) {
|
|
2358
|
+
throw ErrorFactory.createConfigError(`Worker "${name}" is inactive`);
|
|
2359
|
+
}
|
|
2360
|
+
|
|
2361
|
+
const persisted = await store.get(name);
|
|
2362
|
+
if (persisted?.activeStatus === false) {
|
|
2363
|
+
throw ErrorFactory.createConfigError(`Worker "${name}" is inactive`);
|
|
2364
|
+
}
|
|
2365
|
+
|
|
1928
2366
|
const version = instance.config.version ?? '1.0.0';
|
|
1929
2367
|
await WorkerRegistry.start(name, version);
|
|
1930
2368
|
|
|
@@ -1948,7 +2386,7 @@ export const WorkerFactory = Object.freeze({
|
|
|
1948
2386
|
*/
|
|
1949
2387
|
async listPersisted(
|
|
1950
2388
|
persistenceOverride?: WorkerPersistenceConfig,
|
|
1951
|
-
options?: { offset?: number; limit?: number; search?: string }
|
|
2389
|
+
options?: { offset?: number; limit?: number; search?: string; includeInactive?: boolean }
|
|
1952
2390
|
): Promise<string[]> {
|
|
1953
2391
|
const records = await WorkerFactory.listPersistedRecords(persistenceOverride, options);
|
|
1954
2392
|
return records.map((record) => record.name);
|
|
@@ -1956,15 +2394,18 @@ export const WorkerFactory = Object.freeze({
|
|
|
1956
2394
|
|
|
1957
2395
|
async listPersistedRecords(
|
|
1958
2396
|
persistenceOverride?: WorkerPersistenceConfig,
|
|
1959
|
-
options?: { offset?: number; limit?: number; search?: string }
|
|
2397
|
+
options?: { offset?: number; limit?: number; search?: string; includeInactive?: boolean }
|
|
1960
2398
|
): Promise<WorkerRecord[]> {
|
|
2399
|
+
const includeInactive = options?.includeInactive === true;
|
|
1961
2400
|
if (!persistenceOverride) {
|
|
1962
2401
|
await ensureWorkerStoreConfigured();
|
|
1963
|
-
|
|
2402
|
+
const records = await workerStore.list(options);
|
|
2403
|
+
return includeInactive ? records : records.filter((record) => record.activeStatus !== false);
|
|
1964
2404
|
}
|
|
1965
2405
|
|
|
1966
2406
|
const store = await resolveWorkerStoreForPersistence(persistenceOverride);
|
|
1967
|
-
|
|
2407
|
+
const records = await store.list(options);
|
|
2408
|
+
return includeInactive ? records : records.filter((record) => record.activeStatus !== false);
|
|
1968
2409
|
},
|
|
1969
2410
|
|
|
1970
2411
|
/**
|
|
@@ -1979,11 +2420,16 @@ export const WorkerFactory = Object.freeze({
|
|
|
1979
2420
|
throw ErrorFactory.createNotFoundError(`Worker "${name}" not found in persistence store`);
|
|
1980
2421
|
}
|
|
1981
2422
|
|
|
2423
|
+
if (record.activeStatus === false) {
|
|
2424
|
+
throw ErrorFactory.createConfigError(`Worker "${name}" is inactive`);
|
|
2425
|
+
}
|
|
2426
|
+
|
|
1982
2427
|
let processor = await resolveProcessor(name);
|
|
1983
2428
|
|
|
1984
|
-
|
|
2429
|
+
const spec = record.processorSpec ?? undefined;
|
|
2430
|
+
if (!processor && spec) {
|
|
1985
2431
|
try {
|
|
1986
|
-
processor = await
|
|
2432
|
+
processor = await resolveProcessorSpec(spec);
|
|
1987
2433
|
} catch (error) {
|
|
1988
2434
|
Logger.error(`Failed to resolve processor module for "${name}"`, error);
|
|
1989
2435
|
}
|
|
@@ -1991,7 +2437,7 @@ export const WorkerFactory = Object.freeze({
|
|
|
1991
2437
|
|
|
1992
2438
|
if (!processor) {
|
|
1993
2439
|
throw ErrorFactory.createConfigError(
|
|
1994
|
-
`Worker "${name}" processor is not registered or resolvable. Register the processor at startup or persist a
|
|
2440
|
+
`Worker "${name}" processor is not registered or resolvable. Register the processor at startup or persist a processorSpec.`
|
|
1995
2441
|
);
|
|
1996
2442
|
}
|
|
1997
2443
|
|
|
@@ -2000,7 +2446,8 @@ export const WorkerFactory = Object.freeze({
|
|
|
2000
2446
|
queueName: record.queueName,
|
|
2001
2447
|
version: record.version ?? undefined,
|
|
2002
2448
|
processor,
|
|
2003
|
-
|
|
2449
|
+
processorSpec: record.processorSpec ?? undefined,
|
|
2450
|
+
activeStatus: record.activeStatus ?? true,
|
|
2004
2451
|
autoStart: true, // Override to true when manually starting
|
|
2005
2452
|
options: { concurrency: record.concurrency } as WorkerOptions,
|
|
2006
2453
|
infrastructure: record.infrastructure as WorkerFactoryConfig['infrastructure'],
|
|
@@ -2104,9 +2551,47 @@ export const WorkerFactory = Object.freeze({
|
|
|
2104
2551
|
async shutdown(): Promise<void> {
|
|
2105
2552
|
Logger.info('WorkerFactory shutting down...');
|
|
2106
2553
|
|
|
2107
|
-
const
|
|
2554
|
+
const workerEntries = Array.from(workers.entries());
|
|
2555
|
+
const workerNames = workerEntries.map(([name]) => name);
|
|
2108
2556
|
|
|
2109
|
-
|
|
2557
|
+
// Bulk-update persisted statuses before stopping workers to avoid per-worker DB updates
|
|
2558
|
+
// during shutdown (which can fail if DB connections are closing).
|
|
2559
|
+
const storeGroups = new Map<WorkerStore, string[]>();
|
|
2560
|
+
|
|
2561
|
+
// Parallel get stores for all workers
|
|
2562
|
+
const storePromises = workerEntries.map(async ([name, instance]) => {
|
|
2563
|
+
const store = await getStoreForWorker(instance.config);
|
|
2564
|
+
return { name, store };
|
|
2565
|
+
});
|
|
2566
|
+
|
|
2567
|
+
const storeMappings = await Promise.all(storePromises);
|
|
2568
|
+
|
|
2569
|
+
for (const { name, store } of storeMappings) {
|
|
2570
|
+
const existing = storeGroups.get(store);
|
|
2571
|
+
if (existing) {
|
|
2572
|
+
existing.push(name);
|
|
2573
|
+
} else {
|
|
2574
|
+
storeGroups.set(store, [name]);
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
|
|
2578
|
+
// Parallel bulk updates for all store groups
|
|
2579
|
+
const updatePromises = Array.from(storeGroups.entries()).map(async ([store, names]) => {
|
|
2580
|
+
if (typeof store.updateMany === 'function') {
|
|
2581
|
+
await store.updateMany(names, {
|
|
2582
|
+
status: WorkerCreationStatus.STOPPED,
|
|
2583
|
+
updatedAt: new Date(),
|
|
2584
|
+
});
|
|
2585
|
+
}
|
|
2586
|
+
});
|
|
2587
|
+
|
|
2588
|
+
await Promise.all(updatePromises);
|
|
2589
|
+
|
|
2590
|
+
await Promise.all(
|
|
2591
|
+
workerNames.map(async (name) =>
|
|
2592
|
+
WorkerFactory.stop(name, undefined, { skipPersistedUpdate: true })
|
|
2593
|
+
)
|
|
2594
|
+
);
|
|
2110
2595
|
|
|
2111
2596
|
// Shutdown all modules
|
|
2112
2597
|
ResourceMonitor.stop();
|
|
@@ -2129,6 +2614,17 @@ export const WorkerFactory = Object.freeze({
|
|
|
2129
2614
|
|
|
2130
2615
|
Logger.info('WorkerFactory shutdown complete');
|
|
2131
2616
|
},
|
|
2617
|
+
|
|
2618
|
+
/**
|
|
2619
|
+
* Reset persistence connection state.
|
|
2620
|
+
* Useful when connections become stale in long-running processes or serverless environments.
|
|
2621
|
+
*/
|
|
2622
|
+
async resetPersistence(): Promise<void> {
|
|
2623
|
+
workerStoreConfigured = false;
|
|
2624
|
+
workerStore = InMemoryWorkerStore.create();
|
|
2625
|
+
storeInstanceCache.clear();
|
|
2626
|
+
Logger.info('Worker persistence configuration reset');
|
|
2627
|
+
},
|
|
2132
2628
|
});
|
|
2133
2629
|
|
|
2134
2630
|
// Graceful shutdown handled by WorkerShutdown
|