@zintrust/workers 0.4.4 → 0.4.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/dist/BroadcastWorker.d.ts +5 -0
- package/dist/NotificationWorker.d.ts +5 -0
- package/dist/WorkerFactory.d.ts +8 -0
- package/dist/WorkerFactory.js +334 -32
- package/dist/WorkerInit.d.ts +17 -0
- package/dist/WorkerInit.js +54 -2
- package/dist/WorkerShutdownDurableObject.d.ts +12 -0
- package/dist/WorkerShutdownDurableObject.js +41 -0
- package/dist/build-manifest.json +557 -0
- package/dist/createQueueWorker.d.ts +5 -0
- package/dist/createQueueWorker.js +26 -10
- package/dist/dashboard/workers-api.js +46 -8
- package/dist/index.d.ts +3 -2
- package/dist/index.js +1 -1
- package/dist/register.d.ts +3 -3
- package/dist/register.js +10 -4
- package/dist/ui/workers/index.html +202 -0
- package/dist/ui/workers/main.js +1952 -0
- package/dist/ui/workers/styles.css +1350 -0
- package/dist/ui/workers/zintrust.svg +30 -0
- package/package.json +11 -4
- package/src/WorkerFactory.ts +483 -36
- package/src/WorkerInit.ts +81 -3
- package/src/createQueueWorker.ts +44 -14
- package/src/dashboard/workers-api.ts +60 -13
- package/src/index.ts +9 -3
- package/src/register.ts +13 -8
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
<svg width="120" height="120" viewBox="0 0 100 100" fill="none" xmlns="http://www.w3.org/2000/svg">
|
|
2
|
+
<defs>
|
|
3
|
+
<linearGradient id="zt-g2d" x1="10" y1="50" x2="90" y2="50" gradientUnits="userSpaceOnUse">
|
|
4
|
+
<stop stop-color="#22c55e" />
|
|
5
|
+
<stop offset="1" stop-color="#38bdf8" />
|
|
6
|
+
</linearGradient>
|
|
7
|
+
</defs>
|
|
8
|
+
<circle cx="50" cy="50" r="34" stroke="rgba(255,255,255,0.16)" stroke-width="4" />
|
|
9
|
+
<ellipse cx="50" cy="50" rx="40" ry="18" stroke="url(#zt-g2d)" stroke-width="4" />
|
|
10
|
+
<ellipse cx="50" cy="50" rx="18" ry="40" stroke="url(#zt-g2d)" stroke-width="4" opacity="0.75" />
|
|
11
|
+
<circle cx="50" cy="50" r="6" fill="url(#zt-g2d)" />
|
|
12
|
+
<path
|
|
13
|
+
d="M40 52C35 52 32 49 32 44C32 39 35 36 40 36H48"
|
|
14
|
+
stroke="white"
|
|
15
|
+
stroke-width="6"
|
|
16
|
+
stroke-linecap="round"
|
|
17
|
+
/>
|
|
18
|
+
<path
|
|
19
|
+
d="M60 48C65 48 68 51 68 56C68 61 65 64 60 64H52"
|
|
20
|
+
stroke="white"
|
|
21
|
+
stroke-width="6"
|
|
22
|
+
stroke-linecap="round"
|
|
23
|
+
/>
|
|
24
|
+
<path
|
|
25
|
+
d="M44 50H56"
|
|
26
|
+
stroke="rgba(255,255,255,0.22)"
|
|
27
|
+
stroke-width="6"
|
|
28
|
+
stroke-linecap="round"
|
|
29
|
+
/>
|
|
30
|
+
</svg>
|
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zintrust/workers",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.34",
|
|
4
|
+
"description": "Worker orchestration and background job management for ZinTrust.",
|
|
4
5
|
"private": false,
|
|
5
6
|
"type": "module",
|
|
6
7
|
"main": "./dist/index.js",
|
|
@@ -39,7 +40,7 @@
|
|
|
39
40
|
"node": ">=20.0.0"
|
|
40
41
|
},
|
|
41
42
|
"peerDependencies": {
|
|
42
|
-
"@zintrust/core": "^0.4.
|
|
43
|
+
"@zintrust/core": "^0.4.34",
|
|
43
44
|
"@zintrust/queue-monitor": "*",
|
|
44
45
|
"@zintrust/queue-redis": "*"
|
|
45
46
|
},
|
|
@@ -54,6 +55,13 @@
|
|
|
54
55
|
"publishConfig": {
|
|
55
56
|
"access": "public"
|
|
56
57
|
},
|
|
58
|
+
"keywords": [
|
|
59
|
+
"zintrust",
|
|
60
|
+
"workers",
|
|
61
|
+
"jobs",
|
|
62
|
+
"queue",
|
|
63
|
+
"orchestration"
|
|
64
|
+
],
|
|
57
65
|
"scripts": {
|
|
58
66
|
"build": "node scripts/generate-embedded-assets.mjs && tsc -p tsconfig.json && node ../../scripts/fix-dist-esm-imports.mjs dist",
|
|
59
67
|
"prepublishOnly": "npm run build"
|
|
@@ -62,8 +70,7 @@
|
|
|
62
70
|
"@opentelemetry/api": "^1.9.0",
|
|
63
71
|
"hot-shots": "^14.2.0",
|
|
64
72
|
"ioredis": "^5.10.0",
|
|
65
|
-
"ml.js": "^0.0.1",
|
|
66
73
|
"prom-client": "^15.1.3",
|
|
67
74
|
"simple-statistics": "^7.8.9"
|
|
68
75
|
}
|
|
69
|
-
}
|
|
76
|
+
}
|
package/src/WorkerFactory.ts
CHANGED
|
@@ -12,6 +12,9 @@ import {
|
|
|
12
12
|
ErrorFactory,
|
|
13
13
|
generateUuid,
|
|
14
14
|
getBullMQSafeQueueName,
|
|
15
|
+
isFunction,
|
|
16
|
+
isNonEmptyString,
|
|
17
|
+
isObject,
|
|
15
18
|
JobStateTracker,
|
|
16
19
|
Logger,
|
|
17
20
|
NodeSingletons,
|
|
@@ -138,9 +141,6 @@ const resolvePackageSpecifierUrl = (specifier: string): string | null => {
|
|
|
138
141
|
}
|
|
139
142
|
};
|
|
140
143
|
|
|
141
|
-
const escapeRegExp = (value: string): string =>
|
|
142
|
-
value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`);
|
|
143
|
-
|
|
144
144
|
const rewriteProcessorImports = (code: string): string => {
|
|
145
145
|
const replacements: Array<{ from: string; to: string }> = [];
|
|
146
146
|
const coreUrl = resolvePackageSpecifierUrl('@zintrust/core');
|
|
@@ -152,8 +152,8 @@ const rewriteProcessorImports = (code: string): string => {
|
|
|
152
152
|
|
|
153
153
|
let updated = code;
|
|
154
154
|
for (const { from, to } of replacements) {
|
|
155
|
-
|
|
156
|
-
updated = updated.
|
|
155
|
+
updated = updated.replaceAll(`'${from}'`, `'${to}'`);
|
|
156
|
+
updated = updated.replaceAll(`"${from}"`, `"${to}"`);
|
|
157
157
|
}
|
|
158
158
|
|
|
159
159
|
return updated;
|
|
@@ -398,6 +398,56 @@ const processorRegistry = new Map<string, WorkerFactoryConfig['processor']>();
|
|
|
398
398
|
const processorPathRegistry = new Map<string, string>();
|
|
399
399
|
const processorResolvers: ProcessorResolver[] = [];
|
|
400
400
|
const processorSpecRegistry = new Map<string, WorkerFactoryConfig['processor']>();
|
|
401
|
+
const queueWorkerMetaKey = '__zintrustQueueWorkerMeta';
|
|
402
|
+
const fileWorkerDefinitionExportKeys = Object.freeze([
|
|
403
|
+
'workerDefinition',
|
|
404
|
+
'workerConfig',
|
|
405
|
+
'zintrustWorker',
|
|
406
|
+
'ZinTrustWorker',
|
|
407
|
+
'worker',
|
|
408
|
+
'defaultWorkerDefinition',
|
|
409
|
+
]);
|
|
410
|
+
const workerDiscoveryDirectories = Object.freeze([
|
|
411
|
+
['dist', 'app', 'Workers'],
|
|
412
|
+
['dist', 'src', 'workers'],
|
|
413
|
+
['dist', 'src', 'Workers'],
|
|
414
|
+
['app', 'Workers'],
|
|
415
|
+
['src', 'workers'],
|
|
416
|
+
['src', 'Workers'],
|
|
417
|
+
]);
|
|
418
|
+
const workerDiscoveryExtensions = new Set(['.js', '.mjs', '.cjs', '.ts']);
|
|
419
|
+
|
|
420
|
+
type QueueWorkerMeta = Readonly<{
|
|
421
|
+
kindLabel: string;
|
|
422
|
+
defaultQueueName: string;
|
|
423
|
+
maxAttempts: number;
|
|
424
|
+
}>;
|
|
425
|
+
|
|
426
|
+
type FileWorkerDefinition = Partial<
|
|
427
|
+
Pick<
|
|
428
|
+
WorkerRecord,
|
|
429
|
+
| 'name'
|
|
430
|
+
| 'queueName'
|
|
431
|
+
| 'version'
|
|
432
|
+
| 'status'
|
|
433
|
+
| 'autoStart'
|
|
434
|
+
| 'concurrency'
|
|
435
|
+
| 'region'
|
|
436
|
+
| 'processorSpec'
|
|
437
|
+
| 'activeStatus'
|
|
438
|
+
| 'features'
|
|
439
|
+
| 'infrastructure'
|
|
440
|
+
| 'datacenter'
|
|
441
|
+
>
|
|
442
|
+
> & {
|
|
443
|
+
processor?: WorkerFactoryConfig['processor'];
|
|
444
|
+
};
|
|
445
|
+
|
|
446
|
+
type DiscoveredFileWorker = {
|
|
447
|
+
record: WorkerRecord;
|
|
448
|
+
processor?: WorkerFactoryConfig['processor'];
|
|
449
|
+
sourcePath: string;
|
|
450
|
+
};
|
|
401
451
|
|
|
402
452
|
type CachedProcessor = {
|
|
403
453
|
code: string;
|
|
@@ -493,6 +543,411 @@ const parseCacheControl = (value: string | null): { maxAge?: number } => {
|
|
|
493
543
|
const getProcessorSpecConfig = (): typeof workersConfig.processorSpec =>
|
|
494
544
|
workersConfig.processorSpec;
|
|
495
545
|
|
|
546
|
+
const toPosixPath = (value: string): string => value.split(path.sep).join('/');
|
|
547
|
+
|
|
548
|
+
const isUpperAlpha = (value: string): boolean => /^[A-Z]$/.test(value);
|
|
549
|
+
|
|
550
|
+
const isLowerAlphaOrDigit = (value: string): boolean => /^[a-z\d]$/.test(value);
|
|
551
|
+
|
|
552
|
+
const isAlphaNumeric = (value: string): boolean => /^[A-Za-z\d]$/.test(value);
|
|
553
|
+
|
|
554
|
+
const shouldInsertWorkerNameDash = (
|
|
555
|
+
previous: string,
|
|
556
|
+
current: string,
|
|
557
|
+
next: string | undefined
|
|
558
|
+
): boolean => {
|
|
559
|
+
if (!isAlphaNumeric(previous) || !isAlphaNumeric(current)) return false;
|
|
560
|
+
|
|
561
|
+
if (isLowerAlphaOrDigit(previous) && isUpperAlpha(current)) {
|
|
562
|
+
return true;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
if (isUpperAlpha(previous) && isUpperAlpha(current) && isLowerAlphaOrDigit(next ?? '')) {
|
|
566
|
+
return true;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
return false;
|
|
570
|
+
};
|
|
571
|
+
|
|
572
|
+
const toKebabWorkerName = (value: string): string => {
|
|
573
|
+
if (!isNonEmptyString(value)) return value;
|
|
574
|
+
|
|
575
|
+
let normalized = '';
|
|
576
|
+
|
|
577
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
578
|
+
const current = value[index] ?? '';
|
|
579
|
+
const previous = index > 0 ? (value[index - 1] ?? '') : '';
|
|
580
|
+
const next = value[index + 1];
|
|
581
|
+
|
|
582
|
+
if (current === ' ' || current === '_') {
|
|
583
|
+
if (!normalized.endsWith('-')) normalized += '-';
|
|
584
|
+
continue;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
if (shouldInsertWorkerNameDash(previous, current, next) && !normalized.endsWith('-')) {
|
|
588
|
+
normalized += '-';
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
normalized += current;
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
return normalized.replaceAll(/-+/g, '-');
|
|
595
|
+
};
|
|
596
|
+
|
|
597
|
+
const normalizeWorkerFileName = (fileName: string): string => {
|
|
598
|
+
const baseName = fileName.replace(/\.[^.]+$/, '');
|
|
599
|
+
return toKebabWorkerName(baseName).toLowerCase();
|
|
600
|
+
};
|
|
601
|
+
|
|
602
|
+
const supportsWorkerFileDiscovery = (): boolean => {
|
|
603
|
+
return (
|
|
604
|
+
isNodeRuntime() &&
|
|
605
|
+
canUseProjectFileImports() &&
|
|
606
|
+
typeof NodeSingletons.fs.readdirSync === 'function' &&
|
|
607
|
+
typeof NodeSingletons.fs.statSync === 'function'
|
|
608
|
+
);
|
|
609
|
+
};
|
|
610
|
+
|
|
611
|
+
const isSupportedWorkerModuleFile = (fileName: string): boolean => {
|
|
612
|
+
if (!isNonEmptyString(fileName)) return false;
|
|
613
|
+
if (fileName.endsWith('.d.ts')) return false;
|
|
614
|
+
if (fileName.includes('.test.') || fileName.includes('.spec.')) return false;
|
|
615
|
+
return workerDiscoveryExtensions.has(path.extname(fileName));
|
|
616
|
+
};
|
|
617
|
+
|
|
618
|
+
const getWorkerDiscoveryDirectories = (): string[] => {
|
|
619
|
+
if (!supportsWorkerFileDiscovery()) return [];
|
|
620
|
+
const root = resolveProjectRoot();
|
|
621
|
+
return workerDiscoveryDirectories.map((segments) => path.join(root, ...segments));
|
|
622
|
+
};
|
|
623
|
+
|
|
624
|
+
const listWorkerDefinitionFiles = (): string[] => {
|
|
625
|
+
if (!supportsWorkerFileDiscovery()) return [];
|
|
626
|
+
|
|
627
|
+
const discovered = new Set<string>();
|
|
628
|
+
|
|
629
|
+
for (const directory of getWorkerDiscoveryDirectories()) {
|
|
630
|
+
try {
|
|
631
|
+
if (!NodeSingletons.fs.existsSync(directory)) continue;
|
|
632
|
+
const stats = NodeSingletons.fs.statSync(directory);
|
|
633
|
+
if (!stats.isDirectory()) continue;
|
|
634
|
+
|
|
635
|
+
const entries = NodeSingletons.fs.readdirSync(directory, { withFileTypes: true }) as Array<{
|
|
636
|
+
name: string;
|
|
637
|
+
isFile: () => boolean;
|
|
638
|
+
}>;
|
|
639
|
+
|
|
640
|
+
for (const entry of entries) {
|
|
641
|
+
if (!entry.isFile() || !isSupportedWorkerModuleFile(entry.name)) continue;
|
|
642
|
+
discovered.add(path.join(directory, entry.name));
|
|
643
|
+
}
|
|
644
|
+
} catch (error) {
|
|
645
|
+
Logger.debug(`Worker file discovery failed for directory: ${directory}`, error);
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
return Array.from(discovered);
|
|
650
|
+
};
|
|
651
|
+
|
|
652
|
+
const getProjectRelativeWorkerSpec = (sourcePath: string): string => {
|
|
653
|
+
const relativePath = path.relative(resolveProjectRoot(), sourcePath);
|
|
654
|
+
return isNonEmptyString(relativePath) ? toPosixPath(relativePath) : toPosixPath(sourcePath);
|
|
655
|
+
};
|
|
656
|
+
|
|
657
|
+
const isQueueWorkerMeta = (value: unknown): value is QueueWorkerMeta => {
|
|
658
|
+
return (
|
|
659
|
+
isObject(value) &&
|
|
660
|
+
isNonEmptyString(value['kindLabel']) &&
|
|
661
|
+
isNonEmptyString(value['defaultQueueName']) &&
|
|
662
|
+
typeof value['maxAttempts'] === 'number'
|
|
663
|
+
);
|
|
664
|
+
};
|
|
665
|
+
|
|
666
|
+
const getExportedQueueWorkerMeta = (mod: Record<string, unknown>): QueueWorkerMeta | undefined => {
|
|
667
|
+
for (const value of Object.values(mod)) {
|
|
668
|
+
if (!isObject(value)) continue;
|
|
669
|
+
const meta = value[queueWorkerMetaKey];
|
|
670
|
+
if (isQueueWorkerMeta(meta)) return meta;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
return undefined;
|
|
674
|
+
};
|
|
675
|
+
|
|
676
|
+
const assignStringDefinitionField = <TKey extends keyof FileWorkerDefinition>(
|
|
677
|
+
definition: FileWorkerDefinition,
|
|
678
|
+
key: TKey,
|
|
679
|
+
value: unknown
|
|
680
|
+
): void => {
|
|
681
|
+
if (isNonEmptyString(value)) {
|
|
682
|
+
definition[key] = value as FileWorkerDefinition[TKey];
|
|
683
|
+
}
|
|
684
|
+
};
|
|
685
|
+
|
|
686
|
+
const assignBooleanDefinitionField = <TKey extends keyof FileWorkerDefinition>(
|
|
687
|
+
definition: FileWorkerDefinition,
|
|
688
|
+
key: TKey,
|
|
689
|
+
value: unknown
|
|
690
|
+
): void => {
|
|
691
|
+
if (typeof value === 'boolean') {
|
|
692
|
+
definition[key] = value as FileWorkerDefinition[TKey];
|
|
693
|
+
}
|
|
694
|
+
};
|
|
695
|
+
|
|
696
|
+
const assignObjectDefinitionField = <TKey extends keyof FileWorkerDefinition>(
|
|
697
|
+
definition: FileWorkerDefinition,
|
|
698
|
+
key: TKey,
|
|
699
|
+
value: unknown
|
|
700
|
+
): void => {
|
|
701
|
+
if (isObject(value)) {
|
|
702
|
+
definition[key] = value as FileWorkerDefinition[TKey];
|
|
703
|
+
}
|
|
704
|
+
};
|
|
705
|
+
|
|
706
|
+
const assignConcurrencyDefinitionField = (
|
|
707
|
+
definition: FileWorkerDefinition,
|
|
708
|
+
value: unknown
|
|
709
|
+
): void => {
|
|
710
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
711
|
+
definition.concurrency = Math.max(1, Math.floor(value));
|
|
712
|
+
}
|
|
713
|
+
};
|
|
714
|
+
|
|
715
|
+
const assignProcessorDefinitionField = (definition: FileWorkerDefinition, value: unknown): void => {
|
|
716
|
+
if (isFunction(value)) {
|
|
717
|
+
definition.processor = value as WorkerFactoryConfig['processor'];
|
|
718
|
+
}
|
|
719
|
+
};
|
|
720
|
+
|
|
721
|
+
const normalizeFileWorkerDefinition = (value: unknown): FileWorkerDefinition | undefined => {
|
|
722
|
+
if (!isObject(value)) return undefined;
|
|
723
|
+
|
|
724
|
+
const definition: FileWorkerDefinition = {};
|
|
725
|
+
|
|
726
|
+
assignStringDefinitionField(definition, 'name', value['name']);
|
|
727
|
+
assignStringDefinitionField(definition, 'queueName', value['queueName']);
|
|
728
|
+
assignStringDefinitionField(definition, 'version', value['version']);
|
|
729
|
+
assignStringDefinitionField(definition, 'status', value['status']);
|
|
730
|
+
assignStringDefinitionField(definition, 'region', value['region']);
|
|
731
|
+
assignStringDefinitionField(definition, 'processorSpec', value['processorSpec']);
|
|
732
|
+
assignBooleanDefinitionField(definition, 'autoStart', value['autoStart']);
|
|
733
|
+
assignBooleanDefinitionField(definition, 'activeStatus', value['activeStatus']);
|
|
734
|
+
assignObjectDefinitionField(definition, 'features', value['features']);
|
|
735
|
+
assignObjectDefinitionField(definition, 'infrastructure', value['infrastructure']);
|
|
736
|
+
assignObjectDefinitionField(definition, 'datacenter', value['datacenter']);
|
|
737
|
+
assignConcurrencyDefinitionField(definition, value['concurrency']);
|
|
738
|
+
assignProcessorDefinitionField(definition, value['processor']);
|
|
739
|
+
|
|
740
|
+
return definition;
|
|
741
|
+
};
|
|
742
|
+
|
|
743
|
+
const getExportedFileWorkerDefinition = (
|
|
744
|
+
mod: Record<string, unknown>
|
|
745
|
+
): FileWorkerDefinition | undefined => {
|
|
746
|
+
for (const key of fileWorkerDefinitionExportKeys) {
|
|
747
|
+
const normalized = normalizeFileWorkerDefinition(mod[key]);
|
|
748
|
+
if (normalized) return normalized;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
const defaultExport = mod['default'];
|
|
752
|
+
if (!isFunction(defaultExport)) {
|
|
753
|
+
const normalized = normalizeFileWorkerDefinition(defaultExport);
|
|
754
|
+
if (normalized) return normalized;
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
return undefined;
|
|
758
|
+
};
|
|
759
|
+
|
|
760
|
+
const importWorkerDefinitionModule = async (
|
|
761
|
+
sourcePath: string
|
|
762
|
+
): Promise<Record<string, unknown> | undefined> => {
|
|
763
|
+
if (!supportsWorkerFileDiscovery()) return undefined;
|
|
764
|
+
|
|
765
|
+
try {
|
|
766
|
+
return (await import(NodeSingletons.url.pathToFileURL(sourcePath).href)) as Record<
|
|
767
|
+
string,
|
|
768
|
+
unknown
|
|
769
|
+
>;
|
|
770
|
+
} catch (error) {
|
|
771
|
+
Logger.debug(`Failed to import worker definition module: ${sourcePath}`, error);
|
|
772
|
+
return undefined;
|
|
773
|
+
}
|
|
774
|
+
};
|
|
775
|
+
|
|
776
|
+
const resolveDiscoveredWorkerProcessor = (
|
|
777
|
+
definition: FileWorkerDefinition | undefined,
|
|
778
|
+
mod: Record<string, unknown>,
|
|
779
|
+
sourcePath: string
|
|
780
|
+
): WorkerFactoryConfig['processor'] | undefined => {
|
|
781
|
+
if (definition?.processor) return definition.processor;
|
|
782
|
+
return extractZinTrustProcessor(mod, sourcePath) ?? pickProcessorFromModule(mod, sourcePath);
|
|
783
|
+
};
|
|
784
|
+
|
|
785
|
+
const resolveDiscoveredWorkerName = (
|
|
786
|
+
definition: FileWorkerDefinition | undefined,
|
|
787
|
+
sourcePath: string
|
|
788
|
+
): string => {
|
|
789
|
+
return (
|
|
790
|
+
definition?.name ?? normalizeWorkerFileName(path.basename(sourcePath, path.extname(sourcePath)))
|
|
791
|
+
);
|
|
792
|
+
};
|
|
793
|
+
|
|
794
|
+
const resolveDiscoveredQueueName = (
|
|
795
|
+
definition: FileWorkerDefinition | undefined,
|
|
796
|
+
queueWorkerMeta: QueueWorkerMeta | undefined,
|
|
797
|
+
recordName: string
|
|
798
|
+
): string => {
|
|
799
|
+
if (definition?.queueName) return definition.queueName;
|
|
800
|
+
if (queueWorkerMeta?.defaultQueueName) return queueWorkerMeta.defaultQueueName;
|
|
801
|
+
return `${recordName}-queue`;
|
|
802
|
+
};
|
|
803
|
+
|
|
804
|
+
const resolveDiscoveredProcessorSpec = (
|
|
805
|
+
definition: FileWorkerDefinition | undefined,
|
|
806
|
+
sourcePath: string
|
|
807
|
+
): string => {
|
|
808
|
+
return definition?.processorSpec ?? getProjectRelativeWorkerSpec(sourcePath);
|
|
809
|
+
};
|
|
810
|
+
|
|
811
|
+
const resolveDiscoveredVersion = (definition: FileWorkerDefinition | undefined): string => {
|
|
812
|
+
return definition?.version ?? '1.0.0';
|
|
813
|
+
};
|
|
814
|
+
|
|
815
|
+
const resolveDiscoveredStatus = (definition: FileWorkerDefinition | undefined): string => {
|
|
816
|
+
return definition?.status ?? WorkerCreationStatus.STOPPED;
|
|
817
|
+
};
|
|
818
|
+
|
|
819
|
+
const resolveDiscoveredAutoStart = (definition: FileWorkerDefinition | undefined): boolean => {
|
|
820
|
+
return definition?.autoStart ?? false;
|
|
821
|
+
};
|
|
822
|
+
|
|
823
|
+
const resolveDiscoveredConcurrency = (definition: FileWorkerDefinition | undefined): number => {
|
|
824
|
+
return definition?.concurrency ?? 1;
|
|
825
|
+
};
|
|
826
|
+
|
|
827
|
+
const resolveDiscoveredActiveStatus = (definition: FileWorkerDefinition | undefined): boolean => {
|
|
828
|
+
return definition?.activeStatus ?? true;
|
|
829
|
+
};
|
|
830
|
+
|
|
831
|
+
const buildDiscoveredWorkerRecord = (
|
|
832
|
+
definition: FileWorkerDefinition | undefined,
|
|
833
|
+
queueWorkerMeta: QueueWorkerMeta | undefined,
|
|
834
|
+
sourcePath: string
|
|
835
|
+
): WorkerRecord | undefined => {
|
|
836
|
+
const recordName = resolveDiscoveredWorkerName(definition, sourcePath);
|
|
837
|
+
|
|
838
|
+
if (!isNonEmptyString(recordName)) return undefined;
|
|
839
|
+
|
|
840
|
+
return {
|
|
841
|
+
name: recordName,
|
|
842
|
+
queueName: resolveDiscoveredQueueName(definition, queueWorkerMeta, recordName),
|
|
843
|
+
version: resolveDiscoveredVersion(definition),
|
|
844
|
+
status: resolveDiscoveredStatus(definition),
|
|
845
|
+
autoStart: resolveDiscoveredAutoStart(definition),
|
|
846
|
+
concurrency: resolveDiscoveredConcurrency(definition),
|
|
847
|
+
region: definition?.region ?? null,
|
|
848
|
+
processorSpec: resolveDiscoveredProcessorSpec(definition, sourcePath),
|
|
849
|
+
activeStatus: resolveDiscoveredActiveStatus(definition),
|
|
850
|
+
features: definition?.features ?? null,
|
|
851
|
+
infrastructure: definition?.infrastructure ?? null,
|
|
852
|
+
datacenter: definition?.datacenter ?? null,
|
|
853
|
+
createdAt: new Date(),
|
|
854
|
+
updatedAt: new Date(),
|
|
855
|
+
};
|
|
856
|
+
};
|
|
857
|
+
|
|
858
|
+
const buildDiscoveredWorker = (
|
|
859
|
+
mod: Record<string, unknown>,
|
|
860
|
+
sourcePath: string
|
|
861
|
+
): DiscoveredFileWorker | undefined => {
|
|
862
|
+
const definition = getExportedFileWorkerDefinition(mod);
|
|
863
|
+
const queueWorkerMeta = getExportedQueueWorkerMeta(mod);
|
|
864
|
+
const processor = resolveDiscoveredWorkerProcessor(definition, mod, sourcePath);
|
|
865
|
+
const record = buildDiscoveredWorkerRecord(definition, queueWorkerMeta, sourcePath);
|
|
866
|
+
if (!record) return undefined;
|
|
867
|
+
|
|
868
|
+
return {
|
|
869
|
+
record,
|
|
870
|
+
processor,
|
|
871
|
+
sourcePath,
|
|
872
|
+
};
|
|
873
|
+
};
|
|
874
|
+
|
|
875
|
+
const resolveStartFromPersistedRecord = async (
|
|
876
|
+
name: string,
|
|
877
|
+
persistenceOverride?: WorkerPersistenceConfig
|
|
878
|
+
): Promise<{ record: WorkerRecord; discovered: DiscoveredFileWorker | null }> => {
|
|
879
|
+
const persistedRecord = await getPersistedRecord(name, persistenceOverride);
|
|
880
|
+
const discovered = persistedRecord ? null : await getDiscoveredFileWorker(name);
|
|
881
|
+
const effectiveRecord = persistedRecord ?? discovered?.record ?? null;
|
|
882
|
+
|
|
883
|
+
if (!effectiveRecord) {
|
|
884
|
+
throw ErrorFactory.createNotFoundError(`Worker "${name}" not found in persistence store`);
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
return { record: effectiveRecord, discovered };
|
|
888
|
+
};
|
|
889
|
+
|
|
890
|
+
const resolveStartFromPersistedProcessor = async (
|
|
891
|
+
name: string,
|
|
892
|
+
record: WorkerRecord,
|
|
893
|
+
discovered: DiscoveredFileWorker | null
|
|
894
|
+
): Promise<WorkerFactoryConfig['processor']> => {
|
|
895
|
+
let processor = await resolveProcessor(name);
|
|
896
|
+
|
|
897
|
+
if (!processor && discovered?.processor) {
|
|
898
|
+
processor = discovered.processor;
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
const spec = record.processorSpec ?? undefined;
|
|
902
|
+
if (!processor && spec) {
|
|
903
|
+
try {
|
|
904
|
+
processor = await resolveProcessorSpec(spec);
|
|
905
|
+
} catch (error) {
|
|
906
|
+
Logger.error(`Failed to resolve processor module for "${name}"`, error);
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
if (!processor) {
|
|
911
|
+
throw ErrorFactory.createConfigError(
|
|
912
|
+
`Worker "${name}" processor is not registered or resolvable. Register the processor at startup or persist a processorSpec.`
|
|
913
|
+
);
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
return processor;
|
|
917
|
+
};
|
|
918
|
+
|
|
919
|
+
const discoverFileBackedWorkers = async (): Promise<DiscoveredFileWorker[]> => {
|
|
920
|
+
const files = listWorkerDefinitionFiles();
|
|
921
|
+
if (files.length === 0) return [];
|
|
922
|
+
|
|
923
|
+
const discovered = new Map<string, DiscoveredFileWorker>();
|
|
924
|
+
|
|
925
|
+
for (const filePath of files) {
|
|
926
|
+
// eslint-disable-next-line no-await-in-loop
|
|
927
|
+
const mod = await importWorkerDefinitionModule(filePath);
|
|
928
|
+
if (!mod) continue;
|
|
929
|
+
|
|
930
|
+
const discoveredWorker = buildDiscoveredWorker(mod, filePath);
|
|
931
|
+
if (!discoveredWorker) continue;
|
|
932
|
+
|
|
933
|
+
if (discovered.has(discoveredWorker.record.name)) {
|
|
934
|
+
Logger.warn(
|
|
935
|
+
`Duplicate file-backed worker definition detected for "${discoveredWorker.record.name}". Keeping the first discovered module.`
|
|
936
|
+
);
|
|
937
|
+
continue;
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
discovered.set(discoveredWorker.record.name, discoveredWorker);
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
return Array.from(discovered.values());
|
|
944
|
+
};
|
|
945
|
+
|
|
946
|
+
const getDiscoveredFileWorker = async (name: string): Promise<DiscoveredFileWorker | null> => {
|
|
947
|
+
const discovered = await discoverFileBackedWorkers();
|
|
948
|
+
return discovered.find((entry) => entry.record.name === name) ?? null;
|
|
949
|
+
};
|
|
950
|
+
|
|
496
951
|
const computeSha256 = async (value: string): Promise<string> => {
|
|
497
952
|
if (typeof globalThis !== 'undefined' && globalThis.crypto?.subtle) {
|
|
498
953
|
const data = new TextEncoder().encode(value);
|
|
@@ -579,7 +1034,7 @@ const waitForWorkerConnection = async (
|
|
|
579
1034
|
const checkConnection = async (): Promise<void> => {
|
|
580
1035
|
try {
|
|
581
1036
|
// Check if worker is actually running
|
|
582
|
-
const isRunning = await worker.isRunning();
|
|
1037
|
+
const isRunning = await worker.isRunning(); // NOSONAR - BullMQ's isRunning method
|
|
583
1038
|
if (!isRunning) {
|
|
584
1039
|
throw ErrorFactory.createWorkerError('Worker not running');
|
|
585
1040
|
}
|
|
@@ -778,7 +1233,7 @@ const cacheProcessorFromResponse = async (params: {
|
|
|
778
1233
|
const rawCode = await readResponseBody(response, config.fetchMaxSizeBytes);
|
|
779
1234
|
const code = rewriteProcessorImports(rawCode);
|
|
780
1235
|
const mod = await importModuleFromCode({ code, normalized, cacheKey });
|
|
781
|
-
const processor = extractZinTrustProcessor(mod
|
|
1236
|
+
const processor = extractZinTrustProcessor(mod, normalized);
|
|
782
1237
|
if (!processor) {
|
|
783
1238
|
throw ErrorFactory.createConfigError('INVALID_PROCESSOR_URL_EXPORT');
|
|
784
1239
|
}
|
|
@@ -1574,15 +2029,13 @@ const resolveRedisConfigFromDirect = (config: RedisConfig, context: string): Red
|
|
|
1574
2029
|
let normalizedDb = fallbackDb;
|
|
1575
2030
|
if (typeof config.db === 'number') {
|
|
1576
2031
|
normalizedDb = config.db;
|
|
1577
|
-
} else if (typeof (config as { database?: number }).database === 'number') {
|
|
1578
|
-
normalizedDb = (config as { database?: number }).database as number;
|
|
1579
2032
|
}
|
|
1580
2033
|
|
|
1581
2034
|
return {
|
|
1582
2035
|
host: requireRedisHost(config.host, context),
|
|
1583
2036
|
port: config.port,
|
|
1584
2037
|
db: normalizedDb,
|
|
1585
|
-
password: config.password ?? Env.get('REDIS_PASSWORD'
|
|
2038
|
+
password: config.password ?? Env.get('REDIS_PASSWORD'),
|
|
1586
2039
|
};
|
|
1587
2040
|
};
|
|
1588
2041
|
|
|
@@ -1970,13 +2423,9 @@ const resolveAutoScalerConfig = (input: AutoScalerConfig | undefined): AutoScale
|
|
|
1970
2423
|
const resolveWorkerOptions = (config: WorkerFactoryConfig, autoStart: boolean): WorkerOptions => {
|
|
1971
2424
|
const options = config.options ? { ...config.options } : ({} as WorkerOptions);
|
|
1972
2425
|
|
|
1973
|
-
|
|
1974
|
-
options.prefix = getBullMQSafeQueueName();
|
|
1975
|
-
}
|
|
2426
|
+
options.prefix ??= getBullMQSafeQueueName();
|
|
1976
2427
|
|
|
1977
|
-
|
|
1978
|
-
options.autorun = autoStart;
|
|
1979
|
-
}
|
|
2428
|
+
options.autorun ??= autoStart;
|
|
1980
2429
|
if (options.connection) return options;
|
|
1981
2430
|
|
|
1982
2431
|
const redisConfig = resolveRedisConfigWithFallback(
|
|
@@ -2844,31 +3293,13 @@ export const WorkerFactory = Object.freeze({
|
|
|
2844
3293
|
name: string,
|
|
2845
3294
|
persistenceOverride?: WorkerPersistenceConfig
|
|
2846
3295
|
): Promise<void> {
|
|
2847
|
-
const record = await
|
|
2848
|
-
if (!record) {
|
|
2849
|
-
throw ErrorFactory.createNotFoundError(`Worker "${name}" not found in persistence store`);
|
|
2850
|
-
}
|
|
3296
|
+
const { record, discovered } = await resolveStartFromPersistedRecord(name, persistenceOverride);
|
|
2851
3297
|
|
|
2852
3298
|
if (record.activeStatus === false) {
|
|
2853
3299
|
throw ErrorFactory.createConfigError(`Worker "${name}" is inactive`);
|
|
2854
3300
|
}
|
|
2855
3301
|
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
const spec = record.processorSpec ?? undefined;
|
|
2859
|
-
if (!processor && spec) {
|
|
2860
|
-
try {
|
|
2861
|
-
processor = await resolveProcessorSpec(spec);
|
|
2862
|
-
} catch (error) {
|
|
2863
|
-
Logger.error(`Failed to resolve processor module for "${name}"`, error);
|
|
2864
|
-
}
|
|
2865
|
-
}
|
|
2866
|
-
|
|
2867
|
-
if (!processor) {
|
|
2868
|
-
throw ErrorFactory.createConfigError(
|
|
2869
|
-
`Worker "${name}" processor is not registered or resolvable. Register the processor at startup or persist a processorSpec.`
|
|
2870
|
-
);
|
|
2871
|
-
}
|
|
3302
|
+
const processor = await resolveStartFromPersistedProcessor(name, record, discovered);
|
|
2872
3303
|
|
|
2873
3304
|
await WorkerFactory.create({
|
|
2874
3305
|
name: record.name,
|
|
@@ -2885,6 +3316,22 @@ export const WorkerFactory = Object.freeze({
|
|
|
2885
3316
|
});
|
|
2886
3317
|
},
|
|
2887
3318
|
|
|
3319
|
+
/**
|
|
3320
|
+
* List worker definitions discovered from project files.
|
|
3321
|
+
*/
|
|
3322
|
+
async listFileBackedRecords(): Promise<WorkerRecord[]> {
|
|
3323
|
+
const discovered = await discoverFileBackedWorkers();
|
|
3324
|
+
return discovered.map((entry) => entry.record);
|
|
3325
|
+
},
|
|
3326
|
+
|
|
3327
|
+
/**
|
|
3328
|
+
* Get a file-backed worker definition by name.
|
|
3329
|
+
*/
|
|
3330
|
+
async getFileBackedRecord(name: string): Promise<WorkerRecord | null> {
|
|
3331
|
+
const discovered = await getDiscoveredFileWorker(name);
|
|
3332
|
+
return discovered?.record ?? null;
|
|
3333
|
+
},
|
|
3334
|
+
|
|
2888
3335
|
/**
|
|
2889
3336
|
* Get persisted worker record
|
|
2890
3337
|
*/
|