@zintrust/workers 0.4.3 → 0.4.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/dist/BroadcastWorker.d.ts +5 -0
- package/dist/NotificationWorker.d.ts +5 -0
- package/dist/WorkerFactory.d.ts +8 -0
- package/dist/WorkerFactory.js +305 -27
- package/dist/WorkerInit.d.ts +17 -0
- package/dist/WorkerInit.js +54 -2
- package/dist/build-manifest.json +12 -12
- package/dist/createQueueWorker.d.ts +5 -0
- package/dist/createQueueWorker.js +13 -2
- package/dist/dashboard/workers-api.js +46 -8
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -13
- package/dist/register.d.ts +4 -1
- package/dist/register.js +10 -4
- package/package.json +14 -3
- package/src/WorkerFactory.ts +439 -30
- package/src/WorkerInit.ts +81 -3
- package/src/createQueueWorker.ts +18 -2
- package/src/dashboard/workers-api.ts +60 -13
- package/src/index.ts +6 -1
- package/src/register.ts +14 -6
package/src/WorkerFactory.ts
CHANGED
|
@@ -12,6 +12,9 @@ import {
|
|
|
12
12
|
ErrorFactory,
|
|
13
13
|
generateUuid,
|
|
14
14
|
getBullMQSafeQueueName,
|
|
15
|
+
isFunction,
|
|
16
|
+
isNonEmptyString,
|
|
17
|
+
isObject,
|
|
15
18
|
JobStateTracker,
|
|
16
19
|
Logger,
|
|
17
20
|
NodeSingletons,
|
|
@@ -398,6 +401,56 @@ const processorRegistry = new Map<string, WorkerFactoryConfig['processor']>();
|
|
|
398
401
|
const processorPathRegistry = new Map<string, string>();
|
|
399
402
|
const processorResolvers: ProcessorResolver[] = [];
|
|
400
403
|
const processorSpecRegistry = new Map<string, WorkerFactoryConfig['processor']>();
|
|
404
|
+
const queueWorkerMetaKey = '__zintrustQueueWorkerMeta';
|
|
405
|
+
const fileWorkerDefinitionExportKeys = Object.freeze([
|
|
406
|
+
'workerDefinition',
|
|
407
|
+
'workerConfig',
|
|
408
|
+
'zintrustWorker',
|
|
409
|
+
'ZinTrustWorker',
|
|
410
|
+
'worker',
|
|
411
|
+
'defaultWorkerDefinition',
|
|
412
|
+
]);
|
|
413
|
+
const workerDiscoveryDirectories = Object.freeze([
|
|
414
|
+
['dist', 'app', 'Workers'],
|
|
415
|
+
['dist', 'src', 'workers'],
|
|
416
|
+
['dist', 'src', 'Workers'],
|
|
417
|
+
['app', 'Workers'],
|
|
418
|
+
['src', 'workers'],
|
|
419
|
+
['src', 'Workers'],
|
|
420
|
+
]);
|
|
421
|
+
const workerDiscoveryExtensions = new Set(['.js', '.mjs', '.cjs', '.ts']);
|
|
422
|
+
|
|
423
|
+
type QueueWorkerMeta = Readonly<{
|
|
424
|
+
kindLabel: string;
|
|
425
|
+
defaultQueueName: string;
|
|
426
|
+
maxAttempts: number;
|
|
427
|
+
}>;
|
|
428
|
+
|
|
429
|
+
type FileWorkerDefinition = Partial<
|
|
430
|
+
Pick<
|
|
431
|
+
WorkerRecord,
|
|
432
|
+
| 'name'
|
|
433
|
+
| 'queueName'
|
|
434
|
+
| 'version'
|
|
435
|
+
| 'status'
|
|
436
|
+
| 'autoStart'
|
|
437
|
+
| 'concurrency'
|
|
438
|
+
| 'region'
|
|
439
|
+
| 'processorSpec'
|
|
440
|
+
| 'activeStatus'
|
|
441
|
+
| 'features'
|
|
442
|
+
| 'infrastructure'
|
|
443
|
+
| 'datacenter'
|
|
444
|
+
>
|
|
445
|
+
> & {
|
|
446
|
+
processor?: WorkerFactoryConfig['processor'];
|
|
447
|
+
};
|
|
448
|
+
|
|
449
|
+
type DiscoveredFileWorker = {
|
|
450
|
+
record: WorkerRecord;
|
|
451
|
+
processor?: WorkerFactoryConfig['processor'];
|
|
452
|
+
sourcePath: string;
|
|
453
|
+
};
|
|
401
454
|
|
|
402
455
|
type CachedProcessor = {
|
|
403
456
|
code: string;
|
|
@@ -493,6 +546,367 @@ const parseCacheControl = (value: string | null): { maxAge?: number } => {
|
|
|
493
546
|
const getProcessorSpecConfig = (): typeof workersConfig.processorSpec =>
|
|
494
547
|
workersConfig.processorSpec;
|
|
495
548
|
|
|
549
|
+
const toPosixPath = (value: string): string => value.split(path.sep).join('/');
|
|
550
|
+
|
|
551
|
+
const normalizeWorkerFileName = (fileName: string): string => {
|
|
552
|
+
const baseName = fileName.replaceAll(/\.[^.]+$/, '');
|
|
553
|
+
return baseName
|
|
554
|
+
.replaceAll(/([a-z\d])([A-Z])/g, '$1-$2')
|
|
555
|
+
.replaceAll(/([A-Z]+)([A-Z][a-z\d]+)/g, '$1-$2')
|
|
556
|
+
.replaceAll(/[\s_]+/g, '-')
|
|
557
|
+
.replaceAll(/-+/g, '-')
|
|
558
|
+
.toLowerCase();
|
|
559
|
+
};
|
|
560
|
+
|
|
561
|
+
const supportsWorkerFileDiscovery = (): boolean => {
|
|
562
|
+
return (
|
|
563
|
+
isNodeRuntime() &&
|
|
564
|
+
canUseProjectFileImports() &&
|
|
565
|
+
typeof NodeSingletons.fs.readdirSync === 'function' &&
|
|
566
|
+
typeof NodeSingletons.fs.statSync === 'function'
|
|
567
|
+
);
|
|
568
|
+
};
|
|
569
|
+
|
|
570
|
+
const isSupportedWorkerModuleFile = (fileName: string): boolean => {
|
|
571
|
+
if (!isNonEmptyString(fileName)) return false;
|
|
572
|
+
if (fileName.endsWith('.d.ts')) return false;
|
|
573
|
+
if (fileName.includes('.test.') || fileName.includes('.spec.')) return false;
|
|
574
|
+
return workerDiscoveryExtensions.has(path.extname(fileName));
|
|
575
|
+
};
|
|
576
|
+
|
|
577
|
+
const getWorkerDiscoveryDirectories = (): string[] => {
|
|
578
|
+
if (!supportsWorkerFileDiscovery()) return [];
|
|
579
|
+
const root = resolveProjectRoot();
|
|
580
|
+
return workerDiscoveryDirectories.map((segments) => path.join(root, ...segments));
|
|
581
|
+
};
|
|
582
|
+
|
|
583
|
+
const listWorkerDefinitionFiles = (): string[] => {
|
|
584
|
+
if (!supportsWorkerFileDiscovery()) return [];
|
|
585
|
+
|
|
586
|
+
const discovered = new Set<string>();
|
|
587
|
+
|
|
588
|
+
for (const directory of getWorkerDiscoveryDirectories()) {
|
|
589
|
+
try {
|
|
590
|
+
if (!NodeSingletons.fs.existsSync(directory)) continue;
|
|
591
|
+
const stats = NodeSingletons.fs.statSync(directory);
|
|
592
|
+
if (!stats.isDirectory()) continue;
|
|
593
|
+
|
|
594
|
+
const entries = NodeSingletons.fs.readdirSync(directory, { withFileTypes: true }) as Array<{
|
|
595
|
+
name: string;
|
|
596
|
+
isFile: () => boolean;
|
|
597
|
+
}>;
|
|
598
|
+
|
|
599
|
+
for (const entry of entries) {
|
|
600
|
+
if (!entry.isFile() || !isSupportedWorkerModuleFile(entry.name)) continue;
|
|
601
|
+
discovered.add(path.join(directory, entry.name));
|
|
602
|
+
}
|
|
603
|
+
} catch (error) {
|
|
604
|
+
Logger.debug(`Worker file discovery failed for directory: ${directory}`, error);
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
return Array.from(discovered);
|
|
609
|
+
};
|
|
610
|
+
|
|
611
|
+
const getProjectRelativeWorkerSpec = (sourcePath: string): string => {
|
|
612
|
+
const relativePath = path.relative(resolveProjectRoot(), sourcePath);
|
|
613
|
+
return isNonEmptyString(relativePath) ? toPosixPath(relativePath) : toPosixPath(sourcePath);
|
|
614
|
+
};
|
|
615
|
+
|
|
616
|
+
const isQueueWorkerMeta = (value: unknown): value is QueueWorkerMeta => {
|
|
617
|
+
return (
|
|
618
|
+
isObject(value) &&
|
|
619
|
+
isNonEmptyString(value['kindLabel']) &&
|
|
620
|
+
isNonEmptyString(value['defaultQueueName']) &&
|
|
621
|
+
typeof value['maxAttempts'] === 'number'
|
|
622
|
+
);
|
|
623
|
+
};
|
|
624
|
+
|
|
625
|
+
const getExportedQueueWorkerMeta = (mod: Record<string, unknown>): QueueWorkerMeta | undefined => {
|
|
626
|
+
for (const value of Object.values(mod)) {
|
|
627
|
+
if (!isObject(value)) continue;
|
|
628
|
+
const meta = value[queueWorkerMetaKey];
|
|
629
|
+
if (isQueueWorkerMeta(meta)) return meta;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
return undefined;
|
|
633
|
+
};
|
|
634
|
+
|
|
635
|
+
const assignStringDefinitionField = <TKey extends keyof FileWorkerDefinition>(
|
|
636
|
+
definition: FileWorkerDefinition,
|
|
637
|
+
key: TKey,
|
|
638
|
+
value: unknown
|
|
639
|
+
): void => {
|
|
640
|
+
if (isNonEmptyString(value)) {
|
|
641
|
+
definition[key] = value as FileWorkerDefinition[TKey];
|
|
642
|
+
}
|
|
643
|
+
};
|
|
644
|
+
|
|
645
|
+
const assignBooleanDefinitionField = <TKey extends keyof FileWorkerDefinition>(
|
|
646
|
+
definition: FileWorkerDefinition,
|
|
647
|
+
key: TKey,
|
|
648
|
+
value: unknown
|
|
649
|
+
): void => {
|
|
650
|
+
if (typeof value === 'boolean') {
|
|
651
|
+
definition[key] = value as FileWorkerDefinition[TKey];
|
|
652
|
+
}
|
|
653
|
+
};
|
|
654
|
+
|
|
655
|
+
const assignObjectDefinitionField = <TKey extends keyof FileWorkerDefinition>(
|
|
656
|
+
definition: FileWorkerDefinition,
|
|
657
|
+
key: TKey,
|
|
658
|
+
value: unknown
|
|
659
|
+
): void => {
|
|
660
|
+
if (isObject(value)) {
|
|
661
|
+
definition[key] = value as FileWorkerDefinition[TKey];
|
|
662
|
+
}
|
|
663
|
+
};
|
|
664
|
+
|
|
665
|
+
const assignConcurrencyDefinitionField = (
|
|
666
|
+
definition: FileWorkerDefinition,
|
|
667
|
+
value: unknown
|
|
668
|
+
): void => {
|
|
669
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
670
|
+
definition.concurrency = Math.max(1, Math.floor(value));
|
|
671
|
+
}
|
|
672
|
+
};
|
|
673
|
+
|
|
674
|
+
const assignProcessorDefinitionField = (definition: FileWorkerDefinition, value: unknown): void => {
|
|
675
|
+
if (isFunction(value)) {
|
|
676
|
+
definition.processor = value as WorkerFactoryConfig['processor'];
|
|
677
|
+
}
|
|
678
|
+
};
|
|
679
|
+
|
|
680
|
+
const normalizeFileWorkerDefinition = (value: unknown): FileWorkerDefinition | undefined => {
|
|
681
|
+
if (!isObject(value)) return undefined;
|
|
682
|
+
|
|
683
|
+
const definition: FileWorkerDefinition = {};
|
|
684
|
+
|
|
685
|
+
assignStringDefinitionField(definition, 'name', value['name']);
|
|
686
|
+
assignStringDefinitionField(definition, 'queueName', value['queueName']);
|
|
687
|
+
assignStringDefinitionField(definition, 'version', value['version']);
|
|
688
|
+
assignStringDefinitionField(definition, 'status', value['status']);
|
|
689
|
+
assignStringDefinitionField(definition, 'region', value['region']);
|
|
690
|
+
assignStringDefinitionField(definition, 'processorSpec', value['processorSpec']);
|
|
691
|
+
assignBooleanDefinitionField(definition, 'autoStart', value['autoStart']);
|
|
692
|
+
assignBooleanDefinitionField(definition, 'activeStatus', value['activeStatus']);
|
|
693
|
+
assignObjectDefinitionField(definition, 'features', value['features']);
|
|
694
|
+
assignObjectDefinitionField(definition, 'infrastructure', value['infrastructure']);
|
|
695
|
+
assignObjectDefinitionField(definition, 'datacenter', value['datacenter']);
|
|
696
|
+
assignConcurrencyDefinitionField(definition, value['concurrency']);
|
|
697
|
+
assignProcessorDefinitionField(definition, value['processor']);
|
|
698
|
+
|
|
699
|
+
return definition;
|
|
700
|
+
};
|
|
701
|
+
|
|
702
|
+
const getExportedFileWorkerDefinition = (
|
|
703
|
+
mod: Record<string, unknown>
|
|
704
|
+
): FileWorkerDefinition | undefined => {
|
|
705
|
+
for (const key of fileWorkerDefinitionExportKeys) {
|
|
706
|
+
const normalized = normalizeFileWorkerDefinition(mod[key]);
|
|
707
|
+
if (normalized) return normalized;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
const defaultExport = mod['default'];
|
|
711
|
+
if (!isFunction(defaultExport)) {
|
|
712
|
+
const normalized = normalizeFileWorkerDefinition(defaultExport);
|
|
713
|
+
if (normalized) return normalized;
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
return undefined;
|
|
717
|
+
};
|
|
718
|
+
|
|
719
|
+
const importWorkerDefinitionModule = async (
|
|
720
|
+
sourcePath: string
|
|
721
|
+
): Promise<Record<string, unknown> | undefined> => {
|
|
722
|
+
if (!supportsWorkerFileDiscovery()) return undefined;
|
|
723
|
+
|
|
724
|
+
try {
|
|
725
|
+
return (await import(NodeSingletons.url.pathToFileURL(sourcePath).href)) as Record<
|
|
726
|
+
string,
|
|
727
|
+
unknown
|
|
728
|
+
>;
|
|
729
|
+
} catch (error) {
|
|
730
|
+
Logger.debug(`Failed to import worker definition module: ${sourcePath}`, error);
|
|
731
|
+
return undefined;
|
|
732
|
+
}
|
|
733
|
+
};
|
|
734
|
+
|
|
735
|
+
const resolveDiscoveredWorkerProcessor = (
|
|
736
|
+
definition: FileWorkerDefinition | undefined,
|
|
737
|
+
mod: Record<string, unknown>,
|
|
738
|
+
sourcePath: string
|
|
739
|
+
): WorkerFactoryConfig['processor'] | undefined => {
|
|
740
|
+
if (definition?.processor) return definition.processor;
|
|
741
|
+
return extractZinTrustProcessor(mod, sourcePath) ?? pickProcessorFromModule(mod, sourcePath);
|
|
742
|
+
};
|
|
743
|
+
|
|
744
|
+
const resolveDiscoveredWorkerName = (
|
|
745
|
+
definition: FileWorkerDefinition | undefined,
|
|
746
|
+
sourcePath: string
|
|
747
|
+
): string => {
|
|
748
|
+
return (
|
|
749
|
+
definition?.name ?? normalizeWorkerFileName(path.basename(sourcePath, path.extname(sourcePath)))
|
|
750
|
+
);
|
|
751
|
+
};
|
|
752
|
+
|
|
753
|
+
const resolveDiscoveredQueueName = (
|
|
754
|
+
definition: FileWorkerDefinition | undefined,
|
|
755
|
+
queueWorkerMeta: QueueWorkerMeta | undefined,
|
|
756
|
+
recordName: string
|
|
757
|
+
): string => {
|
|
758
|
+
if (definition?.queueName) return definition.queueName;
|
|
759
|
+
if (queueWorkerMeta?.defaultQueueName) return queueWorkerMeta.defaultQueueName;
|
|
760
|
+
return `${recordName}-queue`;
|
|
761
|
+
};
|
|
762
|
+
|
|
763
|
+
const resolveDiscoveredProcessorSpec = (
|
|
764
|
+
definition: FileWorkerDefinition | undefined,
|
|
765
|
+
sourcePath: string
|
|
766
|
+
): string => {
|
|
767
|
+
return definition?.processorSpec ?? getProjectRelativeWorkerSpec(sourcePath);
|
|
768
|
+
};
|
|
769
|
+
|
|
770
|
+
const resolveDiscoveredVersion = (definition: FileWorkerDefinition | undefined): string => {
|
|
771
|
+
return definition?.version ?? '1.0.0';
|
|
772
|
+
};
|
|
773
|
+
|
|
774
|
+
const resolveDiscoveredStatus = (definition: FileWorkerDefinition | undefined): string => {
|
|
775
|
+
return definition?.status ?? WorkerCreationStatus.STOPPED;
|
|
776
|
+
};
|
|
777
|
+
|
|
778
|
+
const resolveDiscoveredAutoStart = (definition: FileWorkerDefinition | undefined): boolean => {
|
|
779
|
+
return definition?.autoStart ?? false;
|
|
780
|
+
};
|
|
781
|
+
|
|
782
|
+
const resolveDiscoveredConcurrency = (definition: FileWorkerDefinition | undefined): number => {
|
|
783
|
+
return definition?.concurrency ?? 1;
|
|
784
|
+
};
|
|
785
|
+
|
|
786
|
+
const resolveDiscoveredActiveStatus = (definition: FileWorkerDefinition | undefined): boolean => {
|
|
787
|
+
return definition?.activeStatus ?? true;
|
|
788
|
+
};
|
|
789
|
+
|
|
790
|
+
const buildDiscoveredWorkerRecord = (
|
|
791
|
+
definition: FileWorkerDefinition | undefined,
|
|
792
|
+
queueWorkerMeta: QueueWorkerMeta | undefined,
|
|
793
|
+
sourcePath: string
|
|
794
|
+
): WorkerRecord | undefined => {
|
|
795
|
+
const recordName = resolveDiscoveredWorkerName(definition, sourcePath);
|
|
796
|
+
|
|
797
|
+
if (!isNonEmptyString(recordName)) return undefined;
|
|
798
|
+
|
|
799
|
+
return {
|
|
800
|
+
name: recordName,
|
|
801
|
+
queueName: resolveDiscoveredQueueName(definition, queueWorkerMeta, recordName),
|
|
802
|
+
version: resolveDiscoveredVersion(definition),
|
|
803
|
+
status: resolveDiscoveredStatus(definition),
|
|
804
|
+
autoStart: resolveDiscoveredAutoStart(definition),
|
|
805
|
+
concurrency: resolveDiscoveredConcurrency(definition),
|
|
806
|
+
region: definition?.region ?? null,
|
|
807
|
+
processorSpec: resolveDiscoveredProcessorSpec(definition, sourcePath),
|
|
808
|
+
activeStatus: resolveDiscoveredActiveStatus(definition),
|
|
809
|
+
features: definition?.features ?? null,
|
|
810
|
+
infrastructure: definition?.infrastructure ?? null,
|
|
811
|
+
datacenter: definition?.datacenter ?? null,
|
|
812
|
+
createdAt: new Date(),
|
|
813
|
+
updatedAt: new Date(),
|
|
814
|
+
};
|
|
815
|
+
};
|
|
816
|
+
|
|
817
|
+
const buildDiscoveredWorker = (
|
|
818
|
+
mod: Record<string, unknown>,
|
|
819
|
+
sourcePath: string
|
|
820
|
+
): DiscoveredFileWorker | undefined => {
|
|
821
|
+
const definition = getExportedFileWorkerDefinition(mod);
|
|
822
|
+
const queueWorkerMeta = getExportedQueueWorkerMeta(mod);
|
|
823
|
+
const processor = resolveDiscoveredWorkerProcessor(definition, mod, sourcePath);
|
|
824
|
+
const record = buildDiscoveredWorkerRecord(definition, queueWorkerMeta, sourcePath);
|
|
825
|
+
if (!record) return undefined;
|
|
826
|
+
|
|
827
|
+
return {
|
|
828
|
+
record,
|
|
829
|
+
processor,
|
|
830
|
+
sourcePath,
|
|
831
|
+
};
|
|
832
|
+
};
|
|
833
|
+
|
|
834
|
+
const resolveStartFromPersistedRecord = async (
|
|
835
|
+
name: string,
|
|
836
|
+
persistenceOverride?: WorkerPersistenceConfig
|
|
837
|
+
): Promise<{ record: WorkerRecord; discovered: DiscoveredFileWorker | null }> => {
|
|
838
|
+
const persistedRecord = await getPersistedRecord(name, persistenceOverride);
|
|
839
|
+
const discovered = persistedRecord ? null : await getDiscoveredFileWorker(name);
|
|
840
|
+
const effectiveRecord = persistedRecord ?? discovered?.record ?? null;
|
|
841
|
+
|
|
842
|
+
if (!effectiveRecord) {
|
|
843
|
+
throw ErrorFactory.createNotFoundError(`Worker "${name}" not found in persistence store`);
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
return { record: effectiveRecord, discovered };
|
|
847
|
+
};
|
|
848
|
+
|
|
849
|
+
const resolveStartFromPersistedProcessor = async (
|
|
850
|
+
name: string,
|
|
851
|
+
record: WorkerRecord,
|
|
852
|
+
discovered: DiscoveredFileWorker | null
|
|
853
|
+
): Promise<WorkerFactoryConfig['processor']> => {
|
|
854
|
+
let processor = await resolveProcessor(name);
|
|
855
|
+
|
|
856
|
+
if (!processor && discovered?.processor) {
|
|
857
|
+
processor = discovered.processor;
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
const spec = record.processorSpec ?? undefined;
|
|
861
|
+
if (!processor && spec) {
|
|
862
|
+
try {
|
|
863
|
+
processor = await resolveProcessorSpec(spec);
|
|
864
|
+
} catch (error) {
|
|
865
|
+
Logger.error(`Failed to resolve processor module for "${name}"`, error);
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
if (!processor) {
|
|
870
|
+
throw ErrorFactory.createConfigError(
|
|
871
|
+
`Worker "${name}" processor is not registered or resolvable. Register the processor at startup or persist a processorSpec.`
|
|
872
|
+
);
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
return processor;
|
|
876
|
+
};
|
|
877
|
+
|
|
878
|
+
const discoverFileBackedWorkers = async (): Promise<DiscoveredFileWorker[]> => {
|
|
879
|
+
const files = listWorkerDefinitionFiles();
|
|
880
|
+
if (files.length === 0) return [];
|
|
881
|
+
|
|
882
|
+
const discovered = new Map<string, DiscoveredFileWorker>();
|
|
883
|
+
|
|
884
|
+
for (const filePath of files) {
|
|
885
|
+
// eslint-disable-next-line no-await-in-loop
|
|
886
|
+
const mod = await importWorkerDefinitionModule(filePath);
|
|
887
|
+
if (!mod) continue;
|
|
888
|
+
|
|
889
|
+
const discoveredWorker = buildDiscoveredWorker(mod, filePath);
|
|
890
|
+
if (!discoveredWorker) continue;
|
|
891
|
+
|
|
892
|
+
if (discovered.has(discoveredWorker.record.name)) {
|
|
893
|
+
Logger.warn(
|
|
894
|
+
`Duplicate file-backed worker definition detected for "${discoveredWorker.record.name}". Keeping the first discovered module.`
|
|
895
|
+
);
|
|
896
|
+
continue;
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
discovered.set(discoveredWorker.record.name, discoveredWorker);
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
return Array.from(discovered.values());
|
|
903
|
+
};
|
|
904
|
+
|
|
905
|
+
const getDiscoveredFileWorker = async (name: string): Promise<DiscoveredFileWorker | null> => {
|
|
906
|
+
const discovered = await discoverFileBackedWorkers();
|
|
907
|
+
return discovered.find((entry) => entry.record.name === name) ?? null;
|
|
908
|
+
};
|
|
909
|
+
|
|
496
910
|
const computeSha256 = async (value: string): Promise<string> => {
|
|
497
911
|
if (typeof globalThis !== 'undefined' && globalThis.crypto?.subtle) {
|
|
498
912
|
const data = new TextEncoder().encode(value);
|
|
@@ -579,7 +993,7 @@ const waitForWorkerConnection = async (
|
|
|
579
993
|
const checkConnection = async (): Promise<void> => {
|
|
580
994
|
try {
|
|
581
995
|
// Check if worker is actually running
|
|
582
|
-
const isRunning = await worker.isRunning();
|
|
996
|
+
const isRunning = await worker.isRunning(); // NOSONAR - BullMQ's isRunning method
|
|
583
997
|
if (!isRunning) {
|
|
584
998
|
throw ErrorFactory.createWorkerError('Worker not running');
|
|
585
999
|
}
|
|
@@ -778,7 +1192,7 @@ const cacheProcessorFromResponse = async (params: {
|
|
|
778
1192
|
const rawCode = await readResponseBody(response, config.fetchMaxSizeBytes);
|
|
779
1193
|
const code = rewriteProcessorImports(rawCode);
|
|
780
1194
|
const mod = await importModuleFromCode({ code, normalized, cacheKey });
|
|
781
|
-
const processor = extractZinTrustProcessor(mod
|
|
1195
|
+
const processor = extractZinTrustProcessor(mod, normalized);
|
|
782
1196
|
if (!processor) {
|
|
783
1197
|
throw ErrorFactory.createConfigError('INVALID_PROCESSOR_URL_EXPORT');
|
|
784
1198
|
}
|
|
@@ -1570,12 +1984,13 @@ const resolveRedisConfigFromEnv = (config: RedisEnvConfig, context: string): Red
|
|
|
1570
1984
|
|
|
1571
1985
|
const resolveRedisConfigFromDirect = (config: RedisConfig, context: string): RedisConfig => {
|
|
1572
1986
|
const fallbackDb = Env.getInt('REDIS_QUEUE_DB', ZintrustLang.REDIS_DEFAULT_DB);
|
|
1987
|
+
const redisConfigWithDatabase = config as RedisConfig & { database?: number };
|
|
1573
1988
|
|
|
1574
1989
|
let normalizedDb = fallbackDb;
|
|
1575
1990
|
if (typeof config.db === 'number') {
|
|
1576
1991
|
normalizedDb = config.db;
|
|
1577
|
-
} else if (typeof
|
|
1578
|
-
normalizedDb =
|
|
1992
|
+
} else if (typeof redisConfigWithDatabase.database === 'number') {
|
|
1993
|
+
normalizedDb = redisConfigWithDatabase.database;
|
|
1579
1994
|
}
|
|
1580
1995
|
|
|
1581
1996
|
return {
|
|
@@ -1970,13 +2385,9 @@ const resolveAutoScalerConfig = (input: AutoScalerConfig | undefined): AutoScale
|
|
|
1970
2385
|
const resolveWorkerOptions = (config: WorkerFactoryConfig, autoStart: boolean): WorkerOptions => {
|
|
1971
2386
|
const options = config.options ? { ...config.options } : ({} as WorkerOptions);
|
|
1972
2387
|
|
|
1973
|
-
|
|
1974
|
-
options.prefix = getBullMQSafeQueueName();
|
|
1975
|
-
}
|
|
2388
|
+
options.prefix ??= getBullMQSafeQueueName();
|
|
1976
2389
|
|
|
1977
|
-
|
|
1978
|
-
options.autorun = autoStart;
|
|
1979
|
-
}
|
|
2390
|
+
options.autorun ??= autoStart;
|
|
1980
2391
|
if (options.connection) return options;
|
|
1981
2392
|
|
|
1982
2393
|
const redisConfig = resolveRedisConfigWithFallback(
|
|
@@ -2844,31 +3255,13 @@ export const WorkerFactory = Object.freeze({
|
|
|
2844
3255
|
name: string,
|
|
2845
3256
|
persistenceOverride?: WorkerPersistenceConfig
|
|
2846
3257
|
): Promise<void> {
|
|
2847
|
-
const record = await
|
|
2848
|
-
if (!record) {
|
|
2849
|
-
throw ErrorFactory.createNotFoundError(`Worker "${name}" not found in persistence store`);
|
|
2850
|
-
}
|
|
3258
|
+
const { record, discovered } = await resolveStartFromPersistedRecord(name, persistenceOverride);
|
|
2851
3259
|
|
|
2852
3260
|
if (record.activeStatus === false) {
|
|
2853
3261
|
throw ErrorFactory.createConfigError(`Worker "${name}" is inactive`);
|
|
2854
3262
|
}
|
|
2855
3263
|
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
const spec = record.processorSpec ?? undefined;
|
|
2859
|
-
if (!processor && spec) {
|
|
2860
|
-
try {
|
|
2861
|
-
processor = await resolveProcessorSpec(spec);
|
|
2862
|
-
} catch (error) {
|
|
2863
|
-
Logger.error(`Failed to resolve processor module for "${name}"`, error);
|
|
2864
|
-
}
|
|
2865
|
-
}
|
|
2866
|
-
|
|
2867
|
-
if (!processor) {
|
|
2868
|
-
throw ErrorFactory.createConfigError(
|
|
2869
|
-
`Worker "${name}" processor is not registered or resolvable. Register the processor at startup or persist a processorSpec.`
|
|
2870
|
-
);
|
|
2871
|
-
}
|
|
3264
|
+
const processor = await resolveStartFromPersistedProcessor(name, record, discovered);
|
|
2872
3265
|
|
|
2873
3266
|
await WorkerFactory.create({
|
|
2874
3267
|
name: record.name,
|
|
@@ -2885,6 +3278,22 @@ export const WorkerFactory = Object.freeze({
|
|
|
2885
3278
|
});
|
|
2886
3279
|
},
|
|
2887
3280
|
|
|
3281
|
+
/**
|
|
3282
|
+
* List worker definitions discovered from project files.
|
|
3283
|
+
*/
|
|
3284
|
+
async listFileBackedRecords(): Promise<WorkerRecord[]> {
|
|
3285
|
+
const discovered = await discoverFileBackedWorkers();
|
|
3286
|
+
return discovered.map((entry) => entry.record);
|
|
3287
|
+
},
|
|
3288
|
+
|
|
3289
|
+
/**
|
|
3290
|
+
* Get a file-backed worker definition by name.
|
|
3291
|
+
*/
|
|
3292
|
+
async getFileBackedRecord(name: string): Promise<WorkerRecord | null> {
|
|
3293
|
+
const discovered = await getDiscoveredFileWorker(name);
|
|
3294
|
+
return discovered?.record ?? null;
|
|
3295
|
+
},
|
|
3296
|
+
|
|
2888
3297
|
/**
|
|
2889
3298
|
* Get persisted worker record
|
|
2890
3299
|
*/
|
package/src/WorkerInit.ts
CHANGED
|
@@ -148,7 +148,7 @@ type PersistenceOverride = WorkerPersistenceConfig;
|
|
|
148
148
|
|
|
149
149
|
type AutoStartTask = AutoStartCandidate & {
|
|
150
150
|
persistenceOverride: PersistenceOverride;
|
|
151
|
-
source: 'database' | 'redis' | 'memory';
|
|
151
|
+
source: 'database' | 'redis' | 'memory' | 'file';
|
|
152
152
|
};
|
|
153
153
|
|
|
154
154
|
const resolveAutoStartCandidates = (records: AutoStartCandidate[]): AutoStartCandidate[] => {
|
|
@@ -229,6 +229,79 @@ const collectAutoStartTasks = async (): Promise<AutoStartTask[]> => {
|
|
|
229
229
|
return tasks;
|
|
230
230
|
};
|
|
231
231
|
|
|
232
|
+
export const buildFileBackedAutoStartTasks = (
|
|
233
|
+
records: AutoStartCandidate[],
|
|
234
|
+
warn: (message: string) => void = Logger.warn
|
|
235
|
+
): AutoStartTask[] => {
|
|
236
|
+
const tasks: AutoStartTask[] = [];
|
|
237
|
+
const seenWorkerNames = new Set<string>();
|
|
238
|
+
const candidates = resolveAutoStartCandidates(records);
|
|
239
|
+
|
|
240
|
+
for (const record of candidates) {
|
|
241
|
+
if (seenWorkerNames.has(record.name)) {
|
|
242
|
+
warn(
|
|
243
|
+
`Worker ${record.name} appears multiple times in file-backed discovery; keeping the first definition and skipping duplicates.`
|
|
244
|
+
);
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
seenWorkerNames.add(record.name);
|
|
249
|
+
tasks.push({
|
|
250
|
+
...record,
|
|
251
|
+
persistenceOverride: { driver: 'memory' },
|
|
252
|
+
source: 'file',
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
return tasks;
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
export const selectAutoStartTasks = (
|
|
260
|
+
persistedTasks: AutoStartTask[],
|
|
261
|
+
fileRecords: AutoStartCandidate[],
|
|
262
|
+
warn: (message: string) => void = Logger.warn
|
|
263
|
+
): AutoStartTask[] => {
|
|
264
|
+
if (persistedTasks.length > 0) {
|
|
265
|
+
return persistedTasks;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
return buildFileBackedAutoStartTasks(fileRecords, warn);
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
export const selectAutoStartNames = (
|
|
272
|
+
persistedRecords: AutoStartCandidate[],
|
|
273
|
+
fileRecords: AutoStartCandidate[],
|
|
274
|
+
warn: (message: string) => void = Logger.warn
|
|
275
|
+
): { names: string[]; source: 'persisted' | 'file' | 'none' } => {
|
|
276
|
+
const persistedNames = resolveAutoStartCandidates(persistedRecords).map((record) => record.name);
|
|
277
|
+
|
|
278
|
+
if (persistedNames.length > 0) {
|
|
279
|
+
return { names: persistedNames, source: 'persisted' };
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
const fileNames = buildFileBackedAutoStartTasks(fileRecords, warn).map((record) => record.name);
|
|
283
|
+
return { names: fileNames, source: fileNames.length > 0 ? 'file' : 'none' };
|
|
284
|
+
};
|
|
285
|
+
|
|
286
|
+
const collectFileBackedAutoStartTasks = async (): Promise<AutoStartTask[]> => {
|
|
287
|
+
try {
|
|
288
|
+
const records = await WorkerFactory.listFileBackedRecords();
|
|
289
|
+
const tasks = buildFileBackedAutoStartTasks(records);
|
|
290
|
+
|
|
291
|
+
Logger.debug('File-backed auto-start discovery', {
|
|
292
|
+
totalRecords: records.length,
|
|
293
|
+
candidateCount: tasks.length,
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
return tasks;
|
|
297
|
+
} catch (error) {
|
|
298
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
299
|
+
Logger.warn(`File-backed auto-start discovery failed: ${message}`);
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
return [];
|
|
303
|
+
};
|
|
304
|
+
|
|
232
305
|
const isWorkerTrulyRunning = async (name: string): Promise<boolean> => {
|
|
233
306
|
const existing = WorkerFactory.get(name);
|
|
234
307
|
if (!existing) return false;
|
|
@@ -357,16 +430,21 @@ async function autoStartPersistedWorkers(): Promise<void> {
|
|
|
357
430
|
}
|
|
358
431
|
|
|
359
432
|
try {
|
|
360
|
-
|
|
433
|
+
let candidates = await collectAutoStartTasks();
|
|
434
|
+
|
|
435
|
+
if (candidates.length === 0) {
|
|
436
|
+
candidates = await collectFileBackedAutoStartTasks();
|
|
437
|
+
}
|
|
361
438
|
|
|
362
439
|
const results = await Promise.all(candidates.map(async (record) => autoStartOneWorker(record)));
|
|
363
440
|
|
|
364
441
|
const startedCount = results.filter((item) => item.started).length;
|
|
365
442
|
const skippedCount = results.filter((item) => item.skipped).length;
|
|
366
|
-
Logger.info('Auto-started
|
|
443
|
+
Logger.info('Auto-started workers', {
|
|
367
444
|
total: candidates.length,
|
|
368
445
|
started: startedCount,
|
|
369
446
|
skipped: skippedCount,
|
|
447
|
+
source: candidates[0]?.source ?? 'none',
|
|
370
448
|
});
|
|
371
449
|
} catch (error) {
|
|
372
450
|
const message = error instanceof Error ? error.message : String(error);
|
package/src/createQueueWorker.ts
CHANGED
|
@@ -127,6 +127,11 @@ type QueueWorker = {
|
|
|
127
127
|
signal?: AbortSignal;
|
|
128
128
|
maxDurationMs?: number;
|
|
129
129
|
}) => Promise<number>;
|
|
130
|
+
__zintrustQueueWorkerMeta?: Readonly<{
|
|
131
|
+
kindLabel: string;
|
|
132
|
+
defaultQueueName: string;
|
|
133
|
+
maxAttempts: number;
|
|
134
|
+
}>;
|
|
130
135
|
};
|
|
131
136
|
|
|
132
137
|
export type CreateQueueWorkerOptions<TPayload> = {
|
|
@@ -404,7 +409,7 @@ const processQueueMessage = async <TPayload>(
|
|
|
404
409
|
queueName: string,
|
|
405
410
|
driverName?: string
|
|
406
411
|
): Promise<boolean> => {
|
|
407
|
-
const message = await Queue.dequeue
|
|
412
|
+
const message = (await Queue.dequeue(queueName, driverName)) as QueueMessage<TPayload> | null;
|
|
408
413
|
if (!message) return false;
|
|
409
414
|
|
|
410
415
|
const baseLogFields = buildBaseLogFields(message, options.getLogFields);
|
|
@@ -597,6 +602,17 @@ export function createQueueWorker<TPayload>(
|
|
|
597
602
|
const processAll = createProcessAll(options.defaultQueueName, processOne);
|
|
598
603
|
const runOnce = createRunOnce(options.defaultQueueName, processOne);
|
|
599
604
|
const startWorker = createStartWorker(options.kindLabel, options.defaultQueueName, processOne);
|
|
605
|
+
const queueWorkerMeta = Object.freeze({
|
|
606
|
+
kindLabel: options.kindLabel,
|
|
607
|
+
defaultQueueName: options.defaultQueueName,
|
|
608
|
+
maxAttempts: options.maxAttempts,
|
|
609
|
+
});
|
|
600
610
|
|
|
601
|
-
return Object.freeze({
|
|
611
|
+
return Object.freeze({
|
|
612
|
+
processOne,
|
|
613
|
+
processAll,
|
|
614
|
+
runOnce,
|
|
615
|
+
startWorker,
|
|
616
|
+
__zintrustQueueWorkerMeta: queueWorkerMeta,
|
|
617
|
+
});
|
|
602
618
|
}
|