@zintrust/workers 0.1.31 → 0.1.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ClusterLock.js +3 -2
- package/dist/DeadLetterQueue.js +3 -2
- package/dist/HealthMonitor.js +24 -13
- package/dist/Observability.js +8 -0
- package/dist/WorkerFactory.d.ts +4 -0
- package/dist/WorkerFactory.js +384 -42
- package/dist/WorkerInit.js +122 -43
- package/dist/WorkerMetrics.js +5 -1
- package/dist/WorkerRegistry.js +8 -0
- package/dist/WorkerShutdown.d.ts +0 -13
- package/dist/WorkerShutdown.js +1 -44
- package/dist/build-manifest.json +99 -83
- package/dist/config/workerConfig.d.ts +1 -0
- package/dist/config/workerConfig.js +7 -1
- package/dist/createQueueWorker.js +281 -42
- package/dist/dashboard/workers-api.js +8 -1
- package/dist/http/WorkerController.js +90 -35
- package/dist/http/WorkerMonitoringService.js +29 -2
- package/dist/index.d.ts +1 -2
- package/dist/index.js +0 -1
- package/dist/routes/workers.js +10 -7
- package/dist/storage/WorkerStore.d.ts +6 -3
- package/dist/storage/WorkerStore.js +16 -0
- package/dist/telemetry/api/TelemetryMonitoringService.js +29 -2
- package/dist/ui/router/ui.js +58 -29
- package/dist/ui/workers/index.html +202 -0
- package/dist/ui/workers/main.js +1952 -0
- package/dist/ui/workers/styles.css +1350 -0
- package/dist/ui/workers/zintrust.svg +30 -0
- package/package.json +5 -5
- package/src/ClusterLock.ts +13 -7
- package/src/ComplianceManager.ts +3 -2
- package/src/DeadLetterQueue.ts +6 -4
- package/src/HealthMonitor.ts +33 -17
- package/src/Observability.ts +11 -0
- package/src/WorkerFactory.ts +446 -43
- package/src/WorkerInit.ts +167 -48
- package/src/WorkerMetrics.ts +14 -8
- package/src/WorkerRegistry.ts +11 -0
- package/src/WorkerShutdown.ts +1 -69
- package/src/config/workerConfig.ts +9 -1
- package/src/createQueueWorker.ts +428 -43
- package/src/dashboard/workers-api.ts +8 -1
- package/src/http/WorkerController.ts +111 -36
- package/src/http/WorkerMonitoringService.ts +35 -2
- package/src/index.ts +2 -3
- package/src/routes/workers.ts +10 -8
- package/src/storage/WorkerStore.ts +21 -3
- package/src/telemetry/api/TelemetryMonitoringService.ts +35 -2
- package/src/types/queue-monitor.d.ts +2 -1
- package/src/ui/router/EmbeddedAssets.ts +3 -0
- package/src/ui/router/ui.ts +57 -39
- package/src/WorkerShutdownDurableObject.ts +0 -64
package/src/WorkerFactory.ts
CHANGED
|
@@ -5,12 +5,14 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import {
|
|
8
|
-
|
|
8
|
+
Cloudflare,
|
|
9
9
|
createRedisConnection,
|
|
10
10
|
databaseConfig,
|
|
11
11
|
Env,
|
|
12
12
|
ErrorFactory,
|
|
13
|
+
generateUuid,
|
|
13
14
|
getBullMQSafeQueueName,
|
|
15
|
+
JobStateTracker,
|
|
14
16
|
Logger,
|
|
15
17
|
NodeSingletons,
|
|
16
18
|
queueConfig,
|
|
@@ -39,6 +41,7 @@ import { ResourceMonitor } from './ResourceMonitor';
|
|
|
39
41
|
import { WorkerMetrics } from './WorkerMetrics';
|
|
40
42
|
import { WorkerRegistry, type WorkerInstance as RegistryWorkerInstance } from './WorkerRegistry';
|
|
41
43
|
import { WorkerVersioning } from './WorkerVersioning';
|
|
44
|
+
import { keyPrefix } from './config/workerConfig';
|
|
42
45
|
import {
|
|
43
46
|
DbWorkerStore,
|
|
44
47
|
InMemoryWorkerStore,
|
|
@@ -49,6 +52,172 @@ import {
|
|
|
49
52
|
|
|
50
53
|
const path = NodeSingletons.path;
|
|
51
54
|
|
|
55
|
+
const isNodeRuntime = (): boolean =>
|
|
56
|
+
typeof process !== 'undefined' && Boolean(process.versions?.node);
|
|
57
|
+
|
|
58
|
+
const resolveProjectRoot = (): string => {
|
|
59
|
+
const envRoot = Env.get('ZINTRUST_PROJECT_ROOT', '').trim();
|
|
60
|
+
return envRoot.length > 0 ? envRoot : process.cwd();
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
const canUseProjectFileImports = (): boolean =>
|
|
64
|
+
typeof NodeSingletons?.fs?.writeFileSync === 'function' &&
|
|
65
|
+
typeof NodeSingletons?.fs?.mkdirSync === 'function' &&
|
|
66
|
+
typeof NodeSingletons?.fs?.existsSync === 'function' &&
|
|
67
|
+
typeof NodeSingletons?.url?.pathToFileURL === 'function' &&
|
|
68
|
+
typeof NodeSingletons?.path?.join === 'function';
|
|
69
|
+
|
|
70
|
+
const buildCandidatesForSpecifier = (specifier: string, root: string): string[] => {
|
|
71
|
+
if (specifier === '@zintrust/core') {
|
|
72
|
+
return [
|
|
73
|
+
path.join(root, 'dist', 'src', 'index.js'),
|
|
74
|
+
path.join(root, 'dist', 'index.js'),
|
|
75
|
+
path.join(root, 'src', 'index.ts'),
|
|
76
|
+
];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (specifier === '@zintrust/workers') {
|
|
80
|
+
return [
|
|
81
|
+
path.join(root, 'dist', 'packages', 'workers', 'src', 'index.js'),
|
|
82
|
+
path.join(root, 'packages', 'workers', 'src', 'index.ts'),
|
|
83
|
+
];
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return [];
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
const getProjectFileCandidates = (paths: string[]): string | null => {
|
|
90
|
+
if (!canUseProjectFileImports()) return null;
|
|
91
|
+
for (const candidate of paths) {
|
|
92
|
+
if (NodeSingletons.fs.existsSync(candidate)) return candidate;
|
|
93
|
+
}
|
|
94
|
+
return null;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const resolveLocalPackageFallback = (specifier: string): string | null => {
|
|
98
|
+
if (!canUseProjectFileImports()) return null;
|
|
99
|
+
const root = resolveProjectRoot();
|
|
100
|
+
|
|
101
|
+
const candidates = buildCandidatesForSpecifier(specifier, root);
|
|
102
|
+
const resolved = getProjectFileCandidates(candidates);
|
|
103
|
+
if (!resolved) return null;
|
|
104
|
+
return NodeSingletons.url.pathToFileURL(resolved).href;
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
const resolvePackageSpecifierUrl = (specifier: string): string | null => {
|
|
108
|
+
if (!isNodeRuntime() || !canUseProjectFileImports()) return null;
|
|
109
|
+
if (typeof NodeSingletons?.module?.createRequire !== 'function') {
|
|
110
|
+
return resolveLocalPackageFallback(specifier);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
try {
|
|
114
|
+
const require = NodeSingletons.module.createRequire(import.meta.url);
|
|
115
|
+
const resolved = require.resolve(specifier);
|
|
116
|
+
if (
|
|
117
|
+
specifier === '@zintrust/workers' &&
|
|
118
|
+
resolved.includes(`${path.sep}node_modules${path.sep}@zintrust${path.sep}workers${path.sep}`)
|
|
119
|
+
) {
|
|
120
|
+
const local = resolveLocalPackageFallback(specifier);
|
|
121
|
+
if (local) return local;
|
|
122
|
+
}
|
|
123
|
+
return NodeSingletons.url.pathToFileURL(resolved).href;
|
|
124
|
+
} catch {
|
|
125
|
+
return resolveLocalPackageFallback(specifier);
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
const escapeRegExp = (value: string): string =>
|
|
130
|
+
value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`);
|
|
131
|
+
|
|
132
|
+
const rewriteProcessorImports = (code: string): string => {
|
|
133
|
+
const replacements: Array<{ from: string; to: string }> = [];
|
|
134
|
+
const coreUrl = resolvePackageSpecifierUrl('@zintrust/core');
|
|
135
|
+
if (coreUrl) replacements.push({ from: '@zintrust/core', to: coreUrl });
|
|
136
|
+
const workersUrl = resolvePackageSpecifierUrl('@zintrust/workers');
|
|
137
|
+
if (workersUrl) replacements.push({ from: '@zintrust/workers', to: workersUrl });
|
|
138
|
+
|
|
139
|
+
if (replacements.length === 0) return code;
|
|
140
|
+
|
|
141
|
+
let updated = code;
|
|
142
|
+
for (const { from, to } of replacements) {
|
|
143
|
+
const pattern = new RegExp(String.raw`(['"])${escapeRegExp(from)}\1`, 'g');
|
|
144
|
+
updated = updated.replace(pattern, `$1${to}$1`);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return updated;
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
const ensureProcessorSpecDir = (): string | null => {
|
|
151
|
+
if (!isNodeRuntime() || !canUseProjectFileImports()) return null;
|
|
152
|
+
const dir = path.join(resolveProjectRoot(), '.zintrust', 'processor-specs');
|
|
153
|
+
try {
|
|
154
|
+
if (!NodeSingletons.fs.existsSync(dir)) {
|
|
155
|
+
NodeSingletons.fs.mkdirSync(dir, { recursive: true });
|
|
156
|
+
}
|
|
157
|
+
return dir;
|
|
158
|
+
} catch (error) {
|
|
159
|
+
Logger.debug('Failed to prepare processor spec cache directory', error);
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
const shouldFallbackToFileImport = (error: unknown): boolean => {
|
|
165
|
+
if (!isNodeRuntime()) return false;
|
|
166
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
167
|
+
const code = (error as { code?: string } | undefined)?.code ?? '';
|
|
168
|
+
if (code === 'ERR_INVALID_URL' || code === 'ERR_UNSUPPORTED_ESM_URL_SCHEME') return true;
|
|
169
|
+
return (
|
|
170
|
+
message.includes('Invalid relative URL') ||
|
|
171
|
+
message.includes('base scheme is not hierarchical') ||
|
|
172
|
+
message.includes('Failed to resolve module specifier')
|
|
173
|
+
);
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
const importModuleFromCode = async (params: {
|
|
177
|
+
code: string;
|
|
178
|
+
normalized: string;
|
|
179
|
+
cacheKey: string;
|
|
180
|
+
}): Promise<Record<string, unknown>> => {
|
|
181
|
+
const { code, normalized, cacheKey } = params;
|
|
182
|
+
const dataUrl = `data:text/javascript;base64,${toBase64(code)}`;
|
|
183
|
+
|
|
184
|
+
try {
|
|
185
|
+
return (await import(dataUrl)) as Record<string, unknown>;
|
|
186
|
+
} catch (error) {
|
|
187
|
+
if (!shouldFallbackToFileImport(error)) throw error;
|
|
188
|
+
const dir = ensureProcessorSpecDir();
|
|
189
|
+
if (!dir) throw error;
|
|
190
|
+
|
|
191
|
+
try {
|
|
192
|
+
const codeHash = await computeSha256(code);
|
|
193
|
+
const filePath = path.join(dir, `${codeHash || cacheKey}.mjs`);
|
|
194
|
+
NodeSingletons.fs.writeFileSync(filePath, code, 'utf8');
|
|
195
|
+
const fileUrl = NodeSingletons.url.pathToFileURL(filePath).href;
|
|
196
|
+
return (await import(fileUrl)) as Record<string, unknown>;
|
|
197
|
+
} catch (fileError) {
|
|
198
|
+
Logger.debug(`Processor URL file fallback failed for ${normalized}`, fileError);
|
|
199
|
+
throw error;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
const isCloudflareRuntime = (): boolean => Cloudflare.getWorkersEnv() !== null;
|
|
205
|
+
|
|
206
|
+
const getDefaultStoreForRuntime = async (): Promise<WorkerStore> => {
|
|
207
|
+
if (!isCloudflareRuntime()) {
|
|
208
|
+
await ensureWorkerStoreConfigured();
|
|
209
|
+
return workerStore;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const bootstrapConfig = buildPersistenceBootstrapConfig();
|
|
213
|
+
const persistence = resolvePersistenceConfig(bootstrapConfig);
|
|
214
|
+
if (!persistence) {
|
|
215
|
+
return InMemoryWorkerStore.create();
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
return resolveWorkerStoreForPersistence(persistence);
|
|
219
|
+
};
|
|
220
|
+
|
|
52
221
|
const getStoreForWorker = async (
|
|
53
222
|
config: WorkerFactoryConfig | undefined,
|
|
54
223
|
persistenceOverride?: WorkerPersistenceConfig
|
|
@@ -66,8 +235,7 @@ const getStoreForWorker = async (
|
|
|
66
235
|
}
|
|
67
236
|
|
|
68
237
|
// Fallback to default/global store
|
|
69
|
-
|
|
70
|
-
return workerStore;
|
|
238
|
+
return getDefaultStoreForRuntime();
|
|
71
239
|
};
|
|
72
240
|
|
|
73
241
|
const validateAndGetStore = async (
|
|
@@ -311,7 +479,7 @@ const computeSha256 = async (value: string): Promise<string> => {
|
|
|
311
479
|
return NodeSingletons.createHash('sha256').update(value).digest('hex');
|
|
312
480
|
}
|
|
313
481
|
|
|
314
|
-
return String(
|
|
482
|
+
return String(generateUuid()).slice(2);
|
|
315
483
|
};
|
|
316
484
|
|
|
317
485
|
const toBase64 = (value: string): string => {
|
|
@@ -448,6 +616,17 @@ const stripProcessorExtension = (value: string): string => value.replace(/\.(ts|
|
|
|
448
616
|
|
|
449
617
|
const normalizeModulePath = (value: string): string => value.replaceAll('\\', '/');
|
|
450
618
|
|
|
619
|
+
const filterExistingFileCandidates = (candidates: string[]): string[] => {
|
|
620
|
+
if (!NodeSingletons?.fs?.existsSync) return candidates;
|
|
621
|
+
return candidates.filter((candidate) => {
|
|
622
|
+
try {
|
|
623
|
+
return NodeSingletons.fs.existsSync(candidate);
|
|
624
|
+
} catch {
|
|
625
|
+
return false;
|
|
626
|
+
}
|
|
627
|
+
});
|
|
628
|
+
};
|
|
629
|
+
|
|
451
630
|
const buildProcessorModuleCandidates = (modulePath: string, resolvedPath: string): string[] => {
|
|
452
631
|
const candidates: string[] = [];
|
|
453
632
|
const normalized = normalizeModulePath(modulePath.trim());
|
|
@@ -470,6 +649,31 @@ const buildProcessorModuleCandidates = (modulePath: string, resolvedPath: string
|
|
|
470
649
|
return Array.from(new Set(candidates));
|
|
471
650
|
};
|
|
472
651
|
|
|
652
|
+
const buildProcessorFilePathCandidates = (_modulePath: string, resolvedPath: string): string[] => {
|
|
653
|
+
const candidates: string[] = [];
|
|
654
|
+
const normalizedResolved = normalizeModulePath(resolvedPath);
|
|
655
|
+
const projectRoot = normalizeModulePath(resolveProjectRoot());
|
|
656
|
+
|
|
657
|
+
const strippedResolved = stripProcessorExtension(resolvedPath);
|
|
658
|
+
candidates.push(`${strippedResolved}.js`, `${strippedResolved}.mjs`);
|
|
659
|
+
|
|
660
|
+
const appIndex = normalizedResolved.lastIndexOf('/app/');
|
|
661
|
+
if (appIndex !== -1) {
|
|
662
|
+
const relative = normalizedResolved.slice(appIndex + 5);
|
|
663
|
+
if (relative) {
|
|
664
|
+
const strippedRelative = stripProcessorExtension(relative);
|
|
665
|
+
candidates.push(
|
|
666
|
+
path.join(projectRoot, 'dist', 'app', `${strippedRelative}.js`),
|
|
667
|
+
path.join(projectRoot, 'app', relative),
|
|
668
|
+
path.join(projectRoot, 'app', `${strippedRelative}.js`),
|
|
669
|
+
path.join('/app', 'dist', 'app', `${strippedRelative}.js`)
|
|
670
|
+
);
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
return filterExistingFileCandidates(Array.from(new Set(candidates)));
|
|
675
|
+
};
|
|
676
|
+
|
|
473
677
|
const pickProcessorFromModule = (
|
|
474
678
|
mod: Record<string, unknown> | undefined,
|
|
475
679
|
source: string
|
|
@@ -544,9 +748,9 @@ const cacheProcessorFromResponse = async (params: {
|
|
|
544
748
|
cacheKey: string;
|
|
545
749
|
}): Promise<WorkerFactoryConfig['processor']> => {
|
|
546
750
|
const { response, normalized, config, cacheKey } = params;
|
|
547
|
-
const
|
|
548
|
-
const
|
|
549
|
-
const mod = await
|
|
751
|
+
const rawCode = await readResponseBody(response, config.fetchMaxSizeBytes);
|
|
752
|
+
const code = rewriteProcessorImports(rawCode);
|
|
753
|
+
const mod = await importModuleFromCode({ code, normalized, cacheKey });
|
|
550
754
|
const processor = extractZinTrustProcessor(mod as Record<string, unknown>, normalized);
|
|
551
755
|
if (!processor) {
|
|
552
756
|
throw ErrorFactory.createConfigError('INVALID_PROCESSOR_URL_EXPORT');
|
|
@@ -656,10 +860,12 @@ const resolveProcessorFromUrl = async (
|
|
|
656
860
|
Logger.warn(
|
|
657
861
|
`Invalid processor URL protocol: ${parsed.protocol}. Only https:// and file:// are supported.`
|
|
658
862
|
);
|
|
863
|
+
return undefined;
|
|
659
864
|
}
|
|
660
865
|
|
|
661
866
|
if (!isAllowedRemoteHost(parsed.host) && parsed.protocol !== 'file:') {
|
|
662
867
|
Logger.warn(`Invalid processor URL host: ${parsed.host}. Host is not in the allowlist.`);
|
|
868
|
+
return undefined;
|
|
663
869
|
}
|
|
664
870
|
|
|
665
871
|
const config = getProcessorSpecConfig();
|
|
@@ -703,8 +909,12 @@ const resolveProcessorFromPath = async (
|
|
|
703
909
|
if (candidates.length === 0) return undefined;
|
|
704
910
|
const [candidatePath, ...rest] = candidates;
|
|
705
911
|
try {
|
|
706
|
-
const
|
|
707
|
-
|
|
912
|
+
const importPath =
|
|
913
|
+
candidatePath.startsWith('/') && !candidatePath.startsWith('//')
|
|
914
|
+
? NodeSingletons.url.pathToFileURL(candidatePath).href
|
|
915
|
+
: candidatePath;
|
|
916
|
+
const mod = await import(importPath);
|
|
917
|
+
const candidate = pickProcessorFromModule(mod as Record<string, unknown>, importPath);
|
|
708
918
|
if (candidate) return candidate;
|
|
709
919
|
} catch (candidateError) {
|
|
710
920
|
Logger.debug(`Processor module candidate import failed: ${candidatePath}`, candidateError);
|
|
@@ -718,9 +928,13 @@ const resolveProcessorFromPath = async (
|
|
|
718
928
|
const candidate = pickProcessorFromModule(mod as Record<string, unknown>, resolved);
|
|
719
929
|
if (candidate) return candidate;
|
|
720
930
|
} catch (err) {
|
|
721
|
-
const
|
|
722
|
-
const
|
|
723
|
-
if (
|
|
931
|
+
const fileCandidates = buildProcessorFilePathCandidates(trimmed, resolved);
|
|
932
|
+
const resolvedFileCandidate = await importProcessorFromCandidates(fileCandidates);
|
|
933
|
+
if (resolvedFileCandidate) return resolvedFileCandidate;
|
|
934
|
+
|
|
935
|
+
const moduleCandidates = buildProcessorModuleCandidates(trimmed, resolved);
|
|
936
|
+
const resolvedModuleCandidate = await importProcessorFromCandidates(moduleCandidates);
|
|
937
|
+
if (resolvedModuleCandidate) return resolvedModuleCandidate;
|
|
724
938
|
Logger.error(`Failed to import processor from path: ${resolved}`, err);
|
|
725
939
|
}
|
|
726
940
|
|
|
@@ -1096,6 +1310,76 @@ const handleFailure = async (params: {
|
|
|
1096
1310
|
await executeFailurePlugins(workerName, job, error, features);
|
|
1097
1311
|
};
|
|
1098
1312
|
|
|
1313
|
+
const toBackoffDelayMs = (backoff: unknown): number => {
|
|
1314
|
+
if (typeof backoff === 'number' && Number.isFinite(backoff)) {
|
|
1315
|
+
return Math.max(0, Math.floor(backoff));
|
|
1316
|
+
}
|
|
1317
|
+
if (backoff !== null && backoff !== undefined && typeof backoff === 'object') {
|
|
1318
|
+
const raw = (backoff as { delay?: unknown }).delay;
|
|
1319
|
+
if (typeof raw === 'number' && Number.isFinite(raw)) {
|
|
1320
|
+
return Math.max(0, Math.floor(raw));
|
|
1321
|
+
}
|
|
1322
|
+
}
|
|
1323
|
+
return 0;
|
|
1324
|
+
};
|
|
1325
|
+
|
|
1326
|
+
const trackJobStarted = async (input: {
|
|
1327
|
+
queueName: string;
|
|
1328
|
+
job: Job;
|
|
1329
|
+
attempts: number;
|
|
1330
|
+
workerName: string;
|
|
1331
|
+
workerVersion: string;
|
|
1332
|
+
}): Promise<void> => {
|
|
1333
|
+
if (!input.job.id) return;
|
|
1334
|
+
await JobStateTracker.started({
|
|
1335
|
+
queueName: input.queueName,
|
|
1336
|
+
jobId: input.job.id,
|
|
1337
|
+
attempts: input.attempts,
|
|
1338
|
+
timeoutMs: Math.max(1000, Env.getInt('QUEUE_JOB_TIMEOUT', 60) * 1000),
|
|
1339
|
+
workerName: input.workerName,
|
|
1340
|
+
workerVersion: input.workerVersion,
|
|
1341
|
+
});
|
|
1342
|
+
};
|
|
1343
|
+
|
|
1344
|
+
const trackJobCompleted = async (input: {
|
|
1345
|
+
queueName: string;
|
|
1346
|
+
job: Job;
|
|
1347
|
+
duration: number;
|
|
1348
|
+
result: unknown;
|
|
1349
|
+
}): Promise<void> => {
|
|
1350
|
+
if (!input.job.id) return;
|
|
1351
|
+
await JobStateTracker.completed({
|
|
1352
|
+
queueName: input.queueName,
|
|
1353
|
+
jobId: input.job.id,
|
|
1354
|
+
processingTimeMs: input.duration,
|
|
1355
|
+
result: input.result,
|
|
1356
|
+
});
|
|
1357
|
+
};
|
|
1358
|
+
|
|
1359
|
+
const trackJobFailed = async (input: {
|
|
1360
|
+
queueName: string;
|
|
1361
|
+
job: Job;
|
|
1362
|
+
attempts: number;
|
|
1363
|
+
maxAttempts?: number;
|
|
1364
|
+
error: Error;
|
|
1365
|
+
}): Promise<void> => {
|
|
1366
|
+
if (!input.job.id) return;
|
|
1367
|
+
const isFinal = input.maxAttempts === undefined ? true : input.attempts >= input.maxAttempts;
|
|
1368
|
+
const backoffDelayMs = toBackoffDelayMs(input.job.opts?.backoff);
|
|
1369
|
+
|
|
1370
|
+
await JobStateTracker.failed({
|
|
1371
|
+
queueName: input.queueName,
|
|
1372
|
+
jobId: input.job.id,
|
|
1373
|
+
attempts: input.attempts,
|
|
1374
|
+
isFinal,
|
|
1375
|
+
retryAt:
|
|
1376
|
+
!isFinal && backoffDelayMs > 0
|
|
1377
|
+
? new Date(Date.now() + backoffDelayMs).toISOString()
|
|
1378
|
+
: undefined,
|
|
1379
|
+
error: input.error,
|
|
1380
|
+
});
|
|
1381
|
+
};
|
|
1382
|
+
|
|
1099
1383
|
/**
|
|
1100
1384
|
* Helper: Create enhanced processor with all features
|
|
1101
1385
|
*/
|
|
@@ -1119,9 +1403,23 @@ const createEnhancedProcessor = (config: WorkerFactoryConfig): ((job: Job) => Pr
|
|
|
1119
1403
|
let result: unknown;
|
|
1120
1404
|
let spanId: string | null = null;
|
|
1121
1405
|
|
|
1406
|
+
const maxAttempts =
|
|
1407
|
+
typeof job.opts?.attempts === 'number' && Number.isFinite(job.opts.attempts)
|
|
1408
|
+
? Math.max(1, Math.floor(job.opts.attempts))
|
|
1409
|
+
: undefined;
|
|
1410
|
+
const attempts = Math.max(1, Math.floor((job.attemptsMade ?? 0) + 1));
|
|
1411
|
+
|
|
1122
1412
|
try {
|
|
1123
1413
|
spanId = startProcessingSpan(name, jobVersion, job, config.queueName, features);
|
|
1124
1414
|
|
|
1415
|
+
await trackJobStarted({
|
|
1416
|
+
queueName: config.queueName,
|
|
1417
|
+
job,
|
|
1418
|
+
attempts,
|
|
1419
|
+
workerName: name,
|
|
1420
|
+
workerVersion: jobVersion,
|
|
1421
|
+
});
|
|
1422
|
+
|
|
1125
1423
|
// Process the job
|
|
1126
1424
|
result = await processor(job);
|
|
1127
1425
|
|
|
@@ -1136,11 +1434,21 @@ const createEnhancedProcessor = (config: WorkerFactoryConfig): ((job: Job) => Pr
|
|
|
1136
1434
|
features,
|
|
1137
1435
|
});
|
|
1138
1436
|
|
|
1437
|
+
await trackJobCompleted({ queueName: config.queueName, job, duration, result });
|
|
1438
|
+
|
|
1139
1439
|
return result;
|
|
1140
1440
|
} catch (err) {
|
|
1141
1441
|
const error = err as Error;
|
|
1142
1442
|
const duration = Date.now() - startTime;
|
|
1143
1443
|
|
|
1444
|
+
await trackJobFailed({
|
|
1445
|
+
queueName: config.queueName,
|
|
1446
|
+
job,
|
|
1447
|
+
attempts,
|
|
1448
|
+
maxAttempts,
|
|
1449
|
+
error,
|
|
1450
|
+
});
|
|
1451
|
+
|
|
1144
1452
|
await handleFailure({
|
|
1145
1453
|
workerName: name,
|
|
1146
1454
|
jobVersion,
|
|
@@ -1226,12 +1534,23 @@ const resolveRedisConfigFromEnv = (config: RedisEnvConfig, context: string): Red
|
|
|
1226
1534
|
};
|
|
1227
1535
|
};
|
|
1228
1536
|
|
|
1229
|
-
const resolveRedisConfigFromDirect = (config: RedisConfig, context: string): RedisConfig =>
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1537
|
+
const resolveRedisConfigFromDirect = (config: RedisConfig, context: string): RedisConfig => {
|
|
1538
|
+
const fallbackDb = Env.getInt('REDIS_QUEUE_DB', ZintrustLang.REDIS_DEFAULT_DB);
|
|
1539
|
+
|
|
1540
|
+
let normalizedDb = fallbackDb;
|
|
1541
|
+
if (typeof config.db === 'number') {
|
|
1542
|
+
normalizedDb = config.db;
|
|
1543
|
+
} else if (typeof (config as { database?: number }).database === 'number') {
|
|
1544
|
+
normalizedDb = (config as { database?: number }).database as number;
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1547
|
+
return {
|
|
1548
|
+
host: requireRedisHost(config.host, context),
|
|
1549
|
+
port: config.port,
|
|
1550
|
+
db: normalizedDb,
|
|
1551
|
+
password: config.password ?? Env.get('REDIS_PASSWORD', undefined),
|
|
1552
|
+
};
|
|
1553
|
+
};
|
|
1235
1554
|
|
|
1236
1555
|
const resolveRedisConfig = (config: RedisConfigInput, context: string): RedisConfig =>
|
|
1237
1556
|
isRedisEnvConfig(config)
|
|
@@ -1252,18 +1571,26 @@ const resolveRedisConfigWithFallback = (
|
|
|
1252
1571
|
return resolveRedisConfig(selected, context);
|
|
1253
1572
|
};
|
|
1254
1573
|
|
|
1574
|
+
const logRedisPersistenceConfig = (
|
|
1575
|
+
redisConfig: RedisConfig,
|
|
1576
|
+
key_prefix: string,
|
|
1577
|
+
source: string
|
|
1578
|
+
): void => {
|
|
1579
|
+
Logger.debug('Worker persistence redis config', {
|
|
1580
|
+
source,
|
|
1581
|
+
host: redisConfig.host,
|
|
1582
|
+
port: redisConfig.port,
|
|
1583
|
+
db: redisConfig.db,
|
|
1584
|
+
key_prefix,
|
|
1585
|
+
});
|
|
1586
|
+
};
|
|
1587
|
+
|
|
1255
1588
|
const normalizeEnvValue = (value: string | undefined): string | undefined => {
|
|
1256
1589
|
if (!value) return undefined;
|
|
1257
1590
|
const trimmed = value.trim();
|
|
1258
1591
|
return trimmed.length > 0 ? trimmed : undefined;
|
|
1259
1592
|
};
|
|
1260
1593
|
|
|
1261
|
-
const normalizeAppName = (value: string | undefined): string => {
|
|
1262
|
-
const normalized = (value ?? '').trim().replaceAll(/\s+/g, '_');
|
|
1263
|
-
return normalized.length > 0 ? normalized : 'zintrust';
|
|
1264
|
-
};
|
|
1265
|
-
|
|
1266
|
-
const resolveDefaultRedisKeyPrefix = (): string => 'worker_' + normalizeAppName(appConfig.prefix);
|
|
1267
1594
|
const resolveDefaultPersistenceTable = (): string =>
|
|
1268
1595
|
normalizeEnvValue(Env.get('WORKER_PERSISTENCE_TABLE', 'zintrust_workers')) ?? 'zintrust_workers';
|
|
1269
1596
|
|
|
@@ -1288,10 +1615,7 @@ const normalizeExplicitPersistence = (
|
|
|
1288
1615
|
return {
|
|
1289
1616
|
driver: 'redis',
|
|
1290
1617
|
redis: persistence.redis,
|
|
1291
|
-
keyPrefix:
|
|
1292
|
-
persistence.keyPrefix ??
|
|
1293
|
-
normalizeEnvValue(Env.get('WORKER_PERSISTENCE_REDIS_KEY_PREFIX', '')) ??
|
|
1294
|
-
resolveDefaultRedisKeyPrefix(),
|
|
1618
|
+
keyPrefix: keyPrefix(),
|
|
1295
1619
|
};
|
|
1296
1620
|
}
|
|
1297
1621
|
|
|
@@ -1326,11 +1650,16 @@ const resolvePersistenceConfig = (
|
|
|
1326
1650
|
if (driver === 'memory') return { driver: 'memory' };
|
|
1327
1651
|
|
|
1328
1652
|
if (driver === 'redis') {
|
|
1329
|
-
const
|
|
1653
|
+
const persistenceDbOverride = normalizeEnvValue(Env.get('WORKER_PERSISTENCE_REDIS_DB', ''));
|
|
1654
|
+
|
|
1330
1655
|
return {
|
|
1331
1656
|
driver: 'redis',
|
|
1332
|
-
|
|
1333
|
-
|
|
1657
|
+
// Optional override; otherwise defaults to REDIS_QUEUE_DB.
|
|
1658
|
+
redis: {
|
|
1659
|
+
env: true,
|
|
1660
|
+
db: persistenceDbOverride ? 'WORKER_PERSISTENCE_REDIS_DB' : 'REDIS_QUEUE_DB',
|
|
1661
|
+
},
|
|
1662
|
+
keyPrefix: keyPrefix(),
|
|
1334
1663
|
};
|
|
1335
1664
|
}
|
|
1336
1665
|
|
|
@@ -1383,8 +1712,10 @@ const resolveWorkerStore = async (config: WorkerFactoryConfig): Promise<WorkerSt
|
|
|
1383
1712
|
'Worker persistence requires redis config (persistence.redis or infrastructure.redis)',
|
|
1384
1713
|
'infrastructure.persistence.redis'
|
|
1385
1714
|
);
|
|
1715
|
+
const key_prefix = persistence.keyPrefix ?? keyPrefix();
|
|
1716
|
+
logRedisPersistenceConfig(redisConfig, key_prefix, 'resolveWorkerStore');
|
|
1386
1717
|
const client = createRedisConnection(redisConfig);
|
|
1387
|
-
next = RedisWorkerStore.create(client,
|
|
1718
|
+
next = RedisWorkerStore.create(client, key_prefix);
|
|
1388
1719
|
} else if (persistence.driver === 'database') {
|
|
1389
1720
|
const explicitConnection =
|
|
1390
1721
|
typeof persistence.client === 'string' ? persistence.client : persistence.connection;
|
|
@@ -1435,8 +1766,10 @@ const createWorkerStore = async (persistence: WorkerPersistenceConfig): Promise<
|
|
|
1435
1766
|
'Worker persistence requires redis config (persistence.redis or REDIS_* env values)',
|
|
1436
1767
|
'persistence.redis'
|
|
1437
1768
|
);
|
|
1769
|
+
const key_prefix = persistence.keyPrefix ?? keyPrefix();
|
|
1770
|
+
logRedisPersistenceConfig(redisConfig, key_prefix, 'createWorkerStore');
|
|
1438
1771
|
const client = createRedisConnection(redisConfig);
|
|
1439
|
-
return RedisWorkerStore.create(client,
|
|
1772
|
+
return RedisWorkerStore.create(client, key_prefix);
|
|
1440
1773
|
}
|
|
1441
1774
|
|
|
1442
1775
|
// Database driver
|
|
@@ -1453,10 +1786,12 @@ const resolveWorkerStoreForPersistence = async (
|
|
|
1453
1786
|
persistence: WorkerPersistenceConfig
|
|
1454
1787
|
): Promise<WorkerStore> => {
|
|
1455
1788
|
const cacheKey = generateCacheKey(persistence);
|
|
1789
|
+
const isCloudflare = Cloudflare.getWorkersEnv() !== null;
|
|
1456
1790
|
|
|
1457
|
-
// Return cached instance if available
|
|
1791
|
+
// Return cached instance if available (disable cache for Cloudflare to assume cleanup)
|
|
1792
|
+
// Or handle cleanup differently. For now, disable cache for Cloudflare to allow per-request connections.
|
|
1458
1793
|
const cached = storeInstanceCache.get(cacheKey);
|
|
1459
|
-
if (cached) {
|
|
1794
|
+
if (cached && !isCloudflare) {
|
|
1460
1795
|
return cached;
|
|
1461
1796
|
}
|
|
1462
1797
|
|
|
@@ -1464,8 +1799,10 @@ const resolveWorkerStoreForPersistence = async (
|
|
|
1464
1799
|
const store = await createWorkerStore(persistence);
|
|
1465
1800
|
await store.init();
|
|
1466
1801
|
|
|
1467
|
-
// Cache the store instance for reuse
|
|
1468
|
-
|
|
1802
|
+
// Cache the store instance for reuse only if not Cloudflare
|
|
1803
|
+
if (!isCloudflare) {
|
|
1804
|
+
storeInstanceCache.set(cacheKey, store);
|
|
1805
|
+
}
|
|
1469
1806
|
|
|
1470
1807
|
return store;
|
|
1471
1808
|
};
|
|
@@ -1475,8 +1812,19 @@ const getPersistedRecord = async (
|
|
|
1475
1812
|
persistenceOverride?: WorkerPersistenceConfig
|
|
1476
1813
|
): Promise<WorkerRecord | null> => {
|
|
1477
1814
|
if (!persistenceOverride) {
|
|
1478
|
-
|
|
1479
|
-
|
|
1815
|
+
if (!isCloudflareRuntime()) {
|
|
1816
|
+
await ensureWorkerStoreConfigured();
|
|
1817
|
+
return workerStore.get(name);
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
const store = await getDefaultStoreForRuntime();
|
|
1821
|
+
try {
|
|
1822
|
+
return await store.get(name);
|
|
1823
|
+
} finally {
|
|
1824
|
+
if (store.close) {
|
|
1825
|
+
await store.close();
|
|
1826
|
+
}
|
|
1827
|
+
}
|
|
1480
1828
|
}
|
|
1481
1829
|
|
|
1482
1830
|
const store = await resolveWorkerStoreForPersistence(persistenceOverride);
|
|
@@ -1934,6 +2282,37 @@ export const WorkerFactory = Object.freeze({
|
|
|
1934
2282
|
resolveProcessorPath,
|
|
1935
2283
|
resolveProcessorSpec,
|
|
1936
2284
|
|
|
2285
|
+
/**
|
|
2286
|
+
* Register a new worker configuration without starting it.
|
|
2287
|
+
*/
|
|
2288
|
+
async register(config: WorkerFactoryConfig): Promise<void> {
|
|
2289
|
+
const { name } = config;
|
|
2290
|
+
// Check in-memory first (though unlikely if we are just registering)
|
|
2291
|
+
if (workers.has(name)) {
|
|
2292
|
+
throw ErrorFactory.createWorkerError(`Worker "${name}" is already running locally`);
|
|
2293
|
+
}
|
|
2294
|
+
|
|
2295
|
+
const store = await getStoreForWorker(config);
|
|
2296
|
+
try {
|
|
2297
|
+
const existing = await store.get(name);
|
|
2298
|
+
if (existing) {
|
|
2299
|
+
throw ErrorFactory.createWorkerError(`Worker "${name}" already exists in persistence`);
|
|
2300
|
+
}
|
|
2301
|
+
|
|
2302
|
+
// Init features to validate config, but mainly we just want to save it.
|
|
2303
|
+
// initializeWorkerFeatures might rely on being active or having resources, so we might skip it or do partial.
|
|
2304
|
+
// For now, just save definition.
|
|
2305
|
+
// Status should be STOPPED or CREATED.
|
|
2306
|
+
await store.save(buildWorkerRecord(config, WorkerCreationStatus.STOPPED));
|
|
2307
|
+
Logger.info(`Worker registered (persistence only): ${name}`);
|
|
2308
|
+
} finally {
|
|
2309
|
+
// If Cloudflare environment, try to close store connection to avoid zombie connections
|
|
2310
|
+
if (Cloudflare.getWorkersEnv() !== null && store.close) {
|
|
2311
|
+
await store.close();
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2314
|
+
},
|
|
2315
|
+
|
|
1937
2316
|
/**
|
|
1938
2317
|
* Create new worker with full setup
|
|
1939
2318
|
*/
|
|
@@ -2398,9 +2777,25 @@ export const WorkerFactory = Object.freeze({
|
|
|
2398
2777
|
): Promise<WorkerRecord[]> {
|
|
2399
2778
|
const includeInactive = options?.includeInactive === true;
|
|
2400
2779
|
if (!persistenceOverride) {
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2780
|
+
if (!isCloudflareRuntime()) {
|
|
2781
|
+
await ensureWorkerStoreConfigured();
|
|
2782
|
+
const records = await workerStore.list(options);
|
|
2783
|
+
return includeInactive
|
|
2784
|
+
? records
|
|
2785
|
+
: records.filter((record) => record.activeStatus !== false);
|
|
2786
|
+
}
|
|
2787
|
+
|
|
2788
|
+
const store = await getDefaultStoreForRuntime();
|
|
2789
|
+
try {
|
|
2790
|
+
const records = await store.list(options);
|
|
2791
|
+
return includeInactive
|
|
2792
|
+
? records
|
|
2793
|
+
: records.filter((record) => record.activeStatus !== false);
|
|
2794
|
+
} finally {
|
|
2795
|
+
if (store.close) {
|
|
2796
|
+
await store.close();
|
|
2797
|
+
}
|
|
2798
|
+
}
|
|
2404
2799
|
}
|
|
2405
2800
|
|
|
2406
2801
|
const store = await resolveWorkerStoreForPersistence(persistenceOverride);
|
|
@@ -2465,7 +2860,15 @@ export const WorkerFactory = Object.freeze({
|
|
|
2465
2860
|
): Promise<WorkerRecord | null> {
|
|
2466
2861
|
const instance = workers.get(name);
|
|
2467
2862
|
const store = await getStoreForWorker(instance?.config, persistenceOverride);
|
|
2468
|
-
|
|
2863
|
+
|
|
2864
|
+
try {
|
|
2865
|
+
const result = await store.get(name);
|
|
2866
|
+
return result;
|
|
2867
|
+
} finally {
|
|
2868
|
+
if (Cloudflare.getWorkersEnv() !== null && store.close) {
|
|
2869
|
+
await store.close();
|
|
2870
|
+
}
|
|
2871
|
+
}
|
|
2469
2872
|
},
|
|
2470
2873
|
|
|
2471
2874
|
/**
|