@zintrust/workers 0.1.31 → 0.1.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ClusterLock.js +3 -2
- package/dist/DeadLetterQueue.js +3 -2
- package/dist/HealthMonitor.js +24 -13
- package/dist/Observability.js +8 -0
- package/dist/WorkerFactory.d.ts +4 -0
- package/dist/WorkerFactory.js +384 -42
- package/dist/WorkerInit.js +122 -43
- package/dist/WorkerMetrics.js +5 -1
- package/dist/WorkerRegistry.js +8 -0
- package/dist/WorkerShutdown.d.ts +0 -13
- package/dist/WorkerShutdown.js +1 -44
- package/dist/build-manifest.json +99 -83
- package/dist/config/workerConfig.d.ts +1 -0
- package/dist/config/workerConfig.js +7 -1
- package/dist/createQueueWorker.js +281 -42
- package/dist/dashboard/workers-api.js +8 -1
- package/dist/http/WorkerController.js +90 -35
- package/dist/http/WorkerMonitoringService.js +29 -2
- package/dist/index.d.ts +1 -2
- package/dist/index.js +0 -1
- package/dist/routes/workers.js +10 -7
- package/dist/storage/WorkerStore.d.ts +6 -3
- package/dist/storage/WorkerStore.js +16 -0
- package/dist/telemetry/api/TelemetryMonitoringService.js +29 -2
- package/dist/ui/router/ui.js +58 -29
- package/dist/ui/workers/index.html +202 -0
- package/dist/ui/workers/main.js +1952 -0
- package/dist/ui/workers/styles.css +1350 -0
- package/dist/ui/workers/zintrust.svg +30 -0
- package/package.json +5 -5
- package/src/ClusterLock.ts +13 -7
- package/src/ComplianceManager.ts +3 -2
- package/src/DeadLetterQueue.ts +6 -4
- package/src/HealthMonitor.ts +33 -17
- package/src/Observability.ts +11 -0
- package/src/WorkerFactory.ts +446 -43
- package/src/WorkerInit.ts +167 -48
- package/src/WorkerMetrics.ts +14 -8
- package/src/WorkerRegistry.ts +11 -0
- package/src/WorkerShutdown.ts +1 -69
- package/src/config/workerConfig.ts +9 -1
- package/src/createQueueWorker.ts +428 -43
- package/src/dashboard/workers-api.ts +8 -1
- package/src/http/WorkerController.ts +111 -36
- package/src/http/WorkerMonitoringService.ts +35 -2
- package/src/index.ts +2 -3
- package/src/routes/workers.ts +10 -8
- package/src/storage/WorkerStore.ts +21 -3
- package/src/telemetry/api/TelemetryMonitoringService.ts +35 -2
- package/src/types/queue-monitor.d.ts +2 -1
- package/src/ui/router/EmbeddedAssets.ts +3 -0
- package/src/ui/router/ui.ts +57 -39
- package/src/WorkerShutdownDurableObject.ts +0 -64
package/dist/WorkerFactory.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Central factory for creating workers with all advanced features
|
|
4
4
|
* Sealed namespace for immutability
|
|
5
5
|
*/
|
|
6
|
-
import {
|
|
6
|
+
import { Cloudflare, createRedisConnection, databaseConfig, Env, ErrorFactory, generateUuid, getBullMQSafeQueueName, JobStateTracker, Logger, NodeSingletons, queueConfig, registerDatabasesFromRuntimeConfig, useEnsureDbConnected, workersConfig, ZintrustLang, } from '@zintrust/core';
|
|
7
7
|
import { Worker } from 'bullmq';
|
|
8
8
|
import { AutoScaler } from './AutoScaler';
|
|
9
9
|
import { CanaryController } from './CanaryController';
|
|
@@ -21,8 +21,157 @@ import { ResourceMonitor } from './ResourceMonitor';
|
|
|
21
21
|
import { WorkerMetrics } from './WorkerMetrics';
|
|
22
22
|
import { WorkerRegistry } from './WorkerRegistry';
|
|
23
23
|
import { WorkerVersioning } from './WorkerVersioning';
|
|
24
|
+
import { keyPrefix } from './config/workerConfig';
|
|
24
25
|
import { DbWorkerStore, InMemoryWorkerStore, RedisWorkerStore, } from './storage/WorkerStore';
|
|
25
26
|
const path = NodeSingletons.path;
|
|
27
|
+
const isNodeRuntime = () => typeof process !== 'undefined' && Boolean(process.versions?.node);
|
|
28
|
+
const resolveProjectRoot = () => {
|
|
29
|
+
const envRoot = Env.get('ZINTRUST_PROJECT_ROOT', '').trim();
|
|
30
|
+
return envRoot.length > 0 ? envRoot : process.cwd();
|
|
31
|
+
};
|
|
32
|
+
const canUseProjectFileImports = () => typeof NodeSingletons?.fs?.writeFileSync === 'function' &&
|
|
33
|
+
typeof NodeSingletons?.fs?.mkdirSync === 'function' &&
|
|
34
|
+
typeof NodeSingletons?.fs?.existsSync === 'function' &&
|
|
35
|
+
typeof NodeSingletons?.url?.pathToFileURL === 'function' &&
|
|
36
|
+
typeof NodeSingletons?.path?.join === 'function';
|
|
37
|
+
const buildCandidatesForSpecifier = (specifier, root) => {
|
|
38
|
+
if (specifier === '@zintrust/core') {
|
|
39
|
+
return [
|
|
40
|
+
path.join(root, 'dist', 'src', 'index.js'),
|
|
41
|
+
path.join(root, 'dist', 'index.js'),
|
|
42
|
+
path.join(root, 'src', 'index.ts'),
|
|
43
|
+
];
|
|
44
|
+
}
|
|
45
|
+
if (specifier === '@zintrust/workers') {
|
|
46
|
+
return [
|
|
47
|
+
path.join(root, 'dist', 'packages', 'workers', 'src', 'index.js'),
|
|
48
|
+
path.join(root, 'packages', 'workers', 'src', 'index.ts'),
|
|
49
|
+
];
|
|
50
|
+
}
|
|
51
|
+
return [];
|
|
52
|
+
};
|
|
53
|
+
const getProjectFileCandidates = (paths) => {
|
|
54
|
+
if (!canUseProjectFileImports())
|
|
55
|
+
return null;
|
|
56
|
+
for (const candidate of paths) {
|
|
57
|
+
if (NodeSingletons.fs.existsSync(candidate))
|
|
58
|
+
return candidate;
|
|
59
|
+
}
|
|
60
|
+
return null;
|
|
61
|
+
};
|
|
62
|
+
const resolveLocalPackageFallback = (specifier) => {
|
|
63
|
+
if (!canUseProjectFileImports())
|
|
64
|
+
return null;
|
|
65
|
+
const root = resolveProjectRoot();
|
|
66
|
+
const candidates = buildCandidatesForSpecifier(specifier, root);
|
|
67
|
+
const resolved = getProjectFileCandidates(candidates);
|
|
68
|
+
if (!resolved)
|
|
69
|
+
return null;
|
|
70
|
+
return NodeSingletons.url.pathToFileURL(resolved).href;
|
|
71
|
+
};
|
|
72
|
+
const resolvePackageSpecifierUrl = (specifier) => {
|
|
73
|
+
if (!isNodeRuntime() || !canUseProjectFileImports())
|
|
74
|
+
return null;
|
|
75
|
+
if (typeof NodeSingletons?.module?.createRequire !== 'function') {
|
|
76
|
+
return resolveLocalPackageFallback(specifier);
|
|
77
|
+
}
|
|
78
|
+
try {
|
|
79
|
+
const require = NodeSingletons.module.createRequire(import.meta.url);
|
|
80
|
+
const resolved = require.resolve(specifier);
|
|
81
|
+
if (specifier === '@zintrust/workers' &&
|
|
82
|
+
resolved.includes(`${path.sep}node_modules${path.sep}@zintrust${path.sep}workers${path.sep}`)) {
|
|
83
|
+
const local = resolveLocalPackageFallback(specifier);
|
|
84
|
+
if (local)
|
|
85
|
+
return local;
|
|
86
|
+
}
|
|
87
|
+
return NodeSingletons.url.pathToFileURL(resolved).href;
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
return resolveLocalPackageFallback(specifier);
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
const escapeRegExp = (value) => value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw `\$&`);
|
|
94
|
+
const rewriteProcessorImports = (code) => {
|
|
95
|
+
const replacements = [];
|
|
96
|
+
const coreUrl = resolvePackageSpecifierUrl('@zintrust/core');
|
|
97
|
+
if (coreUrl)
|
|
98
|
+
replacements.push({ from: '@zintrust/core', to: coreUrl });
|
|
99
|
+
const workersUrl = resolvePackageSpecifierUrl('@zintrust/workers');
|
|
100
|
+
if (workersUrl)
|
|
101
|
+
replacements.push({ from: '@zintrust/workers', to: workersUrl });
|
|
102
|
+
if (replacements.length === 0)
|
|
103
|
+
return code;
|
|
104
|
+
let updated = code;
|
|
105
|
+
for (const { from, to } of replacements) {
|
|
106
|
+
const pattern = new RegExp(String.raw `(['"])${escapeRegExp(from)}\1`, 'g');
|
|
107
|
+
updated = updated.replace(pattern, `$1${to}$1`);
|
|
108
|
+
}
|
|
109
|
+
return updated;
|
|
110
|
+
};
|
|
111
|
+
const ensureProcessorSpecDir = () => {
|
|
112
|
+
if (!isNodeRuntime() || !canUseProjectFileImports())
|
|
113
|
+
return null;
|
|
114
|
+
const dir = path.join(resolveProjectRoot(), '.zintrust', 'processor-specs');
|
|
115
|
+
try {
|
|
116
|
+
if (!NodeSingletons.fs.existsSync(dir)) {
|
|
117
|
+
NodeSingletons.fs.mkdirSync(dir, { recursive: true });
|
|
118
|
+
}
|
|
119
|
+
return dir;
|
|
120
|
+
}
|
|
121
|
+
catch (error) {
|
|
122
|
+
Logger.debug('Failed to prepare processor spec cache directory', error);
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
const shouldFallbackToFileImport = (error) => {
|
|
127
|
+
if (!isNodeRuntime())
|
|
128
|
+
return false;
|
|
129
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
130
|
+
const code = error?.code ?? '';
|
|
131
|
+
if (code === 'ERR_INVALID_URL' || code === 'ERR_UNSUPPORTED_ESM_URL_SCHEME')
|
|
132
|
+
return true;
|
|
133
|
+
return (message.includes('Invalid relative URL') ||
|
|
134
|
+
message.includes('base scheme is not hierarchical') ||
|
|
135
|
+
message.includes('Failed to resolve module specifier'));
|
|
136
|
+
};
|
|
137
|
+
const importModuleFromCode = async (params) => {
|
|
138
|
+
const { code, normalized, cacheKey } = params;
|
|
139
|
+
const dataUrl = `data:text/javascript;base64,${toBase64(code)}`;
|
|
140
|
+
try {
|
|
141
|
+
return (await import(dataUrl));
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
if (!shouldFallbackToFileImport(error))
|
|
145
|
+
throw error;
|
|
146
|
+
const dir = ensureProcessorSpecDir();
|
|
147
|
+
if (!dir)
|
|
148
|
+
throw error;
|
|
149
|
+
try {
|
|
150
|
+
const codeHash = await computeSha256(code);
|
|
151
|
+
const filePath = path.join(dir, `${codeHash || cacheKey}.mjs`);
|
|
152
|
+
NodeSingletons.fs.writeFileSync(filePath, code, 'utf8');
|
|
153
|
+
const fileUrl = NodeSingletons.url.pathToFileURL(filePath).href;
|
|
154
|
+
return (await import(fileUrl));
|
|
155
|
+
}
|
|
156
|
+
catch (fileError) {
|
|
157
|
+
Logger.debug(`Processor URL file fallback failed for ${normalized}`, fileError);
|
|
158
|
+
throw error;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
const isCloudflareRuntime = () => Cloudflare.getWorkersEnv() !== null;
|
|
163
|
+
const getDefaultStoreForRuntime = async () => {
|
|
164
|
+
if (!isCloudflareRuntime()) {
|
|
165
|
+
await ensureWorkerStoreConfigured();
|
|
166
|
+
return workerStore;
|
|
167
|
+
}
|
|
168
|
+
const bootstrapConfig = buildPersistenceBootstrapConfig();
|
|
169
|
+
const persistence = resolvePersistenceConfig(bootstrapConfig);
|
|
170
|
+
if (!persistence) {
|
|
171
|
+
return InMemoryWorkerStore.create();
|
|
172
|
+
}
|
|
173
|
+
return resolveWorkerStoreForPersistence(persistence);
|
|
174
|
+
};
|
|
26
175
|
const getStoreForWorker = async (config, persistenceOverride) => {
|
|
27
176
|
if (persistenceOverride) {
|
|
28
177
|
return resolveWorkerStoreForPersistence(persistenceOverride);
|
|
@@ -35,8 +184,7 @@ const getStoreForWorker = async (config, persistenceOverride) => {
|
|
|
35
184
|
}
|
|
36
185
|
}
|
|
37
186
|
// Fallback to default/global store
|
|
38
|
-
|
|
39
|
-
return workerStore;
|
|
187
|
+
return getDefaultStoreForRuntime();
|
|
40
188
|
};
|
|
41
189
|
const validateAndGetStore = async (name, config, persistenceOverride) => {
|
|
42
190
|
const store = await getStoreForWorker(config, persistenceOverride);
|
|
@@ -153,7 +301,7 @@ const computeSha256 = async (value) => {
|
|
|
153
301
|
if (typeof NodeSingletons.createHash === 'function') {
|
|
154
302
|
return NodeSingletons.createHash('sha256').update(value).digest('hex');
|
|
155
303
|
}
|
|
156
|
-
return String(
|
|
304
|
+
return String(generateUuid()).slice(2);
|
|
157
305
|
};
|
|
158
306
|
const toBase64 = (value) => {
|
|
159
307
|
if (typeof Buffer !== 'undefined') {
|
|
@@ -267,6 +415,18 @@ const sanitizeProcessorPath = (value) => {
|
|
|
267
415
|
};
|
|
268
416
|
const stripProcessorExtension = (value) => value.replace(/\.(ts|js)$/i, '');
|
|
269
417
|
const normalizeModulePath = (value) => value.replaceAll('\\', '/');
|
|
418
|
+
const filterExistingFileCandidates = (candidates) => {
|
|
419
|
+
if (!NodeSingletons?.fs?.existsSync)
|
|
420
|
+
return candidates;
|
|
421
|
+
return candidates.filter((candidate) => {
|
|
422
|
+
try {
|
|
423
|
+
return NodeSingletons.fs.existsSync(candidate);
|
|
424
|
+
}
|
|
425
|
+
catch {
|
|
426
|
+
return false;
|
|
427
|
+
}
|
|
428
|
+
});
|
|
429
|
+
};
|
|
270
430
|
const buildProcessorModuleCandidates = (modulePath, resolvedPath) => {
|
|
271
431
|
const candidates = [];
|
|
272
432
|
const normalized = normalizeModulePath(modulePath.trim());
|
|
@@ -286,6 +446,22 @@ const buildProcessorModuleCandidates = (modulePath, resolvedPath) => {
|
|
|
286
446
|
}
|
|
287
447
|
return Array.from(new Set(candidates));
|
|
288
448
|
};
|
|
449
|
+
const buildProcessorFilePathCandidates = (_modulePath, resolvedPath) => {
|
|
450
|
+
const candidates = [];
|
|
451
|
+
const normalizedResolved = normalizeModulePath(resolvedPath);
|
|
452
|
+
const projectRoot = normalizeModulePath(resolveProjectRoot());
|
|
453
|
+
const strippedResolved = stripProcessorExtension(resolvedPath);
|
|
454
|
+
candidates.push(`${strippedResolved}.js`, `${strippedResolved}.mjs`);
|
|
455
|
+
const appIndex = normalizedResolved.lastIndexOf('/app/');
|
|
456
|
+
if (appIndex !== -1) {
|
|
457
|
+
const relative = normalizedResolved.slice(appIndex + 5);
|
|
458
|
+
if (relative) {
|
|
459
|
+
const strippedRelative = stripProcessorExtension(relative);
|
|
460
|
+
candidates.push(path.join(projectRoot, 'dist', 'app', `${strippedRelative}.js`), path.join(projectRoot, 'app', relative), path.join(projectRoot, 'app', `${strippedRelative}.js`), path.join('/app', 'dist', 'app', `${strippedRelative}.js`));
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
return filterExistingFileCandidates(Array.from(new Set(candidates)));
|
|
464
|
+
};
|
|
289
465
|
const pickProcessorFromModule = (mod, source) => {
|
|
290
466
|
const candidate = mod?.['default'] ?? mod?.['processor'] ?? mod?.['handler'] ?? mod?.['handle'];
|
|
291
467
|
if (typeof candidate !== 'function') {
|
|
@@ -328,9 +504,9 @@ const refreshCachedProcessor = (existing, config, cacheControl) => {
|
|
|
328
504
|
};
|
|
329
505
|
const cacheProcessorFromResponse = async (params) => {
|
|
330
506
|
const { response, normalized, config, cacheKey } = params;
|
|
331
|
-
const
|
|
332
|
-
const
|
|
333
|
-
const mod = await
|
|
507
|
+
const rawCode = await readResponseBody(response, config.fetchMaxSizeBytes);
|
|
508
|
+
const code = rewriteProcessorImports(rawCode);
|
|
509
|
+
const mod = await importModuleFromCode({ code, normalized, cacheKey });
|
|
334
510
|
const processor = extractZinTrustProcessor(mod, normalized);
|
|
335
511
|
if (!processor) {
|
|
336
512
|
throw ErrorFactory.createConfigError('INVALID_PROCESSOR_URL_EXPORT');
|
|
@@ -415,9 +591,11 @@ const resolveProcessorFromUrl = async (spec) => {
|
|
|
415
591
|
}
|
|
416
592
|
if (parsed.protocol !== 'https:' && parsed.protocol !== 'file:') {
|
|
417
593
|
Logger.warn(`Invalid processor URL protocol: ${parsed.protocol}. Only https:// and file:// are supported.`);
|
|
594
|
+
return undefined;
|
|
418
595
|
}
|
|
419
596
|
if (!isAllowedRemoteHost(parsed.host) && parsed.protocol !== 'file:') {
|
|
420
597
|
Logger.warn(`Invalid processor URL host: ${parsed.host}. Host is not in the allowlist.`);
|
|
598
|
+
return undefined;
|
|
421
599
|
}
|
|
422
600
|
const config = getProcessorSpecConfig();
|
|
423
601
|
const cacheKey = await computeSha256(normalized);
|
|
@@ -456,8 +634,11 @@ const resolveProcessorFromPath = async (modulePath) => {
|
|
|
456
634
|
return undefined;
|
|
457
635
|
const [candidatePath, ...rest] = candidates;
|
|
458
636
|
try {
|
|
459
|
-
const
|
|
460
|
-
|
|
637
|
+
const importPath = candidatePath.startsWith('/') && !candidatePath.startsWith('//')
|
|
638
|
+
? NodeSingletons.url.pathToFileURL(candidatePath).href
|
|
639
|
+
: candidatePath;
|
|
640
|
+
const mod = await import(importPath);
|
|
641
|
+
const candidate = pickProcessorFromModule(mod, importPath);
|
|
461
642
|
if (candidate)
|
|
462
643
|
return candidate;
|
|
463
644
|
}
|
|
@@ -473,10 +654,14 @@ const resolveProcessorFromPath = async (modulePath) => {
|
|
|
473
654
|
return candidate;
|
|
474
655
|
}
|
|
475
656
|
catch (err) {
|
|
476
|
-
const
|
|
477
|
-
const
|
|
478
|
-
if (
|
|
479
|
-
return
|
|
657
|
+
const fileCandidates = buildProcessorFilePathCandidates(trimmed, resolved);
|
|
658
|
+
const resolvedFileCandidate = await importProcessorFromCandidates(fileCandidates);
|
|
659
|
+
if (resolvedFileCandidate)
|
|
660
|
+
return resolvedFileCandidate;
|
|
661
|
+
const moduleCandidates = buildProcessorModuleCandidates(trimmed, resolved);
|
|
662
|
+
const resolvedModuleCandidate = await importProcessorFromCandidates(moduleCandidates);
|
|
663
|
+
if (resolvedModuleCandidate)
|
|
664
|
+
return resolvedModuleCandidate;
|
|
480
665
|
Logger.error(`Failed to import processor from path: ${resolved}`, err);
|
|
481
666
|
}
|
|
482
667
|
return undefined;
|
|
@@ -701,6 +886,56 @@ const handleFailure = async (params) => {
|
|
|
701
886
|
await executeAllFailureHandlers(params);
|
|
702
887
|
await executeFailurePlugins(workerName, job, error, features);
|
|
703
888
|
};
|
|
889
|
+
const toBackoffDelayMs = (backoff) => {
|
|
890
|
+
if (typeof backoff === 'number' && Number.isFinite(backoff)) {
|
|
891
|
+
return Math.max(0, Math.floor(backoff));
|
|
892
|
+
}
|
|
893
|
+
if (backoff !== null && backoff !== undefined && typeof backoff === 'object') {
|
|
894
|
+
const raw = backoff.delay;
|
|
895
|
+
if (typeof raw === 'number' && Number.isFinite(raw)) {
|
|
896
|
+
return Math.max(0, Math.floor(raw));
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
return 0;
|
|
900
|
+
};
|
|
901
|
+
const trackJobStarted = async (input) => {
|
|
902
|
+
if (!input.job.id)
|
|
903
|
+
return;
|
|
904
|
+
await JobStateTracker.started({
|
|
905
|
+
queueName: input.queueName,
|
|
906
|
+
jobId: input.job.id,
|
|
907
|
+
attempts: input.attempts,
|
|
908
|
+
timeoutMs: Math.max(1000, Env.getInt('QUEUE_JOB_TIMEOUT', 60) * 1000),
|
|
909
|
+
workerName: input.workerName,
|
|
910
|
+
workerVersion: input.workerVersion,
|
|
911
|
+
});
|
|
912
|
+
};
|
|
913
|
+
const trackJobCompleted = async (input) => {
|
|
914
|
+
if (!input.job.id)
|
|
915
|
+
return;
|
|
916
|
+
await JobStateTracker.completed({
|
|
917
|
+
queueName: input.queueName,
|
|
918
|
+
jobId: input.job.id,
|
|
919
|
+
processingTimeMs: input.duration,
|
|
920
|
+
result: input.result,
|
|
921
|
+
});
|
|
922
|
+
};
|
|
923
|
+
const trackJobFailed = async (input) => {
|
|
924
|
+
if (!input.job.id)
|
|
925
|
+
return;
|
|
926
|
+
const isFinal = input.maxAttempts === undefined ? true : input.attempts >= input.maxAttempts;
|
|
927
|
+
const backoffDelayMs = toBackoffDelayMs(input.job.opts?.backoff);
|
|
928
|
+
await JobStateTracker.failed({
|
|
929
|
+
queueName: input.queueName,
|
|
930
|
+
jobId: input.job.id,
|
|
931
|
+
attempts: input.attempts,
|
|
932
|
+
isFinal,
|
|
933
|
+
retryAt: !isFinal && backoffDelayMs > 0
|
|
934
|
+
? new Date(Date.now() + backoffDelayMs).toISOString()
|
|
935
|
+
: undefined,
|
|
936
|
+
error: input.error,
|
|
937
|
+
});
|
|
938
|
+
};
|
|
704
939
|
/**
|
|
705
940
|
* Helper: Create enhanced processor with all features
|
|
706
941
|
*/
|
|
@@ -719,8 +954,19 @@ const createEnhancedProcessor = (config) => {
|
|
|
719
954
|
const startTime = Date.now();
|
|
720
955
|
let result;
|
|
721
956
|
let spanId = null;
|
|
957
|
+
const maxAttempts = typeof job.opts?.attempts === 'number' && Number.isFinite(job.opts.attempts)
|
|
958
|
+
? Math.max(1, Math.floor(job.opts.attempts))
|
|
959
|
+
: undefined;
|
|
960
|
+
const attempts = Math.max(1, Math.floor((job.attemptsMade ?? 0) + 1));
|
|
722
961
|
try {
|
|
723
962
|
spanId = startProcessingSpan(name, jobVersion, job, config.queueName, features);
|
|
963
|
+
await trackJobStarted({
|
|
964
|
+
queueName: config.queueName,
|
|
965
|
+
job,
|
|
966
|
+
attempts,
|
|
967
|
+
workerName: name,
|
|
968
|
+
workerVersion: jobVersion,
|
|
969
|
+
});
|
|
724
970
|
// Process the job
|
|
725
971
|
result = await processor(job);
|
|
726
972
|
const duration = Date.now() - startTime;
|
|
@@ -733,11 +979,19 @@ const createEnhancedProcessor = (config) => {
|
|
|
733
979
|
spanId,
|
|
734
980
|
features,
|
|
735
981
|
});
|
|
982
|
+
await trackJobCompleted({ queueName: config.queueName, job, duration, result });
|
|
736
983
|
return result;
|
|
737
984
|
}
|
|
738
985
|
catch (err) {
|
|
739
986
|
const error = err;
|
|
740
987
|
const duration = Date.now() - startTime;
|
|
988
|
+
await trackJobFailed({
|
|
989
|
+
queueName: config.queueName,
|
|
990
|
+
job,
|
|
991
|
+
attempts,
|
|
992
|
+
maxAttempts,
|
|
993
|
+
error,
|
|
994
|
+
});
|
|
741
995
|
await handleFailure({
|
|
742
996
|
workerName: name,
|
|
743
997
|
jobVersion,
|
|
@@ -801,12 +1055,22 @@ const resolveRedisConfigFromEnv = (config, context) => {
|
|
|
801
1055
|
password: password || undefined,
|
|
802
1056
|
};
|
|
803
1057
|
};
|
|
804
|
-
const resolveRedisConfigFromDirect = (config, context) =>
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
}
|
|
1058
|
+
const resolveRedisConfigFromDirect = (config, context) => {
|
|
1059
|
+
const fallbackDb = Env.getInt('REDIS_QUEUE_DB', ZintrustLang.REDIS_DEFAULT_DB);
|
|
1060
|
+
let normalizedDb = fallbackDb;
|
|
1061
|
+
if (typeof config.db === 'number') {
|
|
1062
|
+
normalizedDb = config.db;
|
|
1063
|
+
}
|
|
1064
|
+
else if (typeof config.database === 'number') {
|
|
1065
|
+
normalizedDb = config.database;
|
|
1066
|
+
}
|
|
1067
|
+
return {
|
|
1068
|
+
host: requireRedisHost(config.host, context),
|
|
1069
|
+
port: config.port,
|
|
1070
|
+
db: normalizedDb,
|
|
1071
|
+
password: config.password ?? Env.get('REDIS_PASSWORD', undefined),
|
|
1072
|
+
};
|
|
1073
|
+
};
|
|
810
1074
|
const resolveRedisConfig = (config, context) => isRedisEnvConfig(config)
|
|
811
1075
|
? resolveRedisConfigFromEnv(config, context)
|
|
812
1076
|
: resolveRedisConfigFromDirect(config, context);
|
|
@@ -817,17 +1081,21 @@ const resolveRedisConfigWithFallback = (primary, fallback, errorMessage, context
|
|
|
817
1081
|
}
|
|
818
1082
|
return resolveRedisConfig(selected, context);
|
|
819
1083
|
};
|
|
1084
|
+
const logRedisPersistenceConfig = (redisConfig, key_prefix, source) => {
|
|
1085
|
+
Logger.debug('Worker persistence redis config', {
|
|
1086
|
+
source,
|
|
1087
|
+
host: redisConfig.host,
|
|
1088
|
+
port: redisConfig.port,
|
|
1089
|
+
db: redisConfig.db,
|
|
1090
|
+
key_prefix,
|
|
1091
|
+
});
|
|
1092
|
+
};
|
|
820
1093
|
const normalizeEnvValue = (value) => {
|
|
821
1094
|
if (!value)
|
|
822
1095
|
return undefined;
|
|
823
1096
|
const trimmed = value.trim();
|
|
824
1097
|
return trimmed.length > 0 ? trimmed : undefined;
|
|
825
1098
|
};
|
|
826
|
-
const normalizeAppName = (value) => {
|
|
827
|
-
const normalized = (value ?? '').trim().replaceAll(/\s+/g, '_');
|
|
828
|
-
return normalized.length > 0 ? normalized : 'zintrust';
|
|
829
|
-
};
|
|
830
|
-
const resolveDefaultRedisKeyPrefix = () => 'worker_' + normalizeAppName(appConfig.prefix);
|
|
831
1099
|
const resolveDefaultPersistenceTable = () => normalizeEnvValue(Env.get('WORKER_PERSISTENCE_TABLE', 'zintrust_workers')) ?? 'zintrust_workers';
|
|
832
1100
|
const resolveDefaultPersistenceConnection = () => normalizeEnvValue(Env.get('WORKER_PERSISTENCE_DB_CONNECTION', 'default')) ?? 'default';
|
|
833
1101
|
const resolveAutoStart = (config) => {
|
|
@@ -845,9 +1113,7 @@ const normalizeExplicitPersistence = (persistence) => {
|
|
|
845
1113
|
return {
|
|
846
1114
|
driver: 'redis',
|
|
847
1115
|
redis: persistence.redis,
|
|
848
|
-
keyPrefix:
|
|
849
|
-
normalizeEnvValue(Env.get('WORKER_PERSISTENCE_REDIS_KEY_PREFIX', '')) ??
|
|
850
|
-
resolveDefaultRedisKeyPrefix(),
|
|
1116
|
+
keyPrefix: keyPrefix(),
|
|
851
1117
|
};
|
|
852
1118
|
}
|
|
853
1119
|
const clientIsConnection = typeof persistence.client === 'string';
|
|
@@ -875,11 +1141,15 @@ const resolvePersistenceConfig = (config) => {
|
|
|
875
1141
|
if (driver === 'memory')
|
|
876
1142
|
return { driver: 'memory' };
|
|
877
1143
|
if (driver === 'redis') {
|
|
878
|
-
const
|
|
1144
|
+
const persistenceDbOverride = normalizeEnvValue(Env.get('WORKER_PERSISTENCE_REDIS_DB', ''));
|
|
879
1145
|
return {
|
|
880
1146
|
driver: 'redis',
|
|
881
|
-
|
|
882
|
-
|
|
1147
|
+
// Optional override; otherwise defaults to REDIS_QUEUE_DB.
|
|
1148
|
+
redis: {
|
|
1149
|
+
env: true,
|
|
1150
|
+
db: persistenceDbOverride ? 'WORKER_PERSISTENCE_REDIS_DB' : 'REDIS_QUEUE_DB',
|
|
1151
|
+
},
|
|
1152
|
+
keyPrefix: keyPrefix(),
|
|
883
1153
|
};
|
|
884
1154
|
}
|
|
885
1155
|
if (driver === 'db' || driver === 'database') {
|
|
@@ -918,8 +1188,10 @@ const resolveWorkerStore = async (config) => {
|
|
|
918
1188
|
}
|
|
919
1189
|
else if (persistence.driver === 'redis') {
|
|
920
1190
|
const redisConfig = resolveRedisConfigWithFallback(persistence.redis, config.infrastructure?.redis, 'Worker persistence requires redis config (persistence.redis or infrastructure.redis)', 'infrastructure.persistence.redis');
|
|
1191
|
+
const key_prefix = persistence.keyPrefix ?? keyPrefix();
|
|
1192
|
+
logRedisPersistenceConfig(redisConfig, key_prefix, 'resolveWorkerStore');
|
|
921
1193
|
const client = createRedisConnection(redisConfig);
|
|
922
|
-
next = RedisWorkerStore.create(client,
|
|
1194
|
+
next = RedisWorkerStore.create(client, key_prefix);
|
|
923
1195
|
}
|
|
924
1196
|
else if (persistence.driver === 'database') {
|
|
925
1197
|
const explicitConnection = typeof persistence.client === 'string' ? persistence.client : persistence.connection;
|
|
@@ -960,8 +1232,10 @@ const createWorkerStore = async (persistence) => {
|
|
|
960
1232
|
}
|
|
961
1233
|
if (persistence.driver === 'redis') {
|
|
962
1234
|
const redisConfig = resolveRedisConfigWithFallback(persistence.redis ?? { env: true }, undefined, 'Worker persistence requires redis config (persistence.redis or REDIS_* env values)', 'persistence.redis');
|
|
1235
|
+
const key_prefix = persistence.keyPrefix ?? keyPrefix();
|
|
1236
|
+
logRedisPersistenceConfig(redisConfig, key_prefix, 'createWorkerStore');
|
|
963
1237
|
const client = createRedisConnection(redisConfig);
|
|
964
|
-
return RedisWorkerStore.create(client,
|
|
1238
|
+
return RedisWorkerStore.create(client, key_prefix);
|
|
965
1239
|
}
|
|
966
1240
|
// Database driver
|
|
967
1241
|
const explicitConnection = typeof persistence.client === 'string' ? persistence.client : persistence.connection;
|
|
@@ -972,22 +1246,37 @@ const createWorkerStore = async (persistence) => {
|
|
|
972
1246
|
};
|
|
973
1247
|
const resolveWorkerStoreForPersistence = async (persistence) => {
|
|
974
1248
|
const cacheKey = generateCacheKey(persistence);
|
|
975
|
-
|
|
1249
|
+
const isCloudflare = Cloudflare.getWorkersEnv() !== null;
|
|
1250
|
+
// Return cached instance if available (disable cache for Cloudflare to assume cleanup)
|
|
1251
|
+
// Or handle cleanup differently. For now, disable cache for Cloudflare to allow per-request connections.
|
|
976
1252
|
const cached = storeInstanceCache.get(cacheKey);
|
|
977
|
-
if (cached) {
|
|
1253
|
+
if (cached && !isCloudflare) {
|
|
978
1254
|
return cached;
|
|
979
1255
|
}
|
|
980
1256
|
// Create new store instance
|
|
981
1257
|
const store = await createWorkerStore(persistence);
|
|
982
1258
|
await store.init();
|
|
983
|
-
// Cache the store instance for reuse
|
|
984
|
-
|
|
1259
|
+
// Cache the store instance for reuse only if not Cloudflare
|
|
1260
|
+
if (!isCloudflare) {
|
|
1261
|
+
storeInstanceCache.set(cacheKey, store);
|
|
1262
|
+
}
|
|
985
1263
|
return store;
|
|
986
1264
|
};
|
|
987
1265
|
const getPersistedRecord = async (name, persistenceOverride) => {
|
|
988
1266
|
if (!persistenceOverride) {
|
|
989
|
-
|
|
990
|
-
|
|
1267
|
+
if (!isCloudflareRuntime()) {
|
|
1268
|
+
await ensureWorkerStoreConfigured();
|
|
1269
|
+
return workerStore.get(name);
|
|
1270
|
+
}
|
|
1271
|
+
const store = await getDefaultStoreForRuntime();
|
|
1272
|
+
try {
|
|
1273
|
+
return await store.get(name);
|
|
1274
|
+
}
|
|
1275
|
+
finally {
|
|
1276
|
+
if (store.close) {
|
|
1277
|
+
await store.close();
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
991
1280
|
}
|
|
992
1281
|
const store = await resolveWorkerStoreForPersistence(persistenceOverride);
|
|
993
1282
|
return store.get(name);
|
|
@@ -1359,6 +1648,35 @@ export const WorkerFactory = Object.freeze({
|
|
|
1359
1648
|
registerProcessorSpec,
|
|
1360
1649
|
resolveProcessorPath,
|
|
1361
1650
|
resolveProcessorSpec,
|
|
1651
|
+
/**
|
|
1652
|
+
* Register a new worker configuration without starting it.
|
|
1653
|
+
*/
|
|
1654
|
+
async register(config) {
|
|
1655
|
+
const { name } = config;
|
|
1656
|
+
// Check in-memory first (though unlikely if we are just registering)
|
|
1657
|
+
if (workers.has(name)) {
|
|
1658
|
+
throw ErrorFactory.createWorkerError(`Worker "${name}" is already running locally`);
|
|
1659
|
+
}
|
|
1660
|
+
const store = await getStoreForWorker(config);
|
|
1661
|
+
try {
|
|
1662
|
+
const existing = await store.get(name);
|
|
1663
|
+
if (existing) {
|
|
1664
|
+
throw ErrorFactory.createWorkerError(`Worker "${name}" already exists in persistence`);
|
|
1665
|
+
}
|
|
1666
|
+
// Init features to validate config, but mainly we just want to save it.
|
|
1667
|
+
// initializeWorkerFeatures might rely on being active or having resources, so we might skip it or do partial.
|
|
1668
|
+
// For now, just save definition.
|
|
1669
|
+
// Status should be STOPPED or CREATED.
|
|
1670
|
+
await store.save(buildWorkerRecord(config, WorkerCreationStatus.STOPPED));
|
|
1671
|
+
Logger.info(`Worker registered (persistence only): ${name}`);
|
|
1672
|
+
}
|
|
1673
|
+
finally {
|
|
1674
|
+
// If Cloudflare environment, try to close store connection to avoid zombie connections
|
|
1675
|
+
if (Cloudflare.getWorkersEnv() !== null && store.close) {
|
|
1676
|
+
await store.close();
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
},
|
|
1362
1680
|
/**
|
|
1363
1681
|
* Create new worker with full setup
|
|
1364
1682
|
*/
|
|
@@ -1727,9 +2045,25 @@ export const WorkerFactory = Object.freeze({
|
|
|
1727
2045
|
async listPersistedRecords(persistenceOverride, options) {
|
|
1728
2046
|
const includeInactive = options?.includeInactive === true;
|
|
1729
2047
|
if (!persistenceOverride) {
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
2048
|
+
if (!isCloudflareRuntime()) {
|
|
2049
|
+
await ensureWorkerStoreConfigured();
|
|
2050
|
+
const records = await workerStore.list(options);
|
|
2051
|
+
return includeInactive
|
|
2052
|
+
? records
|
|
2053
|
+
: records.filter((record) => record.activeStatus !== false);
|
|
2054
|
+
}
|
|
2055
|
+
const store = await getDefaultStoreForRuntime();
|
|
2056
|
+
try {
|
|
2057
|
+
const records = await store.list(options);
|
|
2058
|
+
return includeInactive
|
|
2059
|
+
? records
|
|
2060
|
+
: records.filter((record) => record.activeStatus !== false);
|
|
2061
|
+
}
|
|
2062
|
+
finally {
|
|
2063
|
+
if (store.close) {
|
|
2064
|
+
await store.close();
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
1733
2067
|
}
|
|
1734
2068
|
const store = await resolveWorkerStoreForPersistence(persistenceOverride);
|
|
1735
2069
|
const records = await store.list(options);
|
|
@@ -1779,7 +2113,15 @@ export const WorkerFactory = Object.freeze({
|
|
|
1779
2113
|
async getPersisted(name, persistenceOverride) {
|
|
1780
2114
|
const instance = workers.get(name);
|
|
1781
2115
|
const store = await getStoreForWorker(instance?.config, persistenceOverride);
|
|
1782
|
-
|
|
2116
|
+
try {
|
|
2117
|
+
const result = await store.get(name);
|
|
2118
|
+
return result;
|
|
2119
|
+
}
|
|
2120
|
+
finally {
|
|
2121
|
+
if (Cloudflare.getWorkersEnv() !== null && store.close) {
|
|
2122
|
+
await store.close();
|
|
2123
|
+
}
|
|
2124
|
+
}
|
|
1783
2125
|
},
|
|
1784
2126
|
/**
|
|
1785
2127
|
* Remove worker
|