@zintrust/workers 0.1.31 → 0.1.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ClusterLock.js +3 -2
- package/dist/DeadLetterQueue.js +3 -2
- package/dist/HealthMonitor.js +24 -13
- package/dist/Observability.js +8 -0
- package/dist/WorkerFactory.d.ts +4 -0
- package/dist/WorkerFactory.js +409 -42
- package/dist/WorkerInit.js +122 -43
- package/dist/WorkerMetrics.js +5 -1
- package/dist/WorkerRegistry.js +8 -0
- package/dist/WorkerShutdown.d.ts +0 -13
- package/dist/WorkerShutdown.js +1 -44
- package/dist/build-manifest.json +101 -85
- package/dist/config/workerConfig.d.ts +1 -0
- package/dist/config/workerConfig.js +7 -1
- package/dist/createQueueWorker.js +281 -42
- package/dist/dashboard/workers-api.js +8 -1
- package/dist/http/WorkerController.js +90 -35
- package/dist/http/WorkerMonitoringService.js +29 -2
- package/dist/http/middleware/FeaturesValidator.js +5 -4
- package/dist/index.d.ts +1 -2
- package/dist/index.js +0 -1
- package/dist/routes/workers.js +10 -7
- package/dist/storage/WorkerStore.d.ts +6 -3
- package/dist/storage/WorkerStore.js +16 -0
- package/dist/telemetry/api/TelemetryMonitoringService.js +29 -2
- package/dist/ui/router/ui.js +58 -29
- package/dist/ui/workers/index.html +202 -0
- package/dist/ui/workers/main.js +1952 -0
- package/dist/ui/workers/styles.css +1350 -0
- package/dist/ui/workers/zintrust.svg +30 -0
- package/package.json +5 -5
- package/src/ClusterLock.ts +13 -7
- package/src/ComplianceManager.ts +3 -2
- package/src/DeadLetterQueue.ts +6 -4
- package/src/HealthMonitor.ts +33 -17
- package/src/Observability.ts +11 -0
- package/src/WorkerFactory.ts +480 -43
- package/src/WorkerInit.ts +167 -48
- package/src/WorkerMetrics.ts +14 -8
- package/src/WorkerRegistry.ts +11 -0
- package/src/WorkerShutdown.ts +1 -69
- package/src/config/workerConfig.ts +9 -1
- package/src/createQueueWorker.ts +428 -43
- package/src/dashboard/workers-api.ts +8 -1
- package/src/http/WorkerController.ts +111 -36
- package/src/http/WorkerMonitoringService.ts +35 -2
- package/src/http/middleware/FeaturesValidator.ts +8 -19
- package/src/index.ts +2 -3
- package/src/routes/workers.ts +10 -8
- package/src/storage/WorkerStore.ts +21 -3
- package/src/telemetry/api/TelemetryMonitoringService.ts +35 -2
- package/src/types/queue-monitor.d.ts +2 -1
- package/src/ui/components/WorkerExpandPanel.js +0 -8
- package/src/ui/router/EmbeddedAssets.ts +3 -0
- package/src/ui/router/ui.ts +57 -39
- package/src/WorkerShutdownDurableObject.ts +0 -64
package/dist/WorkerFactory.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Central factory for creating workers with all advanced features
|
|
4
4
|
* Sealed namespace for immutability
|
|
5
5
|
*/
|
|
6
|
-
import {
|
|
6
|
+
import { Cloudflare, createRedisConnection, databaseConfig, Env, ErrorFactory, generateUuid, getBullMQSafeQueueName, JobStateTracker, Logger, NodeSingletons, queueConfig, registerDatabasesFromRuntimeConfig, useEnsureDbConnected, workersConfig, ZintrustLang, } from '@zintrust/core';
|
|
7
7
|
import { Worker } from 'bullmq';
|
|
8
8
|
import { AutoScaler } from './AutoScaler';
|
|
9
9
|
import { CanaryController } from './CanaryController';
|
|
@@ -21,8 +21,178 @@ import { ResourceMonitor } from './ResourceMonitor';
|
|
|
21
21
|
import { WorkerMetrics } from './WorkerMetrics';
|
|
22
22
|
import { WorkerRegistry } from './WorkerRegistry';
|
|
23
23
|
import { WorkerVersioning } from './WorkerVersioning';
|
|
24
|
+
import { keyPrefix } from './config/workerConfig';
|
|
24
25
|
import { DbWorkerStore, InMemoryWorkerStore, RedisWorkerStore, } from './storage/WorkerStore';
|
|
25
26
|
const path = NodeSingletons.path;
|
|
27
|
+
const isNodeRuntime = () => typeof process !== 'undefined' && Boolean(process.versions?.node);
|
|
28
|
+
const resolveProjectRoot = () => {
|
|
29
|
+
const envRoot = Env.get('ZINTRUST_PROJECT_ROOT', '').trim();
|
|
30
|
+
return envRoot.length > 0 ? envRoot : process.cwd();
|
|
31
|
+
};
|
|
32
|
+
const canUseProjectFileImports = () => typeof NodeSingletons?.fs?.writeFileSync === 'function' &&
|
|
33
|
+
typeof NodeSingletons?.fs?.mkdirSync === 'function' &&
|
|
34
|
+
typeof NodeSingletons?.fs?.existsSync === 'function' &&
|
|
35
|
+
typeof NodeSingletons?.url?.pathToFileURL === 'function' &&
|
|
36
|
+
typeof NodeSingletons?.path?.join === 'function';
|
|
37
|
+
const buildCandidatesForSpecifier = (specifier, root) => {
|
|
38
|
+
if (specifier === '@zintrust/core') {
|
|
39
|
+
return [
|
|
40
|
+
path.join(root, 'dist', 'src', 'index.js'),
|
|
41
|
+
path.join(root, 'dist', 'index.js'),
|
|
42
|
+
path.join(root, 'node_modules', '@zintrust', 'core', 'dist', 'src', 'index.js'),
|
|
43
|
+
path.join(root, 'node_modules', '@zintrust', 'core', 'dist', 'index.js'),
|
|
44
|
+
path.join(root, 'node_modules', '@zintrust', 'core', 'src', 'index.js'),
|
|
45
|
+
path.join(root, 'node_modules', '@zintrust', 'core', 'index.js'),
|
|
46
|
+
path.join(root, 'src', 'index.ts'),
|
|
47
|
+
];
|
|
48
|
+
}
|
|
49
|
+
if (specifier === '@zintrust/workers') {
|
|
50
|
+
return [
|
|
51
|
+
path.join(root, 'dist', 'packages', 'workers', 'src', 'index.js'),
|
|
52
|
+
path.join(root, 'node_modules', '@zintrust', 'workers', 'dist', 'src', 'index.js'),
|
|
53
|
+
path.join(root, 'node_modules', '@zintrust', 'workers', 'dist', 'index.js'),
|
|
54
|
+
path.join(root, 'node_modules', '@zintrust', 'workers', 'src', 'index.js'),
|
|
55
|
+
path.join(root, 'node_modules', '@zintrust', 'workers', 'index.js'),
|
|
56
|
+
path.join(root, 'packages', 'workers', 'src', 'index.ts'),
|
|
57
|
+
];
|
|
58
|
+
}
|
|
59
|
+
return [];
|
|
60
|
+
};
|
|
61
|
+
const getProjectFileCandidates = (paths) => {
|
|
62
|
+
if (!canUseProjectFileImports())
|
|
63
|
+
return null;
|
|
64
|
+
for (const candidate of paths) {
|
|
65
|
+
if (NodeSingletons.fs.existsSync(candidate))
|
|
66
|
+
return candidate;
|
|
67
|
+
}
|
|
68
|
+
return null;
|
|
69
|
+
};
|
|
70
|
+
const resolveLocalPackageFallback = (specifier) => {
|
|
71
|
+
if (!canUseProjectFileImports())
|
|
72
|
+
return null;
|
|
73
|
+
const root = resolveProjectRoot();
|
|
74
|
+
const candidates = buildCandidatesForSpecifier(specifier, root);
|
|
75
|
+
const resolved = getProjectFileCandidates(candidates);
|
|
76
|
+
if (!resolved)
|
|
77
|
+
return null;
|
|
78
|
+
return NodeSingletons.url.pathToFileURL(resolved).href;
|
|
79
|
+
};
|
|
80
|
+
const resolvePackageSpecifierUrl = (specifier) => {
|
|
81
|
+
if (!isNodeRuntime() || !canUseProjectFileImports())
|
|
82
|
+
return null;
|
|
83
|
+
if (typeof NodeSingletons?.module?.createRequire !== 'function') {
|
|
84
|
+
return resolveLocalPackageFallback(specifier);
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
const require = NodeSingletons.module.createRequire(import.meta.url);
|
|
88
|
+
const resolved = require.resolve(specifier);
|
|
89
|
+
if ((specifier === '@zintrust/workers' &&
|
|
90
|
+
resolved.includes(`${path.sep}node_modules${path.sep}@zintrust${path.sep}workers${path.sep}`)) ||
|
|
91
|
+
(specifier === '@zintrust/core' &&
|
|
92
|
+
resolved.includes(`${path.sep}node_modules${path.sep}@zintrust${path.sep}core${path.sep}`))) {
|
|
93
|
+
const local = resolveLocalPackageFallback(specifier);
|
|
94
|
+
if (local)
|
|
95
|
+
return local;
|
|
96
|
+
}
|
|
97
|
+
return NodeSingletons.url.pathToFileURL(resolved).href;
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
return resolveLocalPackageFallback(specifier);
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
const escapeRegExp = (value) => value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw `\$&`);
|
|
104
|
+
const rewriteProcessorImports = (code) => {
|
|
105
|
+
const replacements = [];
|
|
106
|
+
const coreUrl = resolvePackageSpecifierUrl('@zintrust/core');
|
|
107
|
+
if (coreUrl)
|
|
108
|
+
replacements.push({ from: '@zintrust/core', to: coreUrl });
|
|
109
|
+
const workersUrl = resolvePackageSpecifierUrl('@zintrust/workers');
|
|
110
|
+
if (workersUrl)
|
|
111
|
+
replacements.push({ from: '@zintrust/workers', to: workersUrl });
|
|
112
|
+
if (replacements.length === 0)
|
|
113
|
+
return code;
|
|
114
|
+
let updated = code;
|
|
115
|
+
for (const { from, to } of replacements) {
|
|
116
|
+
const pattern = new RegExp(String.raw `(['"])${escapeRegExp(from)}\1`, 'g');
|
|
117
|
+
updated = updated.replace(pattern, `$1${to}$1`);
|
|
118
|
+
}
|
|
119
|
+
return updated;
|
|
120
|
+
};
|
|
121
|
+
const ensureProcessorSpecDir = () => {
|
|
122
|
+
if (!isNodeRuntime() || !canUseProjectFileImports())
|
|
123
|
+
return null;
|
|
124
|
+
const dir = path.join(resolveProjectRoot(), '.zintrust', 'processor-specs');
|
|
125
|
+
try {
|
|
126
|
+
if (!NodeSingletons.fs.existsSync(dir)) {
|
|
127
|
+
NodeSingletons.fs.mkdirSync(dir, { recursive: true });
|
|
128
|
+
}
|
|
129
|
+
return dir;
|
|
130
|
+
}
|
|
131
|
+
catch (error) {
|
|
132
|
+
Logger.debug('Failed to prepare processor spec cache directory', error);
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
const shouldFallbackToFileImport = (error) => {
|
|
137
|
+
if (!isNodeRuntime())
|
|
138
|
+
return false;
|
|
139
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
140
|
+
const code = error?.code ?? '';
|
|
141
|
+
if (code === 'ERR_INVALID_URL' || code === 'ERR_UNSUPPORTED_ESM_URL_SCHEME')
|
|
142
|
+
return true;
|
|
143
|
+
return (message.includes('Invalid relative URL') ||
|
|
144
|
+
message.includes('base scheme is not hierarchical') ||
|
|
145
|
+
message.includes('Failed to resolve module specifier'));
|
|
146
|
+
};
|
|
147
|
+
const isOptionalD1ProxyModuleMissing = (error) => {
|
|
148
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
149
|
+
const code = error?.code ?? '';
|
|
150
|
+
if (code !== 'ERR_MODULE_NOT_FOUND')
|
|
151
|
+
return false;
|
|
152
|
+
return (message.includes('cloudflare-d1-proxy') ||
|
|
153
|
+
message.includes('/packages/cloudflare-d1-proxy/src/index.js'));
|
|
154
|
+
};
|
|
155
|
+
const importModuleFromCode = async (params) => {
|
|
156
|
+
const { code, normalized, cacheKey } = params;
|
|
157
|
+
const dataUrl = `data:text/javascript;base64,${toBase64(code)}`;
|
|
158
|
+
try {
|
|
159
|
+
return (await import(dataUrl));
|
|
160
|
+
}
|
|
161
|
+
catch (error) {
|
|
162
|
+
if (!shouldFallbackToFileImport(error))
|
|
163
|
+
throw error;
|
|
164
|
+
const dir = ensureProcessorSpecDir();
|
|
165
|
+
if (!dir)
|
|
166
|
+
throw error;
|
|
167
|
+
try {
|
|
168
|
+
const codeHash = await computeSha256(code);
|
|
169
|
+
const filePath = path.join(dir, `${codeHash || cacheKey}.mjs`);
|
|
170
|
+
NodeSingletons.fs.writeFileSync(filePath, code, 'utf8');
|
|
171
|
+
const fileUrl = NodeSingletons.url.pathToFileURL(filePath).href;
|
|
172
|
+
return (await import(fileUrl));
|
|
173
|
+
}
|
|
174
|
+
catch (fileError) {
|
|
175
|
+
if (isOptionalD1ProxyModuleMissing(fileError)) {
|
|
176
|
+
throw fileError;
|
|
177
|
+
}
|
|
178
|
+
Logger.debug(`Processor URL file fallback failed for ${normalized}`, fileError);
|
|
179
|
+
throw error;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
const isCloudflareRuntime = () => Cloudflare.getWorkersEnv() !== null;
|
|
184
|
+
const getDefaultStoreForRuntime = async () => {
|
|
185
|
+
if (!isCloudflareRuntime()) {
|
|
186
|
+
await ensureWorkerStoreConfigured();
|
|
187
|
+
return workerStore;
|
|
188
|
+
}
|
|
189
|
+
const bootstrapConfig = buildPersistenceBootstrapConfig();
|
|
190
|
+
const persistence = resolvePersistenceConfig(bootstrapConfig);
|
|
191
|
+
if (!persistence) {
|
|
192
|
+
return InMemoryWorkerStore.create();
|
|
193
|
+
}
|
|
194
|
+
return resolveWorkerStoreForPersistence(persistence);
|
|
195
|
+
};
|
|
26
196
|
const getStoreForWorker = async (config, persistenceOverride) => {
|
|
27
197
|
if (persistenceOverride) {
|
|
28
198
|
return resolveWorkerStoreForPersistence(persistenceOverride);
|
|
@@ -35,8 +205,7 @@ const getStoreForWorker = async (config, persistenceOverride) => {
|
|
|
35
205
|
}
|
|
36
206
|
}
|
|
37
207
|
// Fallback to default/global store
|
|
38
|
-
|
|
39
|
-
return workerStore;
|
|
208
|
+
return getDefaultStoreForRuntime();
|
|
40
209
|
};
|
|
41
210
|
const validateAndGetStore = async (name, config, persistenceOverride) => {
|
|
42
211
|
const store = await getStoreForWorker(config, persistenceOverride);
|
|
@@ -153,7 +322,7 @@ const computeSha256 = async (value) => {
|
|
|
153
322
|
if (typeof NodeSingletons.createHash === 'function') {
|
|
154
323
|
return NodeSingletons.createHash('sha256').update(value).digest('hex');
|
|
155
324
|
}
|
|
156
|
-
return String(
|
|
325
|
+
return String(generateUuid()).slice(2);
|
|
157
326
|
};
|
|
158
327
|
const toBase64 = (value) => {
|
|
159
328
|
if (typeof Buffer !== 'undefined') {
|
|
@@ -267,6 +436,18 @@ const sanitizeProcessorPath = (value) => {
|
|
|
267
436
|
};
|
|
268
437
|
const stripProcessorExtension = (value) => value.replace(/\.(ts|js)$/i, '');
|
|
269
438
|
const normalizeModulePath = (value) => value.replaceAll('\\', '/');
|
|
439
|
+
const filterExistingFileCandidates = (candidates) => {
|
|
440
|
+
if (!NodeSingletons?.fs?.existsSync)
|
|
441
|
+
return candidates;
|
|
442
|
+
return candidates.filter((candidate) => {
|
|
443
|
+
try {
|
|
444
|
+
return NodeSingletons.fs.existsSync(candidate);
|
|
445
|
+
}
|
|
446
|
+
catch {
|
|
447
|
+
return false;
|
|
448
|
+
}
|
|
449
|
+
});
|
|
450
|
+
};
|
|
270
451
|
const buildProcessorModuleCandidates = (modulePath, resolvedPath) => {
|
|
271
452
|
const candidates = [];
|
|
272
453
|
const normalized = normalizeModulePath(modulePath.trim());
|
|
@@ -286,6 +467,22 @@ const buildProcessorModuleCandidates = (modulePath, resolvedPath) => {
|
|
|
286
467
|
}
|
|
287
468
|
return Array.from(new Set(candidates));
|
|
288
469
|
};
|
|
470
|
+
const buildProcessorFilePathCandidates = (_modulePath, resolvedPath) => {
|
|
471
|
+
const candidates = [];
|
|
472
|
+
const normalizedResolved = normalizeModulePath(resolvedPath);
|
|
473
|
+
const projectRoot = normalizeModulePath(resolveProjectRoot());
|
|
474
|
+
const strippedResolved = stripProcessorExtension(resolvedPath);
|
|
475
|
+
candidates.push(`${strippedResolved}.js`, `${strippedResolved}.mjs`);
|
|
476
|
+
const appIndex = normalizedResolved.lastIndexOf('/app/');
|
|
477
|
+
if (appIndex !== -1) {
|
|
478
|
+
const relative = normalizedResolved.slice(appIndex + 5);
|
|
479
|
+
if (relative) {
|
|
480
|
+
const strippedRelative = stripProcessorExtension(relative);
|
|
481
|
+
candidates.push(path.join(projectRoot, 'dist', 'app', `${strippedRelative}.js`), path.join(projectRoot, 'app', relative), path.join(projectRoot, 'app', `${strippedRelative}.js`), path.join('/app', 'dist', 'app', `${strippedRelative}.js`));
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
return filterExistingFileCandidates(Array.from(new Set(candidates)));
|
|
485
|
+
};
|
|
289
486
|
const pickProcessorFromModule = (mod, source) => {
|
|
290
487
|
const candidate = mod?.['default'] ?? mod?.['processor'] ?? mod?.['handler'] ?? mod?.['handle'];
|
|
291
488
|
if (typeof candidate !== 'function') {
|
|
@@ -328,9 +525,9 @@ const refreshCachedProcessor = (existing, config, cacheControl) => {
|
|
|
328
525
|
};
|
|
329
526
|
const cacheProcessorFromResponse = async (params) => {
|
|
330
527
|
const { response, normalized, config, cacheKey } = params;
|
|
331
|
-
const
|
|
332
|
-
const
|
|
333
|
-
const mod = await
|
|
528
|
+
const rawCode = await readResponseBody(response, config.fetchMaxSizeBytes);
|
|
529
|
+
const code = rewriteProcessorImports(rawCode);
|
|
530
|
+
const mod = await importModuleFromCode({ code, normalized, cacheKey });
|
|
334
531
|
const processor = extractZinTrustProcessor(mod, normalized);
|
|
335
532
|
if (!processor) {
|
|
336
533
|
throw ErrorFactory.createConfigError('INVALID_PROCESSOR_URL_EXPORT');
|
|
@@ -376,6 +573,10 @@ const fetchProcessorAttempt = async (params) => {
|
|
|
376
573
|
return await cacheProcessorFromResponse({ response, normalized, config, cacheKey });
|
|
377
574
|
}
|
|
378
575
|
catch (error) {
|
|
576
|
+
if (isOptionalD1ProxyModuleMissing(error)) {
|
|
577
|
+
Logger.warn('Processor URL skipped: optional cloudflare-d1-proxy module is unavailable in this runtime.');
|
|
578
|
+
return undefined;
|
|
579
|
+
}
|
|
379
580
|
if (controller.signal.aborted) {
|
|
380
581
|
Logger.error('Processor URL fetch timeout', error);
|
|
381
582
|
}
|
|
@@ -415,9 +616,11 @@ const resolveProcessorFromUrl = async (spec) => {
|
|
|
415
616
|
}
|
|
416
617
|
if (parsed.protocol !== 'https:' && parsed.protocol !== 'file:') {
|
|
417
618
|
Logger.warn(`Invalid processor URL protocol: ${parsed.protocol}. Only https:// and file:// are supported.`);
|
|
619
|
+
return undefined;
|
|
418
620
|
}
|
|
419
621
|
if (!isAllowedRemoteHost(parsed.host) && parsed.protocol !== 'file:') {
|
|
420
622
|
Logger.warn(`Invalid processor URL host: ${parsed.host}. Host is not in the allowlist.`);
|
|
623
|
+
return undefined;
|
|
421
624
|
}
|
|
422
625
|
const config = getProcessorSpecConfig();
|
|
423
626
|
const cacheKey = await computeSha256(normalized);
|
|
@@ -456,8 +659,11 @@ const resolveProcessorFromPath = async (modulePath) => {
|
|
|
456
659
|
return undefined;
|
|
457
660
|
const [candidatePath, ...rest] = candidates;
|
|
458
661
|
try {
|
|
459
|
-
const
|
|
460
|
-
|
|
662
|
+
const importPath = candidatePath.startsWith('/') && !candidatePath.startsWith('//')
|
|
663
|
+
? NodeSingletons.url.pathToFileURL(candidatePath).href
|
|
664
|
+
: candidatePath;
|
|
665
|
+
const mod = await import(importPath);
|
|
666
|
+
const candidate = pickProcessorFromModule(mod, importPath);
|
|
461
667
|
if (candidate)
|
|
462
668
|
return candidate;
|
|
463
669
|
}
|
|
@@ -473,10 +679,14 @@ const resolveProcessorFromPath = async (modulePath) => {
|
|
|
473
679
|
return candidate;
|
|
474
680
|
}
|
|
475
681
|
catch (err) {
|
|
476
|
-
const
|
|
477
|
-
const
|
|
478
|
-
if (
|
|
479
|
-
return
|
|
682
|
+
const fileCandidates = buildProcessorFilePathCandidates(trimmed, resolved);
|
|
683
|
+
const resolvedFileCandidate = await importProcessorFromCandidates(fileCandidates);
|
|
684
|
+
if (resolvedFileCandidate)
|
|
685
|
+
return resolvedFileCandidate;
|
|
686
|
+
const moduleCandidates = buildProcessorModuleCandidates(trimmed, resolved);
|
|
687
|
+
const resolvedModuleCandidate = await importProcessorFromCandidates(moduleCandidates);
|
|
688
|
+
if (resolvedModuleCandidate)
|
|
689
|
+
return resolvedModuleCandidate;
|
|
480
690
|
Logger.error(`Failed to import processor from path: ${resolved}`, err);
|
|
481
691
|
}
|
|
482
692
|
return undefined;
|
|
@@ -701,6 +911,56 @@ const handleFailure = async (params) => {
|
|
|
701
911
|
await executeAllFailureHandlers(params);
|
|
702
912
|
await executeFailurePlugins(workerName, job, error, features);
|
|
703
913
|
};
|
|
914
|
+
const toBackoffDelayMs = (backoff) => {
|
|
915
|
+
if (typeof backoff === 'number' && Number.isFinite(backoff)) {
|
|
916
|
+
return Math.max(0, Math.floor(backoff));
|
|
917
|
+
}
|
|
918
|
+
if (backoff !== null && backoff !== undefined && typeof backoff === 'object') {
|
|
919
|
+
const raw = backoff.delay;
|
|
920
|
+
if (typeof raw === 'number' && Number.isFinite(raw)) {
|
|
921
|
+
return Math.max(0, Math.floor(raw));
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
return 0;
|
|
925
|
+
};
|
|
926
|
+
const trackJobStarted = async (input) => {
|
|
927
|
+
if (!input.job.id)
|
|
928
|
+
return;
|
|
929
|
+
await JobStateTracker.started({
|
|
930
|
+
queueName: input.queueName,
|
|
931
|
+
jobId: input.job.id,
|
|
932
|
+
attempts: input.attempts,
|
|
933
|
+
timeoutMs: Math.max(1000, Env.getInt('QUEUE_JOB_TIMEOUT', 60) * 1000),
|
|
934
|
+
workerName: input.workerName,
|
|
935
|
+
workerVersion: input.workerVersion,
|
|
936
|
+
});
|
|
937
|
+
};
|
|
938
|
+
const trackJobCompleted = async (input) => {
|
|
939
|
+
if (!input.job.id)
|
|
940
|
+
return;
|
|
941
|
+
await JobStateTracker.completed({
|
|
942
|
+
queueName: input.queueName,
|
|
943
|
+
jobId: input.job.id,
|
|
944
|
+
processingTimeMs: input.duration,
|
|
945
|
+
result: input.result,
|
|
946
|
+
});
|
|
947
|
+
};
|
|
948
|
+
const trackJobFailed = async (input) => {
|
|
949
|
+
if (!input.job.id)
|
|
950
|
+
return;
|
|
951
|
+
const isFinal = input.maxAttempts === undefined ? true : input.attempts >= input.maxAttempts;
|
|
952
|
+
const backoffDelayMs = toBackoffDelayMs(input.job.opts?.backoff);
|
|
953
|
+
await JobStateTracker.failed({
|
|
954
|
+
queueName: input.queueName,
|
|
955
|
+
jobId: input.job.id,
|
|
956
|
+
attempts: input.attempts,
|
|
957
|
+
isFinal,
|
|
958
|
+
retryAt: !isFinal && backoffDelayMs > 0
|
|
959
|
+
? new Date(Date.now() + backoffDelayMs).toISOString()
|
|
960
|
+
: undefined,
|
|
961
|
+
error: input.error,
|
|
962
|
+
});
|
|
963
|
+
};
|
|
704
964
|
/**
|
|
705
965
|
* Helper: Create enhanced processor with all features
|
|
706
966
|
*/
|
|
@@ -719,8 +979,19 @@ const createEnhancedProcessor = (config) => {
|
|
|
719
979
|
const startTime = Date.now();
|
|
720
980
|
let result;
|
|
721
981
|
let spanId = null;
|
|
982
|
+
const maxAttempts = typeof job.opts?.attempts === 'number' && Number.isFinite(job.opts.attempts)
|
|
983
|
+
? Math.max(1, Math.floor(job.opts.attempts))
|
|
984
|
+
: undefined;
|
|
985
|
+
const attempts = Math.max(1, Math.floor((job.attemptsMade ?? 0) + 1));
|
|
722
986
|
try {
|
|
723
987
|
spanId = startProcessingSpan(name, jobVersion, job, config.queueName, features);
|
|
988
|
+
await trackJobStarted({
|
|
989
|
+
queueName: config.queueName,
|
|
990
|
+
job,
|
|
991
|
+
attempts,
|
|
992
|
+
workerName: name,
|
|
993
|
+
workerVersion: jobVersion,
|
|
994
|
+
});
|
|
724
995
|
// Process the job
|
|
725
996
|
result = await processor(job);
|
|
726
997
|
const duration = Date.now() - startTime;
|
|
@@ -733,11 +1004,19 @@ const createEnhancedProcessor = (config) => {
|
|
|
733
1004
|
spanId,
|
|
734
1005
|
features,
|
|
735
1006
|
});
|
|
1007
|
+
await trackJobCompleted({ queueName: config.queueName, job, duration, result });
|
|
736
1008
|
return result;
|
|
737
1009
|
}
|
|
738
1010
|
catch (err) {
|
|
739
1011
|
const error = err;
|
|
740
1012
|
const duration = Date.now() - startTime;
|
|
1013
|
+
await trackJobFailed({
|
|
1014
|
+
queueName: config.queueName,
|
|
1015
|
+
job,
|
|
1016
|
+
attempts,
|
|
1017
|
+
maxAttempts,
|
|
1018
|
+
error,
|
|
1019
|
+
});
|
|
741
1020
|
await handleFailure({
|
|
742
1021
|
workerName: name,
|
|
743
1022
|
jobVersion,
|
|
@@ -801,12 +1080,22 @@ const resolveRedisConfigFromEnv = (config, context) => {
|
|
|
801
1080
|
password: password || undefined,
|
|
802
1081
|
};
|
|
803
1082
|
};
|
|
804
|
-
const resolveRedisConfigFromDirect = (config, context) =>
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
}
|
|
1083
|
+
const resolveRedisConfigFromDirect = (config, context) => {
|
|
1084
|
+
const fallbackDb = Env.getInt('REDIS_QUEUE_DB', ZintrustLang.REDIS_DEFAULT_DB);
|
|
1085
|
+
let normalizedDb = fallbackDb;
|
|
1086
|
+
if (typeof config.db === 'number') {
|
|
1087
|
+
normalizedDb = config.db;
|
|
1088
|
+
}
|
|
1089
|
+
else if (typeof config.database === 'number') {
|
|
1090
|
+
normalizedDb = config.database;
|
|
1091
|
+
}
|
|
1092
|
+
return {
|
|
1093
|
+
host: requireRedisHost(config.host, context),
|
|
1094
|
+
port: config.port,
|
|
1095
|
+
db: normalizedDb,
|
|
1096
|
+
password: config.password ?? Env.get('REDIS_PASSWORD', undefined),
|
|
1097
|
+
};
|
|
1098
|
+
};
|
|
810
1099
|
const resolveRedisConfig = (config, context) => isRedisEnvConfig(config)
|
|
811
1100
|
? resolveRedisConfigFromEnv(config, context)
|
|
812
1101
|
: resolveRedisConfigFromDirect(config, context);
|
|
@@ -817,17 +1106,21 @@ const resolveRedisConfigWithFallback = (primary, fallback, errorMessage, context
|
|
|
817
1106
|
}
|
|
818
1107
|
return resolveRedisConfig(selected, context);
|
|
819
1108
|
};
|
|
1109
|
+
const logRedisPersistenceConfig = (redisConfig, key_prefix, source) => {
|
|
1110
|
+
Logger.debug('Worker persistence redis config', {
|
|
1111
|
+
source,
|
|
1112
|
+
host: redisConfig.host,
|
|
1113
|
+
port: redisConfig.port,
|
|
1114
|
+
db: redisConfig.db,
|
|
1115
|
+
key_prefix,
|
|
1116
|
+
});
|
|
1117
|
+
};
|
|
820
1118
|
const normalizeEnvValue = (value) => {
|
|
821
1119
|
if (!value)
|
|
822
1120
|
return undefined;
|
|
823
1121
|
const trimmed = value.trim();
|
|
824
1122
|
return trimmed.length > 0 ? trimmed : undefined;
|
|
825
1123
|
};
|
|
826
|
-
const normalizeAppName = (value) => {
|
|
827
|
-
const normalized = (value ?? '').trim().replaceAll(/\s+/g, '_');
|
|
828
|
-
return normalized.length > 0 ? normalized : 'zintrust';
|
|
829
|
-
};
|
|
830
|
-
const resolveDefaultRedisKeyPrefix = () => 'worker_' + normalizeAppName(appConfig.prefix);
|
|
831
1124
|
const resolveDefaultPersistenceTable = () => normalizeEnvValue(Env.get('WORKER_PERSISTENCE_TABLE', 'zintrust_workers')) ?? 'zintrust_workers';
|
|
832
1125
|
const resolveDefaultPersistenceConnection = () => normalizeEnvValue(Env.get('WORKER_PERSISTENCE_DB_CONNECTION', 'default')) ?? 'default';
|
|
833
1126
|
const resolveAutoStart = (config) => {
|
|
@@ -845,9 +1138,7 @@ const normalizeExplicitPersistence = (persistence) => {
|
|
|
845
1138
|
return {
|
|
846
1139
|
driver: 'redis',
|
|
847
1140
|
redis: persistence.redis,
|
|
848
|
-
keyPrefix:
|
|
849
|
-
normalizeEnvValue(Env.get('WORKER_PERSISTENCE_REDIS_KEY_PREFIX', '')) ??
|
|
850
|
-
resolveDefaultRedisKeyPrefix(),
|
|
1141
|
+
keyPrefix: keyPrefix(),
|
|
851
1142
|
};
|
|
852
1143
|
}
|
|
853
1144
|
const clientIsConnection = typeof persistence.client === 'string';
|
|
@@ -875,11 +1166,15 @@ const resolvePersistenceConfig = (config) => {
|
|
|
875
1166
|
if (driver === 'memory')
|
|
876
1167
|
return { driver: 'memory' };
|
|
877
1168
|
if (driver === 'redis') {
|
|
878
|
-
const
|
|
1169
|
+
const persistenceDbOverride = normalizeEnvValue(Env.get('WORKER_PERSISTENCE_REDIS_DB', ''));
|
|
879
1170
|
return {
|
|
880
1171
|
driver: 'redis',
|
|
881
|
-
|
|
882
|
-
|
|
1172
|
+
// Optional override; otherwise defaults to REDIS_QUEUE_DB.
|
|
1173
|
+
redis: {
|
|
1174
|
+
env: true,
|
|
1175
|
+
db: persistenceDbOverride ? 'WORKER_PERSISTENCE_REDIS_DB' : 'REDIS_QUEUE_DB',
|
|
1176
|
+
},
|
|
1177
|
+
keyPrefix: keyPrefix(),
|
|
883
1178
|
};
|
|
884
1179
|
}
|
|
885
1180
|
if (driver === 'db' || driver === 'database') {
|
|
@@ -918,8 +1213,10 @@ const resolveWorkerStore = async (config) => {
|
|
|
918
1213
|
}
|
|
919
1214
|
else if (persistence.driver === 'redis') {
|
|
920
1215
|
const redisConfig = resolveRedisConfigWithFallback(persistence.redis, config.infrastructure?.redis, 'Worker persistence requires redis config (persistence.redis or infrastructure.redis)', 'infrastructure.persistence.redis');
|
|
1216
|
+
const key_prefix = persistence.keyPrefix ?? keyPrefix();
|
|
1217
|
+
logRedisPersistenceConfig(redisConfig, key_prefix, 'resolveWorkerStore');
|
|
921
1218
|
const client = createRedisConnection(redisConfig);
|
|
922
|
-
next = RedisWorkerStore.create(client,
|
|
1219
|
+
next = RedisWorkerStore.create(client, key_prefix);
|
|
923
1220
|
}
|
|
924
1221
|
else if (persistence.driver === 'database') {
|
|
925
1222
|
const explicitConnection = typeof persistence.client === 'string' ? persistence.client : persistence.connection;
|
|
@@ -960,8 +1257,10 @@ const createWorkerStore = async (persistence) => {
|
|
|
960
1257
|
}
|
|
961
1258
|
if (persistence.driver === 'redis') {
|
|
962
1259
|
const redisConfig = resolveRedisConfigWithFallback(persistence.redis ?? { env: true }, undefined, 'Worker persistence requires redis config (persistence.redis or REDIS_* env values)', 'persistence.redis');
|
|
1260
|
+
const key_prefix = persistence.keyPrefix ?? keyPrefix();
|
|
1261
|
+
logRedisPersistenceConfig(redisConfig, key_prefix, 'createWorkerStore');
|
|
963
1262
|
const client = createRedisConnection(redisConfig);
|
|
964
|
-
return RedisWorkerStore.create(client,
|
|
1263
|
+
return RedisWorkerStore.create(client, key_prefix);
|
|
965
1264
|
}
|
|
966
1265
|
// Database driver
|
|
967
1266
|
const explicitConnection = typeof persistence.client === 'string' ? persistence.client : persistence.connection;
|
|
@@ -972,22 +1271,37 @@ const createWorkerStore = async (persistence) => {
|
|
|
972
1271
|
};
|
|
973
1272
|
const resolveWorkerStoreForPersistence = async (persistence) => {
|
|
974
1273
|
const cacheKey = generateCacheKey(persistence);
|
|
975
|
-
|
|
1274
|
+
const isCloudflare = Cloudflare.getWorkersEnv() !== null;
|
|
1275
|
+
// Return cached instance if available (disable cache for Cloudflare to assume cleanup)
|
|
1276
|
+
// Or handle cleanup differently. For now, disable cache for Cloudflare to allow per-request connections.
|
|
976
1277
|
const cached = storeInstanceCache.get(cacheKey);
|
|
977
|
-
if (cached) {
|
|
1278
|
+
if (cached && !isCloudflare) {
|
|
978
1279
|
return cached;
|
|
979
1280
|
}
|
|
980
1281
|
// Create new store instance
|
|
981
1282
|
const store = await createWorkerStore(persistence);
|
|
982
1283
|
await store.init();
|
|
983
|
-
// Cache the store instance for reuse
|
|
984
|
-
|
|
1284
|
+
// Cache the store instance for reuse only if not Cloudflare
|
|
1285
|
+
if (!isCloudflare) {
|
|
1286
|
+
storeInstanceCache.set(cacheKey, store);
|
|
1287
|
+
}
|
|
985
1288
|
return store;
|
|
986
1289
|
};
|
|
987
1290
|
const getPersistedRecord = async (name, persistenceOverride) => {
|
|
988
1291
|
if (!persistenceOverride) {
|
|
989
|
-
|
|
990
|
-
|
|
1292
|
+
if (!isCloudflareRuntime()) {
|
|
1293
|
+
await ensureWorkerStoreConfigured();
|
|
1294
|
+
return workerStore.get(name);
|
|
1295
|
+
}
|
|
1296
|
+
const store = await getDefaultStoreForRuntime();
|
|
1297
|
+
try {
|
|
1298
|
+
return await store.get(name);
|
|
1299
|
+
}
|
|
1300
|
+
finally {
|
|
1301
|
+
if (store.close) {
|
|
1302
|
+
await store.close();
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
991
1305
|
}
|
|
992
1306
|
const store = await resolveWorkerStoreForPersistence(persistenceOverride);
|
|
993
1307
|
return store.get(name);
|
|
@@ -1359,6 +1673,35 @@ export const WorkerFactory = Object.freeze({
|
|
|
1359
1673
|
registerProcessorSpec,
|
|
1360
1674
|
resolveProcessorPath,
|
|
1361
1675
|
resolveProcessorSpec,
|
|
1676
|
+
/**
|
|
1677
|
+
* Register a new worker configuration without starting it.
|
|
1678
|
+
*/
|
|
1679
|
+
async register(config) {
|
|
1680
|
+
const { name } = config;
|
|
1681
|
+
// Check in-memory first (though unlikely if we are just registering)
|
|
1682
|
+
if (workers.has(name)) {
|
|
1683
|
+
throw ErrorFactory.createWorkerError(`Worker "${name}" is already running locally`);
|
|
1684
|
+
}
|
|
1685
|
+
const store = await getStoreForWorker(config);
|
|
1686
|
+
try {
|
|
1687
|
+
const existing = await store.get(name);
|
|
1688
|
+
if (existing) {
|
|
1689
|
+
throw ErrorFactory.createWorkerError(`Worker "${name}" already exists in persistence`);
|
|
1690
|
+
}
|
|
1691
|
+
// Init features to validate config, but mainly we just want to save it.
|
|
1692
|
+
// initializeWorkerFeatures might rely on being active or having resources, so we might skip it or do partial.
|
|
1693
|
+
// For now, just save definition.
|
|
1694
|
+
// Status should be STOPPED or CREATED.
|
|
1695
|
+
await store.save(buildWorkerRecord(config, WorkerCreationStatus.STOPPED));
|
|
1696
|
+
Logger.info(`Worker registered (persistence only): ${name}`);
|
|
1697
|
+
}
|
|
1698
|
+
finally {
|
|
1699
|
+
// If Cloudflare environment, try to close store connection to avoid zombie connections
|
|
1700
|
+
if (Cloudflare.getWorkersEnv() !== null && store.close) {
|
|
1701
|
+
await store.close();
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
},
|
|
1362
1705
|
/**
|
|
1363
1706
|
* Create new worker with full setup
|
|
1364
1707
|
*/
|
|
@@ -1727,9 +2070,25 @@ export const WorkerFactory = Object.freeze({
|
|
|
1727
2070
|
async listPersistedRecords(persistenceOverride, options) {
|
|
1728
2071
|
const includeInactive = options?.includeInactive === true;
|
|
1729
2072
|
if (!persistenceOverride) {
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
2073
|
+
if (!isCloudflareRuntime()) {
|
|
2074
|
+
await ensureWorkerStoreConfigured();
|
|
2075
|
+
const records = await workerStore.list(options);
|
|
2076
|
+
return includeInactive
|
|
2077
|
+
? records
|
|
2078
|
+
: records.filter((record) => record.activeStatus !== false);
|
|
2079
|
+
}
|
|
2080
|
+
const store = await getDefaultStoreForRuntime();
|
|
2081
|
+
try {
|
|
2082
|
+
const records = await store.list(options);
|
|
2083
|
+
return includeInactive
|
|
2084
|
+
? records
|
|
2085
|
+
: records.filter((record) => record.activeStatus !== false);
|
|
2086
|
+
}
|
|
2087
|
+
finally {
|
|
2088
|
+
if (store.close) {
|
|
2089
|
+
await store.close();
|
|
2090
|
+
}
|
|
2091
|
+
}
|
|
1733
2092
|
}
|
|
1734
2093
|
const store = await resolveWorkerStoreForPersistence(persistenceOverride);
|
|
1735
2094
|
const records = await store.list(options);
|
|
@@ -1779,7 +2138,15 @@ export const WorkerFactory = Object.freeze({
|
|
|
1779
2138
|
async getPersisted(name, persistenceOverride) {
|
|
1780
2139
|
const instance = workers.get(name);
|
|
1781
2140
|
const store = await getStoreForWorker(instance?.config, persistenceOverride);
|
|
1782
|
-
|
|
2141
|
+
try {
|
|
2142
|
+
const result = await store.get(name);
|
|
2143
|
+
return result;
|
|
2144
|
+
}
|
|
2145
|
+
finally {
|
|
2146
|
+
if (Cloudflare.getWorkersEnv() !== null && store.close) {
|
|
2147
|
+
await store.close();
|
|
2148
|
+
}
|
|
2149
|
+
}
|
|
1783
2150
|
},
|
|
1784
2151
|
/**
|
|
1785
2152
|
* Remove worker
|