@zintrust/workers 0.1.31 → 0.1.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ClusterLock.js +3 -2
- package/dist/DeadLetterQueue.js +3 -2
- package/dist/HealthMonitor.js +24 -13
- package/dist/Observability.js +8 -0
- package/dist/WorkerFactory.d.ts +4 -0
- package/dist/WorkerFactory.js +409 -42
- package/dist/WorkerInit.js +122 -43
- package/dist/WorkerMetrics.js +5 -1
- package/dist/WorkerRegistry.js +8 -0
- package/dist/WorkerShutdown.d.ts +0 -13
- package/dist/WorkerShutdown.js +1 -44
- package/dist/build-manifest.json +101 -85
- package/dist/config/workerConfig.d.ts +1 -0
- package/dist/config/workerConfig.js +7 -1
- package/dist/createQueueWorker.js +281 -42
- package/dist/dashboard/workers-api.js +8 -1
- package/dist/http/WorkerController.js +90 -35
- package/dist/http/WorkerMonitoringService.js +29 -2
- package/dist/http/middleware/FeaturesValidator.js +5 -4
- package/dist/index.d.ts +1 -2
- package/dist/index.js +0 -1
- package/dist/routes/workers.js +10 -7
- package/dist/storage/WorkerStore.d.ts +6 -3
- package/dist/storage/WorkerStore.js +16 -0
- package/dist/telemetry/api/TelemetryMonitoringService.js +29 -2
- package/dist/ui/router/ui.js +58 -29
- package/dist/ui/workers/index.html +202 -0
- package/dist/ui/workers/main.js +1952 -0
- package/dist/ui/workers/styles.css +1350 -0
- package/dist/ui/workers/zintrust.svg +30 -0
- package/package.json +5 -5
- package/src/ClusterLock.ts +13 -7
- package/src/ComplianceManager.ts +3 -2
- package/src/DeadLetterQueue.ts +6 -4
- package/src/HealthMonitor.ts +33 -17
- package/src/Observability.ts +11 -0
- package/src/WorkerFactory.ts +480 -43
- package/src/WorkerInit.ts +167 -48
- package/src/WorkerMetrics.ts +14 -8
- package/src/WorkerRegistry.ts +11 -0
- package/src/WorkerShutdown.ts +1 -69
- package/src/config/workerConfig.ts +9 -1
- package/src/createQueueWorker.ts +428 -43
- package/src/dashboard/workers-api.ts +8 -1
- package/src/http/WorkerController.ts +111 -36
- package/src/http/WorkerMonitoringService.ts +35 -2
- package/src/http/middleware/FeaturesValidator.ts +8 -19
- package/src/index.ts +2 -3
- package/src/routes/workers.ts +10 -8
- package/src/storage/WorkerStore.ts +21 -3
- package/src/telemetry/api/TelemetryMonitoringService.ts +35 -2
- package/src/types/queue-monitor.d.ts +2 -1
- package/src/ui/components/WorkerExpandPanel.js +0 -8
- package/src/ui/router/EmbeddedAssets.ts +3 -0
- package/src/ui/router/ui.ts +57 -39
- package/src/WorkerShutdownDurableObject.ts +0 -64
package/src/WorkerFactory.ts
CHANGED
|
@@ -5,12 +5,14 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import {
|
|
8
|
-
|
|
8
|
+
Cloudflare,
|
|
9
9
|
createRedisConnection,
|
|
10
10
|
databaseConfig,
|
|
11
11
|
Env,
|
|
12
12
|
ErrorFactory,
|
|
13
|
+
generateUuid,
|
|
13
14
|
getBullMQSafeQueueName,
|
|
15
|
+
JobStateTracker,
|
|
14
16
|
Logger,
|
|
15
17
|
NodeSingletons,
|
|
16
18
|
queueConfig,
|
|
@@ -39,6 +41,7 @@ import { ResourceMonitor } from './ResourceMonitor';
|
|
|
39
41
|
import { WorkerMetrics } from './WorkerMetrics';
|
|
40
42
|
import { WorkerRegistry, type WorkerInstance as RegistryWorkerInstance } from './WorkerRegistry';
|
|
41
43
|
import { WorkerVersioning } from './WorkerVersioning';
|
|
44
|
+
import { keyPrefix } from './config/workerConfig';
|
|
42
45
|
import {
|
|
43
46
|
DbWorkerStore,
|
|
44
47
|
InMemoryWorkerStore,
|
|
@@ -49,6 +52,199 @@ import {
|
|
|
49
52
|
|
|
50
53
|
const path = NodeSingletons.path;
|
|
51
54
|
|
|
55
|
+
const isNodeRuntime = (): boolean =>
|
|
56
|
+
typeof process !== 'undefined' && Boolean(process.versions?.node);
|
|
57
|
+
|
|
58
|
+
const resolveProjectRoot = (): string => {
|
|
59
|
+
const envRoot = Env.get('ZINTRUST_PROJECT_ROOT', '').trim();
|
|
60
|
+
return envRoot.length > 0 ? envRoot : process.cwd();
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
const canUseProjectFileImports = (): boolean =>
|
|
64
|
+
typeof NodeSingletons?.fs?.writeFileSync === 'function' &&
|
|
65
|
+
typeof NodeSingletons?.fs?.mkdirSync === 'function' &&
|
|
66
|
+
typeof NodeSingletons?.fs?.existsSync === 'function' &&
|
|
67
|
+
typeof NodeSingletons?.url?.pathToFileURL === 'function' &&
|
|
68
|
+
typeof NodeSingletons?.path?.join === 'function';
|
|
69
|
+
|
|
70
|
+
const buildCandidatesForSpecifier = (specifier: string, root: string): string[] => {
|
|
71
|
+
if (specifier === '@zintrust/core') {
|
|
72
|
+
return [
|
|
73
|
+
path.join(root, 'dist', 'src', 'index.js'),
|
|
74
|
+
path.join(root, 'dist', 'index.js'),
|
|
75
|
+
path.join(root, 'node_modules', '@zintrust', 'core', 'dist', 'src', 'index.js'),
|
|
76
|
+
path.join(root, 'node_modules', '@zintrust', 'core', 'dist', 'index.js'),
|
|
77
|
+
path.join(root, 'node_modules', '@zintrust', 'core', 'src', 'index.js'),
|
|
78
|
+
path.join(root, 'node_modules', '@zintrust', 'core', 'index.js'),
|
|
79
|
+
path.join(root, 'src', 'index.ts'),
|
|
80
|
+
];
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (specifier === '@zintrust/workers') {
|
|
84
|
+
return [
|
|
85
|
+
path.join(root, 'dist', 'packages', 'workers', 'src', 'index.js'),
|
|
86
|
+
path.join(root, 'node_modules', '@zintrust', 'workers', 'dist', 'src', 'index.js'),
|
|
87
|
+
path.join(root, 'node_modules', '@zintrust', 'workers', 'dist', 'index.js'),
|
|
88
|
+
path.join(root, 'node_modules', '@zintrust', 'workers', 'src', 'index.js'),
|
|
89
|
+
path.join(root, 'node_modules', '@zintrust', 'workers', 'index.js'),
|
|
90
|
+
path.join(root, 'packages', 'workers', 'src', 'index.ts'),
|
|
91
|
+
];
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return [];
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const getProjectFileCandidates = (paths: string[]): string | null => {
|
|
98
|
+
if (!canUseProjectFileImports()) return null;
|
|
99
|
+
for (const candidate of paths) {
|
|
100
|
+
if (NodeSingletons.fs.existsSync(candidate)) return candidate;
|
|
101
|
+
}
|
|
102
|
+
return null;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
const resolveLocalPackageFallback = (specifier: string): string | null => {
|
|
106
|
+
if (!canUseProjectFileImports()) return null;
|
|
107
|
+
const root = resolveProjectRoot();
|
|
108
|
+
|
|
109
|
+
const candidates = buildCandidatesForSpecifier(specifier, root);
|
|
110
|
+
const resolved = getProjectFileCandidates(candidates);
|
|
111
|
+
if (!resolved) return null;
|
|
112
|
+
return NodeSingletons.url.pathToFileURL(resolved).href;
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
const resolvePackageSpecifierUrl = (specifier: string): string | null => {
|
|
116
|
+
if (!isNodeRuntime() || !canUseProjectFileImports()) return null;
|
|
117
|
+
if (typeof NodeSingletons?.module?.createRequire !== 'function') {
|
|
118
|
+
return resolveLocalPackageFallback(specifier);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
try {
|
|
122
|
+
const require = NodeSingletons.module.createRequire(import.meta.url);
|
|
123
|
+
const resolved = require.resolve(specifier);
|
|
124
|
+
if (
|
|
125
|
+
(specifier === '@zintrust/workers' &&
|
|
126
|
+
resolved.includes(
|
|
127
|
+
`${path.sep}node_modules${path.sep}@zintrust${path.sep}workers${path.sep}`
|
|
128
|
+
)) ||
|
|
129
|
+
(specifier === '@zintrust/core' &&
|
|
130
|
+
resolved.includes(`${path.sep}node_modules${path.sep}@zintrust${path.sep}core${path.sep}`))
|
|
131
|
+
) {
|
|
132
|
+
const local = resolveLocalPackageFallback(specifier);
|
|
133
|
+
if (local) return local;
|
|
134
|
+
}
|
|
135
|
+
return NodeSingletons.url.pathToFileURL(resolved).href;
|
|
136
|
+
} catch {
|
|
137
|
+
return resolveLocalPackageFallback(specifier);
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
const escapeRegExp = (value: string): string =>
|
|
142
|
+
value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`);
|
|
143
|
+
|
|
144
|
+
const rewriteProcessorImports = (code: string): string => {
|
|
145
|
+
const replacements: Array<{ from: string; to: string }> = [];
|
|
146
|
+
const coreUrl = resolvePackageSpecifierUrl('@zintrust/core');
|
|
147
|
+
if (coreUrl) replacements.push({ from: '@zintrust/core', to: coreUrl });
|
|
148
|
+
const workersUrl = resolvePackageSpecifierUrl('@zintrust/workers');
|
|
149
|
+
if (workersUrl) replacements.push({ from: '@zintrust/workers', to: workersUrl });
|
|
150
|
+
|
|
151
|
+
if (replacements.length === 0) return code;
|
|
152
|
+
|
|
153
|
+
let updated = code;
|
|
154
|
+
for (const { from, to } of replacements) {
|
|
155
|
+
const pattern = new RegExp(String.raw`(['"])${escapeRegExp(from)}\1`, 'g');
|
|
156
|
+
updated = updated.replace(pattern, `$1${to}$1`);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return updated;
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
const ensureProcessorSpecDir = (): string | null => {
|
|
163
|
+
if (!isNodeRuntime() || !canUseProjectFileImports()) return null;
|
|
164
|
+
const dir = path.join(resolveProjectRoot(), '.zintrust', 'processor-specs');
|
|
165
|
+
try {
|
|
166
|
+
if (!NodeSingletons.fs.existsSync(dir)) {
|
|
167
|
+
NodeSingletons.fs.mkdirSync(dir, { recursive: true });
|
|
168
|
+
}
|
|
169
|
+
return dir;
|
|
170
|
+
} catch (error) {
|
|
171
|
+
Logger.debug('Failed to prepare processor spec cache directory', error);
|
|
172
|
+
return null;
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
const shouldFallbackToFileImport = (error: unknown): boolean => {
|
|
177
|
+
if (!isNodeRuntime()) return false;
|
|
178
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
179
|
+
const code = (error as { code?: string } | undefined)?.code ?? '';
|
|
180
|
+
if (code === 'ERR_INVALID_URL' || code === 'ERR_UNSUPPORTED_ESM_URL_SCHEME') return true;
|
|
181
|
+
return (
|
|
182
|
+
message.includes('Invalid relative URL') ||
|
|
183
|
+
message.includes('base scheme is not hierarchical') ||
|
|
184
|
+
message.includes('Failed to resolve module specifier')
|
|
185
|
+
);
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
const isOptionalD1ProxyModuleMissing = (error: unknown): boolean => {
|
|
189
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
190
|
+
const code = (error as { code?: string } | undefined)?.code ?? '';
|
|
191
|
+
|
|
192
|
+
if (code !== 'ERR_MODULE_NOT_FOUND') return false;
|
|
193
|
+
|
|
194
|
+
return (
|
|
195
|
+
message.includes('cloudflare-d1-proxy') ||
|
|
196
|
+
message.includes('/packages/cloudflare-d1-proxy/src/index.js')
|
|
197
|
+
);
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
const importModuleFromCode = async (params: {
|
|
201
|
+
code: string;
|
|
202
|
+
normalized: string;
|
|
203
|
+
cacheKey: string;
|
|
204
|
+
}): Promise<Record<string, unknown>> => {
|
|
205
|
+
const { code, normalized, cacheKey } = params;
|
|
206
|
+
const dataUrl = `data:text/javascript;base64,${toBase64(code)}`;
|
|
207
|
+
|
|
208
|
+
try {
|
|
209
|
+
return (await import(dataUrl)) as Record<string, unknown>;
|
|
210
|
+
} catch (error) {
|
|
211
|
+
if (!shouldFallbackToFileImport(error)) throw error;
|
|
212
|
+
const dir = ensureProcessorSpecDir();
|
|
213
|
+
if (!dir) throw error;
|
|
214
|
+
|
|
215
|
+
try {
|
|
216
|
+
const codeHash = await computeSha256(code);
|
|
217
|
+
const filePath = path.join(dir, `${codeHash || cacheKey}.mjs`);
|
|
218
|
+
NodeSingletons.fs.writeFileSync(filePath, code, 'utf8');
|
|
219
|
+
const fileUrl = NodeSingletons.url.pathToFileURL(filePath).href;
|
|
220
|
+
return (await import(fileUrl)) as Record<string, unknown>;
|
|
221
|
+
} catch (fileError) {
|
|
222
|
+
if (isOptionalD1ProxyModuleMissing(fileError)) {
|
|
223
|
+
throw fileError;
|
|
224
|
+
}
|
|
225
|
+
Logger.debug(`Processor URL file fallback failed for ${normalized}`, fileError);
|
|
226
|
+
throw error;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
const isCloudflareRuntime = (): boolean => Cloudflare.getWorkersEnv() !== null;
|
|
232
|
+
|
|
233
|
+
const getDefaultStoreForRuntime = async (): Promise<WorkerStore> => {
|
|
234
|
+
if (!isCloudflareRuntime()) {
|
|
235
|
+
await ensureWorkerStoreConfigured();
|
|
236
|
+
return workerStore;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const bootstrapConfig = buildPersistenceBootstrapConfig();
|
|
240
|
+
const persistence = resolvePersistenceConfig(bootstrapConfig);
|
|
241
|
+
if (!persistence) {
|
|
242
|
+
return InMemoryWorkerStore.create();
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
return resolveWorkerStoreForPersistence(persistence);
|
|
246
|
+
};
|
|
247
|
+
|
|
52
248
|
const getStoreForWorker = async (
|
|
53
249
|
config: WorkerFactoryConfig | undefined,
|
|
54
250
|
persistenceOverride?: WorkerPersistenceConfig
|
|
@@ -66,8 +262,7 @@ const getStoreForWorker = async (
|
|
|
66
262
|
}
|
|
67
263
|
|
|
68
264
|
// Fallback to default/global store
|
|
69
|
-
|
|
70
|
-
return workerStore;
|
|
265
|
+
return getDefaultStoreForRuntime();
|
|
71
266
|
};
|
|
72
267
|
|
|
73
268
|
const validateAndGetStore = async (
|
|
@@ -311,7 +506,7 @@ const computeSha256 = async (value: string): Promise<string> => {
|
|
|
311
506
|
return NodeSingletons.createHash('sha256').update(value).digest('hex');
|
|
312
507
|
}
|
|
313
508
|
|
|
314
|
-
return String(
|
|
509
|
+
return String(generateUuid()).slice(2);
|
|
315
510
|
};
|
|
316
511
|
|
|
317
512
|
const toBase64 = (value: string): string => {
|
|
@@ -448,6 +643,17 @@ const stripProcessorExtension = (value: string): string => value.replace(/\.(ts|
|
|
|
448
643
|
|
|
449
644
|
const normalizeModulePath = (value: string): string => value.replaceAll('\\', '/');
|
|
450
645
|
|
|
646
|
+
const filterExistingFileCandidates = (candidates: string[]): string[] => {
|
|
647
|
+
if (!NodeSingletons?.fs?.existsSync) return candidates;
|
|
648
|
+
return candidates.filter((candidate) => {
|
|
649
|
+
try {
|
|
650
|
+
return NodeSingletons.fs.existsSync(candidate);
|
|
651
|
+
} catch {
|
|
652
|
+
return false;
|
|
653
|
+
}
|
|
654
|
+
});
|
|
655
|
+
};
|
|
656
|
+
|
|
451
657
|
const buildProcessorModuleCandidates = (modulePath: string, resolvedPath: string): string[] => {
|
|
452
658
|
const candidates: string[] = [];
|
|
453
659
|
const normalized = normalizeModulePath(modulePath.trim());
|
|
@@ -470,6 +676,31 @@ const buildProcessorModuleCandidates = (modulePath: string, resolvedPath: string
|
|
|
470
676
|
return Array.from(new Set(candidates));
|
|
471
677
|
};
|
|
472
678
|
|
|
679
|
+
const buildProcessorFilePathCandidates = (_modulePath: string, resolvedPath: string): string[] => {
|
|
680
|
+
const candidates: string[] = [];
|
|
681
|
+
const normalizedResolved = normalizeModulePath(resolvedPath);
|
|
682
|
+
const projectRoot = normalizeModulePath(resolveProjectRoot());
|
|
683
|
+
|
|
684
|
+
const strippedResolved = stripProcessorExtension(resolvedPath);
|
|
685
|
+
candidates.push(`${strippedResolved}.js`, `${strippedResolved}.mjs`);
|
|
686
|
+
|
|
687
|
+
const appIndex = normalizedResolved.lastIndexOf('/app/');
|
|
688
|
+
if (appIndex !== -1) {
|
|
689
|
+
const relative = normalizedResolved.slice(appIndex + 5);
|
|
690
|
+
if (relative) {
|
|
691
|
+
const strippedRelative = stripProcessorExtension(relative);
|
|
692
|
+
candidates.push(
|
|
693
|
+
path.join(projectRoot, 'dist', 'app', `${strippedRelative}.js`),
|
|
694
|
+
path.join(projectRoot, 'app', relative),
|
|
695
|
+
path.join(projectRoot, 'app', `${strippedRelative}.js`),
|
|
696
|
+
path.join('/app', 'dist', 'app', `${strippedRelative}.js`)
|
|
697
|
+
);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
return filterExistingFileCandidates(Array.from(new Set(candidates)));
|
|
702
|
+
};
|
|
703
|
+
|
|
473
704
|
const pickProcessorFromModule = (
|
|
474
705
|
mod: Record<string, unknown> | undefined,
|
|
475
706
|
source: string
|
|
@@ -544,9 +775,9 @@ const cacheProcessorFromResponse = async (params: {
|
|
|
544
775
|
cacheKey: string;
|
|
545
776
|
}): Promise<WorkerFactoryConfig['processor']> => {
|
|
546
777
|
const { response, normalized, config, cacheKey } = params;
|
|
547
|
-
const
|
|
548
|
-
const
|
|
549
|
-
const mod = await
|
|
778
|
+
const rawCode = await readResponseBody(response, config.fetchMaxSizeBytes);
|
|
779
|
+
const code = rewriteProcessorImports(rawCode);
|
|
780
|
+
const mod = await importModuleFromCode({ code, normalized, cacheKey });
|
|
550
781
|
const processor = extractZinTrustProcessor(mod as Record<string, unknown>, normalized);
|
|
551
782
|
if (!processor) {
|
|
552
783
|
throw ErrorFactory.createConfigError('INVALID_PROCESSOR_URL_EXPORT');
|
|
@@ -611,6 +842,13 @@ const fetchProcessorAttempt = async (params: {
|
|
|
611
842
|
|
|
612
843
|
return await cacheProcessorFromResponse({ response, normalized, config, cacheKey });
|
|
613
844
|
} catch (error) {
|
|
845
|
+
if (isOptionalD1ProxyModuleMissing(error)) {
|
|
846
|
+
Logger.warn(
|
|
847
|
+
'Processor URL skipped: optional cloudflare-d1-proxy module is unavailable in this runtime.'
|
|
848
|
+
);
|
|
849
|
+
return undefined;
|
|
850
|
+
}
|
|
851
|
+
|
|
614
852
|
if (controller.signal.aborted) {
|
|
615
853
|
Logger.error('Processor URL fetch timeout', error);
|
|
616
854
|
} else {
|
|
@@ -656,10 +894,12 @@ const resolveProcessorFromUrl = async (
|
|
|
656
894
|
Logger.warn(
|
|
657
895
|
`Invalid processor URL protocol: ${parsed.protocol}. Only https:// and file:// are supported.`
|
|
658
896
|
);
|
|
897
|
+
return undefined;
|
|
659
898
|
}
|
|
660
899
|
|
|
661
900
|
if (!isAllowedRemoteHost(parsed.host) && parsed.protocol !== 'file:') {
|
|
662
901
|
Logger.warn(`Invalid processor URL host: ${parsed.host}. Host is not in the allowlist.`);
|
|
902
|
+
return undefined;
|
|
663
903
|
}
|
|
664
904
|
|
|
665
905
|
const config = getProcessorSpecConfig();
|
|
@@ -703,8 +943,12 @@ const resolveProcessorFromPath = async (
|
|
|
703
943
|
if (candidates.length === 0) return undefined;
|
|
704
944
|
const [candidatePath, ...rest] = candidates;
|
|
705
945
|
try {
|
|
706
|
-
const
|
|
707
|
-
|
|
946
|
+
const importPath =
|
|
947
|
+
candidatePath.startsWith('/') && !candidatePath.startsWith('//')
|
|
948
|
+
? NodeSingletons.url.pathToFileURL(candidatePath).href
|
|
949
|
+
: candidatePath;
|
|
950
|
+
const mod = await import(importPath);
|
|
951
|
+
const candidate = pickProcessorFromModule(mod as Record<string, unknown>, importPath);
|
|
708
952
|
if (candidate) return candidate;
|
|
709
953
|
} catch (candidateError) {
|
|
710
954
|
Logger.debug(`Processor module candidate import failed: ${candidatePath}`, candidateError);
|
|
@@ -718,9 +962,13 @@ const resolveProcessorFromPath = async (
|
|
|
718
962
|
const candidate = pickProcessorFromModule(mod as Record<string, unknown>, resolved);
|
|
719
963
|
if (candidate) return candidate;
|
|
720
964
|
} catch (err) {
|
|
721
|
-
const
|
|
722
|
-
const
|
|
723
|
-
if (
|
|
965
|
+
const fileCandidates = buildProcessorFilePathCandidates(trimmed, resolved);
|
|
966
|
+
const resolvedFileCandidate = await importProcessorFromCandidates(fileCandidates);
|
|
967
|
+
if (resolvedFileCandidate) return resolvedFileCandidate;
|
|
968
|
+
|
|
969
|
+
const moduleCandidates = buildProcessorModuleCandidates(trimmed, resolved);
|
|
970
|
+
const resolvedModuleCandidate = await importProcessorFromCandidates(moduleCandidates);
|
|
971
|
+
if (resolvedModuleCandidate) return resolvedModuleCandidate;
|
|
724
972
|
Logger.error(`Failed to import processor from path: ${resolved}`, err);
|
|
725
973
|
}
|
|
726
974
|
|
|
@@ -1096,6 +1344,76 @@ const handleFailure = async (params: {
|
|
|
1096
1344
|
await executeFailurePlugins(workerName, job, error, features);
|
|
1097
1345
|
};
|
|
1098
1346
|
|
|
1347
|
+
const toBackoffDelayMs = (backoff: unknown): number => {
|
|
1348
|
+
if (typeof backoff === 'number' && Number.isFinite(backoff)) {
|
|
1349
|
+
return Math.max(0, Math.floor(backoff));
|
|
1350
|
+
}
|
|
1351
|
+
if (backoff !== null && backoff !== undefined && typeof backoff === 'object') {
|
|
1352
|
+
const raw = (backoff as { delay?: unknown }).delay;
|
|
1353
|
+
if (typeof raw === 'number' && Number.isFinite(raw)) {
|
|
1354
|
+
return Math.max(0, Math.floor(raw));
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
return 0;
|
|
1358
|
+
};
|
|
1359
|
+
|
|
1360
|
+
const trackJobStarted = async (input: {
|
|
1361
|
+
queueName: string;
|
|
1362
|
+
job: Job;
|
|
1363
|
+
attempts: number;
|
|
1364
|
+
workerName: string;
|
|
1365
|
+
workerVersion: string;
|
|
1366
|
+
}): Promise<void> => {
|
|
1367
|
+
if (!input.job.id) return;
|
|
1368
|
+
await JobStateTracker.started({
|
|
1369
|
+
queueName: input.queueName,
|
|
1370
|
+
jobId: input.job.id,
|
|
1371
|
+
attempts: input.attempts,
|
|
1372
|
+
timeoutMs: Math.max(1000, Env.getInt('QUEUE_JOB_TIMEOUT', 60) * 1000),
|
|
1373
|
+
workerName: input.workerName,
|
|
1374
|
+
workerVersion: input.workerVersion,
|
|
1375
|
+
});
|
|
1376
|
+
};
|
|
1377
|
+
|
|
1378
|
+
const trackJobCompleted = async (input: {
|
|
1379
|
+
queueName: string;
|
|
1380
|
+
job: Job;
|
|
1381
|
+
duration: number;
|
|
1382
|
+
result: unknown;
|
|
1383
|
+
}): Promise<void> => {
|
|
1384
|
+
if (!input.job.id) return;
|
|
1385
|
+
await JobStateTracker.completed({
|
|
1386
|
+
queueName: input.queueName,
|
|
1387
|
+
jobId: input.job.id,
|
|
1388
|
+
processingTimeMs: input.duration,
|
|
1389
|
+
result: input.result,
|
|
1390
|
+
});
|
|
1391
|
+
};
|
|
1392
|
+
|
|
1393
|
+
const trackJobFailed = async (input: {
|
|
1394
|
+
queueName: string;
|
|
1395
|
+
job: Job;
|
|
1396
|
+
attempts: number;
|
|
1397
|
+
maxAttempts?: number;
|
|
1398
|
+
error: Error;
|
|
1399
|
+
}): Promise<void> => {
|
|
1400
|
+
if (!input.job.id) return;
|
|
1401
|
+
const isFinal = input.maxAttempts === undefined ? true : input.attempts >= input.maxAttempts;
|
|
1402
|
+
const backoffDelayMs = toBackoffDelayMs(input.job.opts?.backoff);
|
|
1403
|
+
|
|
1404
|
+
await JobStateTracker.failed({
|
|
1405
|
+
queueName: input.queueName,
|
|
1406
|
+
jobId: input.job.id,
|
|
1407
|
+
attempts: input.attempts,
|
|
1408
|
+
isFinal,
|
|
1409
|
+
retryAt:
|
|
1410
|
+
!isFinal && backoffDelayMs > 0
|
|
1411
|
+
? new Date(Date.now() + backoffDelayMs).toISOString()
|
|
1412
|
+
: undefined,
|
|
1413
|
+
error: input.error,
|
|
1414
|
+
});
|
|
1415
|
+
};
|
|
1416
|
+
|
|
1099
1417
|
/**
|
|
1100
1418
|
* Helper: Create enhanced processor with all features
|
|
1101
1419
|
*/
|
|
@@ -1119,9 +1437,23 @@ const createEnhancedProcessor = (config: WorkerFactoryConfig): ((job: Job) => Pr
|
|
|
1119
1437
|
let result: unknown;
|
|
1120
1438
|
let spanId: string | null = null;
|
|
1121
1439
|
|
|
1440
|
+
const maxAttempts =
|
|
1441
|
+
typeof job.opts?.attempts === 'number' && Number.isFinite(job.opts.attempts)
|
|
1442
|
+
? Math.max(1, Math.floor(job.opts.attempts))
|
|
1443
|
+
: undefined;
|
|
1444
|
+
const attempts = Math.max(1, Math.floor((job.attemptsMade ?? 0) + 1));
|
|
1445
|
+
|
|
1122
1446
|
try {
|
|
1123
1447
|
spanId = startProcessingSpan(name, jobVersion, job, config.queueName, features);
|
|
1124
1448
|
|
|
1449
|
+
await trackJobStarted({
|
|
1450
|
+
queueName: config.queueName,
|
|
1451
|
+
job,
|
|
1452
|
+
attempts,
|
|
1453
|
+
workerName: name,
|
|
1454
|
+
workerVersion: jobVersion,
|
|
1455
|
+
});
|
|
1456
|
+
|
|
1125
1457
|
// Process the job
|
|
1126
1458
|
result = await processor(job);
|
|
1127
1459
|
|
|
@@ -1136,11 +1468,21 @@ const createEnhancedProcessor = (config: WorkerFactoryConfig): ((job: Job) => Pr
|
|
|
1136
1468
|
features,
|
|
1137
1469
|
});
|
|
1138
1470
|
|
|
1471
|
+
await trackJobCompleted({ queueName: config.queueName, job, duration, result });
|
|
1472
|
+
|
|
1139
1473
|
return result;
|
|
1140
1474
|
} catch (err) {
|
|
1141
1475
|
const error = err as Error;
|
|
1142
1476
|
const duration = Date.now() - startTime;
|
|
1143
1477
|
|
|
1478
|
+
await trackJobFailed({
|
|
1479
|
+
queueName: config.queueName,
|
|
1480
|
+
job,
|
|
1481
|
+
attempts,
|
|
1482
|
+
maxAttempts,
|
|
1483
|
+
error,
|
|
1484
|
+
});
|
|
1485
|
+
|
|
1144
1486
|
await handleFailure({
|
|
1145
1487
|
workerName: name,
|
|
1146
1488
|
jobVersion,
|
|
@@ -1226,12 +1568,23 @@ const resolveRedisConfigFromEnv = (config: RedisEnvConfig, context: string): Red
|
|
|
1226
1568
|
};
|
|
1227
1569
|
};
|
|
1228
1570
|
|
|
1229
|
-
const resolveRedisConfigFromDirect = (config: RedisConfig, context: string): RedisConfig =>
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1571
|
+
const resolveRedisConfigFromDirect = (config: RedisConfig, context: string): RedisConfig => {
|
|
1572
|
+
const fallbackDb = Env.getInt('REDIS_QUEUE_DB', ZintrustLang.REDIS_DEFAULT_DB);
|
|
1573
|
+
|
|
1574
|
+
let normalizedDb = fallbackDb;
|
|
1575
|
+
if (typeof config.db === 'number') {
|
|
1576
|
+
normalizedDb = config.db;
|
|
1577
|
+
} else if (typeof (config as { database?: number }).database === 'number') {
|
|
1578
|
+
normalizedDb = (config as { database?: number }).database as number;
|
|
1579
|
+
}
|
|
1580
|
+
|
|
1581
|
+
return {
|
|
1582
|
+
host: requireRedisHost(config.host, context),
|
|
1583
|
+
port: config.port,
|
|
1584
|
+
db: normalizedDb,
|
|
1585
|
+
password: config.password ?? Env.get('REDIS_PASSWORD', undefined),
|
|
1586
|
+
};
|
|
1587
|
+
};
|
|
1235
1588
|
|
|
1236
1589
|
const resolveRedisConfig = (config: RedisConfigInput, context: string): RedisConfig =>
|
|
1237
1590
|
isRedisEnvConfig(config)
|
|
@@ -1252,18 +1605,26 @@ const resolveRedisConfigWithFallback = (
|
|
|
1252
1605
|
return resolveRedisConfig(selected, context);
|
|
1253
1606
|
};
|
|
1254
1607
|
|
|
1608
|
+
const logRedisPersistenceConfig = (
|
|
1609
|
+
redisConfig: RedisConfig,
|
|
1610
|
+
key_prefix: string,
|
|
1611
|
+
source: string
|
|
1612
|
+
): void => {
|
|
1613
|
+
Logger.debug('Worker persistence redis config', {
|
|
1614
|
+
source,
|
|
1615
|
+
host: redisConfig.host,
|
|
1616
|
+
port: redisConfig.port,
|
|
1617
|
+
db: redisConfig.db,
|
|
1618
|
+
key_prefix,
|
|
1619
|
+
});
|
|
1620
|
+
};
|
|
1621
|
+
|
|
1255
1622
|
const normalizeEnvValue = (value: string | undefined): string | undefined => {
|
|
1256
1623
|
if (!value) return undefined;
|
|
1257
1624
|
const trimmed = value.trim();
|
|
1258
1625
|
return trimmed.length > 0 ? trimmed : undefined;
|
|
1259
1626
|
};
|
|
1260
1627
|
|
|
1261
|
-
const normalizeAppName = (value: string | undefined): string => {
|
|
1262
|
-
const normalized = (value ?? '').trim().replaceAll(/\s+/g, '_');
|
|
1263
|
-
return normalized.length > 0 ? normalized : 'zintrust';
|
|
1264
|
-
};
|
|
1265
|
-
|
|
1266
|
-
const resolveDefaultRedisKeyPrefix = (): string => 'worker_' + normalizeAppName(appConfig.prefix);
|
|
1267
1628
|
const resolveDefaultPersistenceTable = (): string =>
|
|
1268
1629
|
normalizeEnvValue(Env.get('WORKER_PERSISTENCE_TABLE', 'zintrust_workers')) ?? 'zintrust_workers';
|
|
1269
1630
|
|
|
@@ -1288,10 +1649,7 @@ const normalizeExplicitPersistence = (
|
|
|
1288
1649
|
return {
|
|
1289
1650
|
driver: 'redis',
|
|
1290
1651
|
redis: persistence.redis,
|
|
1291
|
-
keyPrefix:
|
|
1292
|
-
persistence.keyPrefix ??
|
|
1293
|
-
normalizeEnvValue(Env.get('WORKER_PERSISTENCE_REDIS_KEY_PREFIX', '')) ??
|
|
1294
|
-
resolveDefaultRedisKeyPrefix(),
|
|
1652
|
+
keyPrefix: keyPrefix(),
|
|
1295
1653
|
};
|
|
1296
1654
|
}
|
|
1297
1655
|
|
|
@@ -1326,11 +1684,16 @@ const resolvePersistenceConfig = (
|
|
|
1326
1684
|
if (driver === 'memory') return { driver: 'memory' };
|
|
1327
1685
|
|
|
1328
1686
|
if (driver === 'redis') {
|
|
1329
|
-
const
|
|
1687
|
+
const persistenceDbOverride = normalizeEnvValue(Env.get('WORKER_PERSISTENCE_REDIS_DB', ''));
|
|
1688
|
+
|
|
1330
1689
|
return {
|
|
1331
1690
|
driver: 'redis',
|
|
1332
|
-
|
|
1333
|
-
|
|
1691
|
+
// Optional override; otherwise defaults to REDIS_QUEUE_DB.
|
|
1692
|
+
redis: {
|
|
1693
|
+
env: true,
|
|
1694
|
+
db: persistenceDbOverride ? 'WORKER_PERSISTENCE_REDIS_DB' : 'REDIS_QUEUE_DB',
|
|
1695
|
+
},
|
|
1696
|
+
keyPrefix: keyPrefix(),
|
|
1334
1697
|
};
|
|
1335
1698
|
}
|
|
1336
1699
|
|
|
@@ -1383,8 +1746,10 @@ const resolveWorkerStore = async (config: WorkerFactoryConfig): Promise<WorkerSt
|
|
|
1383
1746
|
'Worker persistence requires redis config (persistence.redis or infrastructure.redis)',
|
|
1384
1747
|
'infrastructure.persistence.redis'
|
|
1385
1748
|
);
|
|
1749
|
+
const key_prefix = persistence.keyPrefix ?? keyPrefix();
|
|
1750
|
+
logRedisPersistenceConfig(redisConfig, key_prefix, 'resolveWorkerStore');
|
|
1386
1751
|
const client = createRedisConnection(redisConfig);
|
|
1387
|
-
next = RedisWorkerStore.create(client,
|
|
1752
|
+
next = RedisWorkerStore.create(client, key_prefix);
|
|
1388
1753
|
} else if (persistence.driver === 'database') {
|
|
1389
1754
|
const explicitConnection =
|
|
1390
1755
|
typeof persistence.client === 'string' ? persistence.client : persistence.connection;
|
|
@@ -1435,8 +1800,10 @@ const createWorkerStore = async (persistence: WorkerPersistenceConfig): Promise<
|
|
|
1435
1800
|
'Worker persistence requires redis config (persistence.redis or REDIS_* env values)',
|
|
1436
1801
|
'persistence.redis'
|
|
1437
1802
|
);
|
|
1803
|
+
const key_prefix = persistence.keyPrefix ?? keyPrefix();
|
|
1804
|
+
logRedisPersistenceConfig(redisConfig, key_prefix, 'createWorkerStore');
|
|
1438
1805
|
const client = createRedisConnection(redisConfig);
|
|
1439
|
-
return RedisWorkerStore.create(client,
|
|
1806
|
+
return RedisWorkerStore.create(client, key_prefix);
|
|
1440
1807
|
}
|
|
1441
1808
|
|
|
1442
1809
|
// Database driver
|
|
@@ -1453,10 +1820,12 @@ const resolveWorkerStoreForPersistence = async (
|
|
|
1453
1820
|
persistence: WorkerPersistenceConfig
|
|
1454
1821
|
): Promise<WorkerStore> => {
|
|
1455
1822
|
const cacheKey = generateCacheKey(persistence);
|
|
1823
|
+
const isCloudflare = Cloudflare.getWorkersEnv() !== null;
|
|
1456
1824
|
|
|
1457
|
-
// Return cached instance if available
|
|
1825
|
+
// Return cached instance if available (disable cache for Cloudflare to assume cleanup)
|
|
1826
|
+
// Or handle cleanup differently. For now, disable cache for Cloudflare to allow per-request connections.
|
|
1458
1827
|
const cached = storeInstanceCache.get(cacheKey);
|
|
1459
|
-
if (cached) {
|
|
1828
|
+
if (cached && !isCloudflare) {
|
|
1460
1829
|
return cached;
|
|
1461
1830
|
}
|
|
1462
1831
|
|
|
@@ -1464,8 +1833,10 @@ const resolveWorkerStoreForPersistence = async (
|
|
|
1464
1833
|
const store = await createWorkerStore(persistence);
|
|
1465
1834
|
await store.init();
|
|
1466
1835
|
|
|
1467
|
-
// Cache the store instance for reuse
|
|
1468
|
-
|
|
1836
|
+
// Cache the store instance for reuse only if not Cloudflare
|
|
1837
|
+
if (!isCloudflare) {
|
|
1838
|
+
storeInstanceCache.set(cacheKey, store);
|
|
1839
|
+
}
|
|
1469
1840
|
|
|
1470
1841
|
return store;
|
|
1471
1842
|
};
|
|
@@ -1475,8 +1846,19 @@ const getPersistedRecord = async (
|
|
|
1475
1846
|
persistenceOverride?: WorkerPersistenceConfig
|
|
1476
1847
|
): Promise<WorkerRecord | null> => {
|
|
1477
1848
|
if (!persistenceOverride) {
|
|
1478
|
-
|
|
1479
|
-
|
|
1849
|
+
if (!isCloudflareRuntime()) {
|
|
1850
|
+
await ensureWorkerStoreConfigured();
|
|
1851
|
+
return workerStore.get(name);
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
const store = await getDefaultStoreForRuntime();
|
|
1855
|
+
try {
|
|
1856
|
+
return await store.get(name);
|
|
1857
|
+
} finally {
|
|
1858
|
+
if (store.close) {
|
|
1859
|
+
await store.close();
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
1480
1862
|
}
|
|
1481
1863
|
|
|
1482
1864
|
const store = await resolveWorkerStoreForPersistence(persistenceOverride);
|
|
@@ -1934,6 +2316,37 @@ export const WorkerFactory = Object.freeze({
|
|
|
1934
2316
|
resolveProcessorPath,
|
|
1935
2317
|
resolveProcessorSpec,
|
|
1936
2318
|
|
|
2319
|
+
/**
|
|
2320
|
+
* Register a new worker configuration without starting it.
|
|
2321
|
+
*/
|
|
2322
|
+
async register(config: WorkerFactoryConfig): Promise<void> {
|
|
2323
|
+
const { name } = config;
|
|
2324
|
+
// Check in-memory first (though unlikely if we are just registering)
|
|
2325
|
+
if (workers.has(name)) {
|
|
2326
|
+
throw ErrorFactory.createWorkerError(`Worker "${name}" is already running locally`);
|
|
2327
|
+
}
|
|
2328
|
+
|
|
2329
|
+
const store = await getStoreForWorker(config);
|
|
2330
|
+
try {
|
|
2331
|
+
const existing = await store.get(name);
|
|
2332
|
+
if (existing) {
|
|
2333
|
+
throw ErrorFactory.createWorkerError(`Worker "${name}" already exists in persistence`);
|
|
2334
|
+
}
|
|
2335
|
+
|
|
2336
|
+
// Init features to validate config, but mainly we just want to save it.
|
|
2337
|
+
// initializeWorkerFeatures might rely on being active or having resources, so we might skip it or do partial.
|
|
2338
|
+
// For now, just save definition.
|
|
2339
|
+
// Status should be STOPPED or CREATED.
|
|
2340
|
+
await store.save(buildWorkerRecord(config, WorkerCreationStatus.STOPPED));
|
|
2341
|
+
Logger.info(`Worker registered (persistence only): ${name}`);
|
|
2342
|
+
} finally {
|
|
2343
|
+
// If Cloudflare environment, try to close store connection to avoid zombie connections
|
|
2344
|
+
if (Cloudflare.getWorkersEnv() !== null && store.close) {
|
|
2345
|
+
await store.close();
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
},
|
|
2349
|
+
|
|
1937
2350
|
/**
|
|
1938
2351
|
* Create new worker with full setup
|
|
1939
2352
|
*/
|
|
@@ -2398,9 +2811,25 @@ export const WorkerFactory = Object.freeze({
|
|
|
2398
2811
|
): Promise<WorkerRecord[]> {
|
|
2399
2812
|
const includeInactive = options?.includeInactive === true;
|
|
2400
2813
|
if (!persistenceOverride) {
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2814
|
+
if (!isCloudflareRuntime()) {
|
|
2815
|
+
await ensureWorkerStoreConfigured();
|
|
2816
|
+
const records = await workerStore.list(options);
|
|
2817
|
+
return includeInactive
|
|
2818
|
+
? records
|
|
2819
|
+
: records.filter((record) => record.activeStatus !== false);
|
|
2820
|
+
}
|
|
2821
|
+
|
|
2822
|
+
const store = await getDefaultStoreForRuntime();
|
|
2823
|
+
try {
|
|
2824
|
+
const records = await store.list(options);
|
|
2825
|
+
return includeInactive
|
|
2826
|
+
? records
|
|
2827
|
+
: records.filter((record) => record.activeStatus !== false);
|
|
2828
|
+
} finally {
|
|
2829
|
+
if (store.close) {
|
|
2830
|
+
await store.close();
|
|
2831
|
+
}
|
|
2832
|
+
}
|
|
2404
2833
|
}
|
|
2405
2834
|
|
|
2406
2835
|
const store = await resolveWorkerStoreForPersistence(persistenceOverride);
|
|
@@ -2465,7 +2894,15 @@ export const WorkerFactory = Object.freeze({
|
|
|
2465
2894
|
): Promise<WorkerRecord | null> {
|
|
2466
2895
|
const instance = workers.get(name);
|
|
2467
2896
|
const store = await getStoreForWorker(instance?.config, persistenceOverride);
|
|
2468
|
-
|
|
2897
|
+
|
|
2898
|
+
try {
|
|
2899
|
+
const result = await store.get(name);
|
|
2900
|
+
return result;
|
|
2901
|
+
} finally {
|
|
2902
|
+
if (Cloudflare.getWorkersEnv() !== null && store.close) {
|
|
2903
|
+
await store.close();
|
|
2904
|
+
}
|
|
2905
|
+
}
|
|
2469
2906
|
},
|
|
2470
2907
|
|
|
2471
2908
|
/**
|