@zintrust/core 0.1.42 → 0.1.44

package/packages/workers/src/WorkerFactory.js

@@ -0,0 +1,2253 @@
+/**
+ * Worker Factory
+ * Central factory for creating workers with all advanced features
+ * Sealed namespace for immutability
+ */
+import { Cloudflare, createRedisConnection, databaseConfig, Env, ErrorFactory, generateUuid, getBullMQSafeQueueName, JobStateTracker, Logger, NodeSingletons, queueConfig, registerDatabasesFromRuntimeConfig, useEnsureDbConnected, workersConfig, ZintrustLang, } from '../../../src/index.js';
+import { Worker } from 'bullmq';
+import { AutoScaler } from './AutoScaler.js';
+import { CanaryController } from './CanaryController.js';
+import { CircuitBreaker } from './CircuitBreaker.js';
+import { ClusterLock } from './ClusterLock.js';
+import { ComplianceManager } from './ComplianceManager.js';
+import { DatacenterOrchestrator } from './DatacenterOrchestrator.js';
+import { DeadLetterQueue } from './DeadLetterQueue.js';
+import { HealthMonitor } from './HealthMonitor.js';
+import { MultiQueueWorker } from './MultiQueueWorker.js';
+import { Observability } from './Observability.js';
+import { PluginManager } from './PluginManager.js';
+import { PriorityQueue } from './PriorityQueue.js';
+import { ResourceMonitor } from './ResourceMonitor.js';
+import { WorkerMetrics } from './WorkerMetrics.js';
+import { WorkerRegistry } from './WorkerRegistry.js';
+import { WorkerVersioning } from './WorkerVersioning.js';
+import { keyPrefix } from './config/workerConfig.js';
+import { DbWorkerStore, InMemoryWorkerStore, RedisWorkerStore, } from './storage/WorkerStore.js';
+const path = NodeSingletons.path;
+const isNodeRuntime = () => typeof process !== 'undefined' && Boolean(process.versions?.node);
+const resolveProjectRoot = () => {
+    const envRoot = Env.get('ZINTRUST_PROJECT_ROOT', '').trim();
+    return envRoot.length > 0 ? envRoot : process.cwd();
+};
+const canUseProjectFileImports = () => typeof NodeSingletons?.fs?.writeFileSync === 'function' &&
+    typeof NodeSingletons?.fs?.mkdirSync === 'function' &&
+    typeof NodeSingletons?.fs?.existsSync === 'function' &&
+    typeof NodeSingletons?.url?.pathToFileURL === 'function' &&
+    typeof NodeSingletons?.path?.join === 'function';
+const buildCandidatesForSpecifier = (specifier, root) => {
+    if (specifier === '@zintrust/core') {
+        return [
+            path.join(root, 'dist', 'src', 'index.js'),
+            path.join(root, 'dist', 'index.js'),
+            path.join(root, 'src', 'index.ts'),
+        ];
+    }
+    if (specifier === '@zintrust/workers') {
+        return [
+            path.join(root, 'dist', 'packages', 'workers', 'src', 'index.js'),
+            path.join(root, 'packages', 'workers', 'src', 'index.ts'),
+        ];
+    }
+    return [];
+};
+const getProjectFileCandidates = (paths) => {
+    if (!canUseProjectFileImports())
+        return null;
+    for (const candidate of paths) {
+        if (NodeSingletons.fs.existsSync(candidate))
+            return candidate;
+    }
+    return null;
+};
+const resolveLocalPackageFallback = (specifier) => {
+    if (!canUseProjectFileImports())
+        return null;
+    const root = resolveProjectRoot();
+    const candidates = buildCandidatesForSpecifier(specifier, root);
+    const resolved = getProjectFileCandidates(candidates);
+    if (!resolved)
+        return null;
+    return NodeSingletons.url.pathToFileURL(resolved).href;
+};
+const resolvePackageSpecifierUrl = (specifier) => {
+    if (!isNodeRuntime() || !canUseProjectFileImports())
+        return null;
+    if (typeof NodeSingletons?.module?.createRequire !== 'function') {
+        return resolveLocalPackageFallback(specifier);
+    }
+    try {
+        const require = NodeSingletons.module.createRequire(import.meta.url);
+        const resolved = require.resolve(specifier);
+        if (specifier === '@zintrust/workers' &&
+            resolved.includes(`${path.sep}node_modules${path.sep}@zintrust${path.sep}workers${path.sep}`)) {
+            const local = resolveLocalPackageFallback(specifier);
+            if (local)
+                return local;
+        }
+        return NodeSingletons.url.pathToFileURL(resolved).href;
+    }
+    catch {
+        return resolveLocalPackageFallback(specifier);
+    }
+};
+const escapeRegExp = (value) => value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw `\$&`);
+const rewriteProcessorImports = (code) => {
+    const replacements = [];
+    const coreUrl = resolvePackageSpecifierUrl('@zintrust/core');
+    if (coreUrl)
+        replacements.push({ from: '@zintrust/core', to: coreUrl });
+    const workersUrl = resolvePackageSpecifierUrl('@zintrust/workers');
+    if (workersUrl)
+        replacements.push({ from: '@zintrust/workers', to: workersUrl });
+    if (replacements.length === 0)
+        return code;
+    let updated = code;
+    for (const { from, to } of replacements) {
+        const pattern = new RegExp(String.raw `(['"])${escapeRegExp(from)}\1`, 'g');
+        updated = updated.replace(pattern, `$1${to}$1`);
+    }
+    return updated;
+};
+const ensureProcessorSpecDir = () => {
+    if (!isNodeRuntime() || !canUseProjectFileImports())
+        return null;
+    const dir = path.join(resolveProjectRoot(), '.zintrust', 'processor-specs');
+    try {
+        if (!NodeSingletons.fs.existsSync(dir)) {
+            NodeSingletons.fs.mkdirSync(dir, { recursive: true });
+        }
+        return dir;
+    }
+    catch (error) {
+        Logger.debug('Failed to prepare processor spec cache directory', error);
+        return null;
+    }
+};
+const shouldFallbackToFileImport = (error) => {
+    if (!isNodeRuntime())
+        return false;
+    const message = error instanceof Error ? error.message : String(error);
+    const code = error?.code ?? '';
+    if (code === 'ERR_INVALID_URL' || code === 'ERR_UNSUPPORTED_ESM_URL_SCHEME')
+        return true;
+    return (message.includes('Invalid relative URL') ||
+        message.includes('base scheme is not hierarchical') ||
+        message.includes('Failed to resolve module specifier'));
+};
+const importModuleFromCode = async (params) => {
+    const { code, normalized, cacheKey } = params;
+    const dataUrl = `data:text/javascript;base64,${toBase64(code)}`;
+    try {
+        return (await import(dataUrl));
+    }
+    catch (error) {
+        if (!shouldFallbackToFileImport(error))
+            throw error;
+        const dir = ensureProcessorSpecDir();
+        if (!dir)
+            throw error;
+        try {
+            const codeHash = await computeSha256(code);
+            const filePath = path.join(dir, `${codeHash || cacheKey}.mjs`);
+            NodeSingletons.fs.writeFileSync(filePath, code, 'utf8');
+            const fileUrl = NodeSingletons.url.pathToFileURL(filePath).href;
+            return (await import(fileUrl));
+        }
+        catch (fileError) {
+            Logger.debug(`Processor URL file fallback failed for ${normalized}`, fileError);
+            throw error;
+        }
+    }
+};
+const isCloudflareRuntime = () => Cloudflare.getWorkersEnv() !== null;
+const getDefaultStoreForRuntime = async () => {
+    if (!isCloudflareRuntime()) {
+        await ensureWorkerStoreConfigured();
+        return workerStore;
+    }
+    const bootstrapConfig = buildPersistenceBootstrapConfig();
+    const persistence = resolvePersistenceConfig(bootstrapConfig);
+    if (!persistence) {
+        return InMemoryWorkerStore.create();
+    }
+    return resolveWorkerStoreForPersistence(persistence);
+};
+const getStoreForWorker = async (config, persistenceOverride) => {
+    if (persistenceOverride) {
+        return resolveWorkerStoreForPersistence(persistenceOverride);
+    }
+    // If worker has specific configuration, use it
+    if (config) {
+        const persistence = resolvePersistenceConfig(config);
+        if (persistence) {
+            return resolveWorkerStoreForPersistence(persistence);
+        }
+    }
+    // Fallback to default/global store
+    return getDefaultStoreForRuntime();
+};
+const validateAndGetStore = async (name, config, persistenceOverride) => {
+    const store = await getStoreForWorker(config, persistenceOverride);
+    const record = await store.get(name);
+    if (!record) {
+        throw ErrorFactory.createNotFoundError(`Worker "${name}" not found in the specified driver. Ensure you are addressing the correct storage backend.`);
+    }
+    return store;
+};
+// Worker creation status enum for proper lifecycle management
+export const WorkerCreationStatus = {
+    CREATING: 'creating', // Initial state - worker is being created
+    CONNECTING: 'connecting', // Connecting to Redis/Queue
+    STARTING: 'starting', // Starting BullMQ worker
+    RUNNING: 'running', // Actually processing jobs
+    FAILED: 'failed', // Connection/startup failed
+    STOPPED: 'stopped', // Intentionally stopped
+};
+// Internal initialization state to prevent memory leaks and redundant calls
+let clusteringInitialized = false;
+let metricsInitialized = false;
+let autoScalingInitialized = false;
+let deadLetterQueueInitialized = false;
+let resourceMonitoringInitialized = false;
+let complianceInitialized = false;
+let observabilityInitialized = false;
+// Internal state
+const workers = new Map();
+let workerStore = InMemoryWorkerStore.create();
+let workerStoreConfigured = false;
+let workerStoreConfig = null;
+const processorRegistry = new Map();
+const processorPathRegistry = new Map();
+const processorResolvers = [];
+const processorSpecRegistry = new Map();
+const processorCache = new Map();
+let processorCacheSize = 0;
+const buildPersistenceBootstrapConfig = () => {
+    const driver = Env.get('WORKER_PERSISTENCE_DRIVER', 'memory');
+    const config = {
+        name: '__zintrust_persistence_bootstrap__',
+        queueName: '__zintrust_bootstrap__',
+        processor: async () => undefined,
+        infrastructure: {
+            persistence: {
+                driver,
+            },
+        },
+    };
+    // Add Redis config if using Redis persistence
+    if (driver === 'redis') {
+        config.infrastructure = {
+            ...config.infrastructure,
+            redis: queueConfig.drivers.redis,
+        };
+    }
+    return config;
+};
+const registerProcessor = (name, processor) => {
+    processorRegistry.set(name, processor);
+};
+const registerProcessors = (processors) => {
+    Object.entries(processors).forEach(([name, processor]) => {
+        if (typeof processor === 'function') {
+            processorRegistry.set(name, processor);
+        }
+    });
+};
+const registerProcessorPaths = (paths) => {
+    Object.entries(paths).forEach(([name, modulePath]) => {
+        if (typeof modulePath === 'string' && modulePath.trim().length > 0) {
+            processorPathRegistry.set(name, modulePath);
+        }
+    });
+};
+const registerProcessorResolver = (resolver) => {
+    processorResolvers.push(resolver);
+};
+const registerProcessorSpec = (spec, processor) => {
+    if (!spec || typeof processor !== 'function')
+        return;
+    processorSpecRegistry.set(normalizeProcessorSpec(spec), processor);
+};
+const decodeProcessorPathEntities = (value) => value
+    .replaceAll(///gi, '/')
+    .replaceAll('/', '/')
+    .replaceAll(///gi, '/');
+const isUrlSpec = (spec) => {
+    if (spec.startsWith('url:'))
+        return true;
+    return spec.includes('://');
+};
+const normalizeProcessorSpec = (spec) => spec.startsWith('url:') ? spec.slice(4) : spec;
+const parseCacheControl = (value) => {
+    if (!value)
+        return {};
+    const parts = value.split(',').map((part) => part.trim().toLowerCase());
+    const maxAge = parts.find((part) => part.startsWith('max-age='));
+    if (!maxAge)
+        return {};
+    const raw = maxAge.split('=')[1];
+    const parsed = Number.parseInt(raw ?? '', 10);
+    return Number.isFinite(parsed) ? { maxAge: parsed } : {};
+};
+const getProcessorSpecConfig = () => workersConfig.processorSpec;
+const computeSha256 = async (value) => {
+    if (typeof globalThis !== 'undefined' && globalThis.crypto?.subtle) {
+        const data = new TextEncoder().encode(value);
+        const digest = await globalThis.crypto.subtle.digest('SHA-256', data);
+        return Array.from(new Uint8Array(digest))
+            .map((b) => b.toString(16).padStart(2, '0'))
+            .join('');
+    }
+    if (typeof NodeSingletons.createHash === 'function') {
+        return NodeSingletons.createHash('sha256').update(value).digest('hex');
+    }
+    return String(generateUuid()).slice(2);
+};
+const toBase64 = (value) => {
+    if (typeof Buffer !== 'undefined') {
+        return Buffer.from(value, 'utf-8').toString('base64');
+    }
+    if (typeof globalThis !== 'undefined' && typeof globalThis.btoa === 'function') {
+        const bytes = new TextEncoder().encode(value);
+        let binary = '';
+        bytes.forEach((byte) => {
+            binary += String.fromCodePoint(byte);
+        });
+        return globalThis.btoa(binary);
+    }
+    return value;
+};
+const getCachedProcessor = (key) => {
+    const entry = processorCache.get(key);
+    if (!entry)
+        return null;
+    const now = Date.now();
+    if (entry.expiresAt <= now) {
+        processorCache.delete(key);
+        processorCacheSize -= entry.size;
+        return null;
+    }
+    entry.lastAccess = now;
+    return entry;
+};
+const evictCacheIfNeeded = (maxSize) => {
+    if (processorCacheSize <= maxSize)
+        return;
+    const entries = Array.from(processorCache.entries());
+    entries.sort((a, b) => a[1].lastAccess - b[1].lastAccess);
+    for (const [key, entry] of entries) {
+        if (processorCacheSize <= maxSize)
+            break;
+        processorCache.delete(key);
+        processorCacheSize -= entry.size;
+    }
+};
+const setCachedProcessor = (key, entry, maxSize) => {
+    const existing = processorCache.get(key);
+    if (existing) {
+        processorCacheSize -= existing.size;
+    }
+    processorCache.set(key, entry);
+    processorCacheSize += entry.size;
+    evictCacheIfNeeded(maxSize);
+};
+const isAllowedRemoteHost = (host) => {
+    const allowlist = getProcessorSpecConfig().remoteAllowlist.map((value) => value.toLowerCase());
+    return allowlist.includes(host.toLowerCase());
+};
+const waitForWorkerConnection = async (worker, name, _queueName, timeoutMs) => {
+    const startTime = Date.now();
+    const checkInterval = 100; // 100ms between checks
+    let timeoutId = null;
+    return new Promise((resolve, reject) => {
+        const checkConnection = async () => {
+            try {
+                // Check if worker is actually running
+                const isRunning = await worker.isRunning();
+                if (!isRunning) {
+                    throw ErrorFactory.createWorkerError('Worker not running');
+                }
+                // Check Redis connection
+                const client = await worker.client;
+                const pingResult = await client.ping();
+                if (pingResult !== 'PONG') {
+                    throw ErrorFactory.createWorkerError('Redis ping failed');
+                }
+                // Removed heavy Queue instantiation loop - relying on Redis ping for connectivity check
+                // The queue instance creation was causing memory pressure and potential connection leaks in this retry loop
+                Logger.debug(`Worker health verification passed for ${name}`, {
+                    isRunning,
+                    pingResult,
+                });
+                if (timeoutId)
+                    clearTimeout(timeoutId);
+                resolve();
+                return;
+            }
+            catch (error) {
+                Logger.debug(`Worker health verification failed for ${name}, retrying...`, error);
+                // Check timeout
+                if (Date.now() - startTime >= timeoutMs) {
+                    if (timeoutId)
+                        clearTimeout(timeoutId);
+                    reject(ErrorFactory.createWorkerError('Worker failed health verification within timeout period'));
+                    return;
+                }
+                // Schedule next check
+                timeoutId = globalThis.setTimeout(checkConnection, checkInterval);
+            }
+        };
+        // Start checking
+        checkConnection();
+    });
+};
+const startHealthMonitoring = (name, worker, queueName) => {
+    HealthMonitor.register(name, worker, queueName);
+};
+const sanitizeProcessorPath = (value) => {
+    const decoded = decodeProcessorPathEntities(value);
+    const base = decoded.split(/[?#&]/)[0]?.trim() ?? '';
+    if (!base)
+        return '';
+    const isAbsolutePath = base.startsWith('/') || /^[A-Za-z]:[\\/]/.test(base);
+    const relativePath = base.startsWith('.') ? base : `./${base}`;
+    return isAbsolutePath ? base : path.resolve(process.cwd(), relativePath);
+};
+const stripProcessorExtension = (value) => value.replace(/\.(ts|js)$/i, '');
+const normalizeModulePath = (value) => value.replaceAll('\\', '/');
+const filterExistingFileCandidates = (candidates) => {
+    if (!NodeSingletons?.fs?.existsSync)
+        return candidates;
+    return candidates.filter((candidate) => {
+        try {
+            return NodeSingletons.fs.existsSync(candidate);
+        }
+        catch {
+            return false;
+        }
+    });
+};
+const buildProcessorModuleCandidates = (modulePath, resolvedPath) => {
+    const candidates = [];
+    const normalized = normalizeModulePath(modulePath.trim());
+    const normalizedResolved = normalizeModulePath(resolvedPath);
+    if (normalized.startsWith('/app/')) {
+        candidates.push(`@app/${stripProcessorExtension(normalized.slice(5))}`);
+    }
+    else if (normalized.startsWith('app/')) {
+        candidates.push(`@app/${stripProcessorExtension(normalized.slice(4))}`);
+    }
+    const appIndex = normalizedResolved.lastIndexOf('/app/');
+    if (appIndex !== -1) {
+        const relative = normalizedResolved.slice(appIndex + 5);
+        if (relative) {
+            candidates.push(`@app/${stripProcessorExtension(relative)}`);
+        }
+    }
+    return Array.from(new Set(candidates));
+};
+const buildProcessorFilePathCandidates = (_modulePath, resolvedPath) => {
+    const candidates = [];
+    const normalizedResolved = normalizeModulePath(resolvedPath);
+    const projectRoot = normalizeModulePath(resolveProjectRoot());
+    const strippedResolved = stripProcessorExtension(resolvedPath);
+    candidates.push(`${strippedResolved}.js`, `${strippedResolved}.mjs`);
+    const appIndex = normalizedResolved.lastIndexOf('/app/');
+    if (appIndex !== -1) {
+        const relative = normalizedResolved.slice(appIndex + 5);
+        if (relative) {
+            const strippedRelative = stripProcessorExtension(relative);
+            candidates.push(path.join(projectRoot, 'dist', 'app', `${strippedRelative}.js`), path.join(projectRoot, 'app', relative), path.join(projectRoot, 'app', `${strippedRelative}.js`), path.join('/app', 'dist', 'app', `${strippedRelative}.js`));
+        }
+    }
+    return filterExistingFileCandidates(Array.from(new Set(candidates)));
+};
+const pickProcessorFromModule = (mod, source) => {
+    const candidate = mod?.['default'] ?? mod?.['processor'] ?? mod?.['handler'] ?? mod?.['handle'];
+    if (typeof candidate !== 'function') {
+        const keys = mod ? Object.keys(mod) : [];
+        Logger.warn(`Module imported from ${source} but no valid processor function found (exported: ${keys.join(', ')})`);
+        return undefined;
+    }
+    return candidate;
+};
+const extractZinTrustProcessor = (mod, source) => {
+    const candidate = mod?.['ZinTrustProcessor'];
+    if (typeof candidate !== 'function') {
+        const keys = mod ? Object.keys(mod) : [];
+        Logger.warn(`Module imported from ${source} but missing ZinTrustProcessor export (exported: ${keys.join(', ')})`);
+        return undefined;
+    }
+    return candidate;
+};
+const readResponseBody = async (response, maxSize) => {
+    const contentLength = response.headers.get('content-length');
+    if (contentLength) {
+        const size = Number.parseInt(contentLength, 10);
+        if (Number.isFinite(size) && size > maxSize) {
+            throw ErrorFactory.createConfigError('PROCESSOR_FETCH_SIZE_EXCEEDED');
+        }
+    }
+    const buffer = await response.arrayBuffer();
+    if (buffer.byteLength > maxSize) {
+        throw ErrorFactory.createConfigError('PROCESSOR_FETCH_SIZE_EXCEEDED');
+    }
+    return new TextDecoder().decode(buffer);
+};
+const computeCacheTtlSeconds = (config, cacheControl) => Math.min(config.cacheMaxTtlSeconds, cacheControl.maxAge ?? config.cacheDefaultTtlSeconds);
+const refreshCachedProcessor = (existing, config, cacheControl) => {
+    const ttl = computeCacheTtlSeconds(config, cacheControl);
+    const now = Date.now();
+    existing.expiresAt = now + ttl * 1000;
+    existing.lastAccess = now;
+    return existing.processor;
+};
+const cacheProcessorFromResponse = async (params) => {
+    const { response, normalized, config, cacheKey } = params;
+    const rawCode = await readResponseBody(response, config.fetchMaxSizeBytes);
+    const code = rewriteProcessorImports(rawCode);
+    const mod = await importModuleFromCode({ code, normalized, cacheKey });
+    const processor = extractZinTrustProcessor(mod, normalized);
+    if (!processor) {
+        throw ErrorFactory.createConfigError('INVALID_PROCESSOR_URL_EXPORT');
+    }
+    const cacheControl = parseCacheControl(response.headers.get('cache-control'));
+    const ttl = computeCacheTtlSeconds(config, cacheControl);
+    const size = new TextEncoder().encode(code).byteLength;
+    const now = Date.now();
+    setCachedProcessor(cacheKey, {
+        code,
+        processor,
+        etag: response.headers.get('etag') ?? undefined,
+        cachedAt: now,
+        expiresAt: now + ttl * 1000,
+        size,
+        lastAccess: now,
+    }, config.cacheMaxSizeBytes);
+    return processor;
+};
+const delay = (ms) => new Promise((resolve) => {
+    globalThis.setTimeout(resolve, ms);
+});
+const fetchProcessorAttempt = async (params) => {
+    const { normalized, config, cacheKey, existing, attempt, maxAttempts } = params;
+    const controller = new AbortController();
+    const timeoutId = globalThis.setTimeout(() => controller.abort(), config.fetchTimeoutMs);
+    try {
+        const headers = {};
+        if (existing?.etag)
+            headers['If-None-Match'] = existing.etag;
+        const response = await fetch(normalized, {
+            method: 'GET',
+            headers,
+            signal: controller.signal,
+        });
+        if (response.status === 304 && existing) {
+            const cacheControl = parseCacheControl(response.headers.get('cache-control'));
+            return refreshCachedProcessor(existing, config, cacheControl);
+        }
+        if (!response.ok) {
+            throw ErrorFactory.createConfigError(`PROCESSOR_FETCH_FAILED:${response.status}`);
+        }
+        return await cacheProcessorFromResponse({ response, normalized, config, cacheKey });
+    }
+    catch (error) {
+        if (controller.signal.aborted) {
+            Logger.error('Processor URL fetch timeout', error);
+        }
+        else {
+            Logger.error('Processor URL fetch failed', error);
+        }
+        if (attempt >= maxAttempts) {
+            return undefined;
+        }
+        await delay(config.retryBackoffMs * attempt);
+        return fetchProcessorAttempt({
+            normalized,
+            config,
+            cacheKey,
+            existing,
+            attempt: attempt + 1,
+            maxAttempts,
+        });
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+};
+const resolveProcessorFromUrl = async (spec) => {
+    const normalized = normalizeProcessorSpec(spec);
+    let parsed;
+    try {
+        parsed = new URL(normalized);
+    }
+    catch (error) {
+        Logger.error('Invalid processor URL spec', error);
+        return undefined;
+    }
+    if (parsed.protocol === 'file:') {
+        const filePath = decodeURIComponent(parsed.pathname);
+        return resolveProcessorFromPath(filePath);
+    }
+    if (parsed.protocol !== 'https:' && parsed.protocol !== 'file:') {
+        Logger.warn(`Invalid processor URL protocol: ${parsed.protocol}. Only https:// and file:// are supported.`);
+        return undefined;
+    }
+    if (!isAllowedRemoteHost(parsed.host) && parsed.protocol !== 'file:') {
+        Logger.warn(`Invalid processor URL host: ${parsed.host}. Host is not in the allowlist.`);
+        return undefined;
+    }
+    const config = getProcessorSpecConfig();
+    const cacheKey = await computeSha256(normalized);
+    const cached = getCachedProcessor(cacheKey);
+    if (cached)
+        return cached.processor;
+    return fetchProcessorAttempt({
+        normalized,
+        config,
+        cacheKey,
+        existing: processorCache.get(cacheKey),
+        attempt: 1,
+        maxAttempts: Math.max(1, config.retryAttempts),
+    });
+};
+const resolveProcessorSpec = async (spec) => {
+    if (!spec)
+        return undefined;
+    const normalized = normalizeProcessorSpec(spec);
+    const prebuilt = processorSpecRegistry.get(normalized) ?? processorSpecRegistry.get(spec);
+    if (prebuilt)
+        return prebuilt;
+    if (isUrlSpec(spec))
+        return resolveProcessorFromUrl(spec);
+    return resolveProcessorFromPath(spec);
+};
+const resolveProcessorFromPath = async (modulePath) => {
+    const trimmed = modulePath.trim();
+    if (!trimmed)
+        return undefined;
+    const resolved = sanitizeProcessorPath(trimmed);
+    if (!resolved)
+        return undefined;
+    const importProcessorFromCandidates = async (candidates) => {
+        if (candidates.length === 0)
+            return undefined;
+        const [candidatePath, ...rest] = candidates;
+        try {
+            const importPath = candidatePath.startsWith('/') && !candidatePath.startsWith('//')
+                ? NodeSingletons.url.pathToFileURL(candidatePath).href
+                : candidatePath;
+            const mod = await import(importPath);
+            const candidate = pickProcessorFromModule(mod, importPath);
+            if (candidate)
+                return candidate;
+        }
+        catch (candidateError) {
+            Logger.debug(`Processor module candidate import failed: ${candidatePath}`, candidateError);
+        }
+        return importProcessorFromCandidates(rest);
+    };
+    try {
+        const mod = await import(resolved);
+        const candidate = pickProcessorFromModule(mod, resolved);
+        if (candidate)
+            return candidate;
+    }
+    catch (err) {
+        const fileCandidates = buildProcessorFilePathCandidates(trimmed, resolved);
+        const resolvedFileCandidate = await importProcessorFromCandidates(fileCandidates);
+        if (resolvedFileCandidate)
+            return resolvedFileCandidate;
+        const moduleCandidates = buildProcessorModuleCandidates(trimmed, resolved);
+        const resolvedModuleCandidate = await importProcessorFromCandidates(moduleCandidates);
+        if (resolvedModuleCandidate)
+            return resolvedModuleCandidate;
+        Logger.error(`Failed to import processor from path: ${resolved}`, err);
+    }
+    return undefined;
+};
+const resolveProcessor = async (name) => {
+    const direct = processorRegistry.get(name);
+    if (direct)
+        return direct;
+    const pathHint = processorPathRegistry.get(name);
+    if (pathHint) {
+        try {
+            const resolved = await resolveProcessorSpec(pathHint);
+            if (resolved)
+                return resolved;
+        }
+        catch (error) {
+            Logger.error(`Failed to resolve processor module for "${name}"`, error);
+        }
+    }
+    const resolverResults = await Promise.all(processorResolvers.map(async (resolver) => {
+        try {
+            return await resolver(name);
+        }
+        catch (error) {
+            Logger.error(`Processor resolver failed for "${name}"`, error);
+            return undefined;
+        }
+    }));
+    const resolvedFromResolvers = resolverResults.find((result) => result !== undefined);
+    if (resolvedFromResolvers)
+        return resolvedFromResolvers;
+    return undefined;
+};
+const resolveProcessorPath = async (modulePath) => resolveProcessorFromPath(modulePath);
+const recordMetricSafely = (workerName, metricType, value, metadata) => {
+    WorkerMetrics.record(workerName, metricType, value, metadata).catch((error) => {
+        Logger.error(`Failed to record worker metric: ${workerName}/${metricType}`, error);
+    });
+};
+const ensureCircuitAllowsExecution = (workerName, version, jobId, features) => {
+    if (!(features?.circuitBreaker ?? false))
+        return;
+    const canExecute = CircuitBreaker.canExecute(workerName, version);
+    if (canExecute)
+        return;
+    const state = CircuitBreaker.getState(workerName, version);
+    Logger.warn('Circuit breaker is open, rejecting job', {
+        workerName,
+        version,
+        jobId,
+        circuitState: state?.state,
+    });
+    CircuitBreaker.recordRejection(workerName, version);
+    throw ErrorFactory.createGeneralError(`Circuit breaker is open for ${workerName}@${version}`);
+};
+const runBeforeProcessHooks = async (workerName, job, features) => {
+    if (!(features?.plugins ?? false)) {
+        return { skip: false, jobData: job.data };
+    }
+    const hookResult = await PluginManager.executeHook('beforeProcess', {
+        workerName,
+        jobId: job.id ?? '',
+        jobData: job.data,
+        timestamp: new Date(),
+    });
+    if (hookResult.stopped) {
+        const errorMessage = hookResult.errors[0]?.error?.message ?? 'Stopped by plugin';
+        Logger.info('Job processing stopped by plugin', {
+            workerName,
+            jobId: job.id,
+            reason: errorMessage,
+        });
+        return { skip: true, reason: errorMessage };
+    }
+    if (hookResult.modified) {
+        return { skip: false, jobData: hookResult.context.jobData };
+    }
+    return { skip: false, jobData: job.data };
+};
+const startProcessingSpan = (workerName, version, job, queueName, features) => {
+    if (!(features?.observability ?? false))
+        return null;
+    return Observability.startSpan(`worker.${workerName}.process`, {
+        attributes: {
+            worker_name: workerName,
+            worker_version: version,
+            job_id: job.id ?? '',
+            queue_name: queueName,
+        },
+    });
+};
+const usePluginManager = async (workerName, job, result) => {
+    await PluginManager.executeHook('afterProcess', {
+        workerName,
+        jobId: job.id ?? '',
+        jobData: job.data,
+        metadata: { result },
+        timestamp: new Date(),
+    });
+    await PluginManager.executeHook('onComplete', {
+        workerName,
+        jobId: job.id ?? '',
+        jobData: job.data,
+        metadata: { result },
+        timestamp: new Date(),
+    });
+};
+const handleSuccess = async (params) => {
+    const { workerName, jobVersion, job, result, duration, spanId, features } = params;
+    if (features?.metrics ?? false) {
+        recordMetricSafely(workerName, 'processed', 1);
+        recordMetricSafely(workerName, 'duration', duration);
+    }
+    if (features?.circuitBreaker ?? false) {
+        CircuitBreaker.recordSuccess(workerName, jobVersion);
+    }
+    if (features?.observability ?? false) {
+        Observability.recordJobMetrics(workerName, job.name, {
+            processed: 1,
+            failed: 0,
+            durationMs: duration,
+        });
+        if (spanId !== null) {
+            Observability.endSpan(spanId, { success: true });
+        }
+    }
+    if (features?.plugins ?? false) {
+        await usePluginManager(workerName, { id: job.id ?? '', data: job.data }, result);
+    }
+};
+const recordFailureMetrics = (workerName, _jobVersion, duration, features) => {
+    if (features?.metrics === true) {
+        recordMetricSafely(workerName, 'errors', 1);
+        recordMetricSafely(workerName, 'duration', duration);
+    }
+};
+const recordFailureObservability = (workerName, jobName, duration, spanId, features) => {
+    if (features?.observability === true) {
+        Observability.recordJobMetrics(workerName, jobName, {
+            processed: 0,
+            failed: 1,
+            durationMs: duration,
+        });
+        if (spanId !== null) {
+            Observability.recordSpanError(spanId, ErrorFactory.createGeneralError('Job processing failed'));
+            Observability.endSpan(spanId, { success: false });
+        }
+    }
+};
+const addFailedJobToDeadLetterQueue = async (workerName, job, error, duration, jobVersion, queueName, features) => {
+    if (features?.deadLetterQueue === true) {
+        await DeadLetterQueue.addFailedJob({
+            id: job.id ?? '',
+            queueName,
+            workerName,
+            jobName: job.name,
+            data: job.data,
+            error: {
+                name: error.name,
+                message: error.message,
+                stack: error.stack,
+            },
+            attemptsMade: job.attemptsMade ?? 0,
+            maxAttempts: job.opts.attempts ?? 0,
+            failedAt: new Date(),
+            firstAttemptAt: new Date(job.timestamp ?? Date.now()),
+            lastAttemptAt: new Date(),
+            processingTime: duration,
+            metadata: {
+                version: jobVersion,
+            },
+            complianceFlags: {
+                containsPII: false,
+                containsPHI: false,
+                dataClassification: 'public',
+            },
+        });
+    }
+};
+const executeFailurePlugins = async (workerName, job, error, features) => {
+    if (features?.plugins === true) {
+        await PluginManager.executeHook('onError', {
+            workerName,
+            jobId: job.id ?? '',
+            jobData: job.data,
+            error,
+            timestamp: new Date(),
+        });
+    }
+};
+const recordCircuitBreakerFailure = (workerName, jobVersion, error, features) => {
+    if (features?.circuitBreaker === true) {
+        CircuitBreaker.recordFailure(workerName, jobVersion, error);
+    }
+};
+const logAndRecordFailure = (workerName, jobVersion, job, error, features) => {
+    Logger.error(`Worker job failed: ${workerName}`, { error, jobId: job.id, version: jobVersion }, 'workers');
+    recordCircuitBreakerFailure(workerName, jobVersion, error, features);
+};
+const recordFailureObservabilityAndMetrics = (params) => {
+    const { workerName, jobVersion, jobName, duration, spanId, features } = params;
+    recordFailureMetrics(workerName, jobVersion, duration, features);
+    recordFailureObservability(workerName, jobName, duration, spanId, features);
+};
+const executeAllFailureHandlers = async (params) => {
+    const { workerName, jobVersion, job, error, duration, spanId, features, queueName } = params;
+    recordFailureObservabilityAndMetrics({
+        workerName,
+        jobVersion,
+        jobName: job.name,
+        duration,
+        spanId,
+        features,
+    });
+    if (features?.deadLetterQueue === true) {
+        await addFailedJobToDeadLetterQueue(workerName, job, error, duration, jobVersion, queueName, features);
+    }
+};
+const handleFailure = async (params) => {
+    const { workerName, jobVersion, job, error, features } = params;
+    logAndRecordFailure(workerName, jobVersion, job, error, features);
+    await executeAllFailureHandlers(params);
+    await executeFailurePlugins(workerName, job, error, features);
+};
+const toBackoffDelayMs = (backoff) => {
+    if (typeof backoff === 'number' && Number.isFinite(backoff)) {
+        return Math.max(0, Math.floor(backoff));
+    }
+    if (backoff !== null && backoff !== undefined && typeof backoff === 'object') {
+        const raw = backoff.delay;
+        if (typeof raw === 'number' && Number.isFinite(raw)) {
+            return Math.max(0, Math.floor(raw));
+        }
+    }
+    return 0;
+};
+const trackJobStarted = async (input) => {
+    if (!input.job.id)
+        return;
+    await JobStateTracker.started({
+        queueName: input.queueName,
+        jobId: input.job.id,
+        attempts: input.attempts,
+        timeoutMs: Math.max(1000, Env.getInt('QUEUE_JOB_TIMEOUT', 60) * 1000),
+        workerName: input.workerName,
+        workerVersion: input.workerVersion,
+    });
+};
+const trackJobCompleted = async (input) => {
+    if (!input.job.id)
+        return;
+    await JobStateTracker.completed({
+        queueName: input.queueName,
+        jobId: input.job.id,
+        processingTimeMs: input.duration,
+        result: input.result,
+    });
+};
+const trackJobFailed = async (input) => {
+    if (!input.job.id)
+        return;
+    const isFinal = input.maxAttempts === undefined ? true : input.attempts >= input.maxAttempts;
+    const backoffDelayMs = toBackoffDelayMs(input.job.opts?.backoff);
+    await JobStateTracker.failed({
+        queueName: input.queueName,
+        jobId: input.job.id,
+        attempts: input.attempts,
+        isFinal,
+        retryAt: !isFinal && backoffDelayMs > 0
+            ? new Date(Date.now() + backoffDelayMs).toISOString()
+            : undefined,
+        error: input.error,
+    });
+};
+/**
+ * Helper: Create enhanced processor with all features
+ */
+const createEnhancedProcessor = (config) => {
+    return async (job) => {
+        const { name, version, processor, features } = config;
+        const jobVersion = version ?? '1.0.0';
+        ensureCircuitAllowsExecution(name, jobVersion, job.id, features);
+        const beforeOutcome = await runBeforeProcessHooks(name, job, features);
+        if (beforeOutcome.skip) {
+            return { skipped: true, reason: beforeOutcome.reason };
+        }
+        if (beforeOutcome.jobData !== undefined) {
+            job.data = beforeOutcome.jobData;
+        }
+        const startTime = Date.now();
+        let result;
+        let spanId = null;
+        const maxAttempts = typeof job.opts?.attempts === 'number' && Number.isFinite(job.opts.attempts)
+            ? Math.max(1, Math.floor(job.opts.attempts))
+            : undefined;
+        const attempts = Math.max(1, Math.floor((job.attemptsMade ?? 0) + 1));
+        try {
+            spanId = startProcessingSpan(name, jobVersion, job, config.queueName, features);
+            await trackJobStarted({
+                queueName: config.queueName,
+                job,
+                attempts,
+                workerName: name,
+                workerVersion: jobVersion,
+            });
+            // Process the job
+            result = await processor(job);
+            const duration = Date.now() - startTime;
+            await handleSuccess({
+                workerName: name,
+                jobVersion,
+                job,
+                result,
+                duration,
+                spanId,
+                features,
+            });
+            await trackJobCompleted({ queueName: config.queueName, job, duration, result });
+            return result;
+        }
+        catch (err) {
+            const error = err;
+            const duration = Date.now() - startTime;
+            await trackJobFailed({
+                queueName: config.queueName,
+                job,
+                attempts,
+                maxAttempts,
+                error,
+            });
+            await handleFailure({
+                workerName: name,
+                jobVersion,
+                job,
+                error,
+                duration,
+                spanId,
+                features,
+                queueName: config.queueName,
+            });
+            throw error;
+        }
+    };
+};
+const requireInfrastructure = (value, message) => {
+    if (value === null || value === undefined) {
+        throw ErrorFactory.createConfigError(message);
+    }
+    return value;
+};
+const resolveEnvString = (envKey, fallback) => {
+    if (!envKey)
+        return fallback;
+    return Env.get(envKey, fallback);
+};
+const resolveEnvInt = (envKey, fallback) => {
+    if (!envKey)
+        return fallback;
+    return Env.getInt(envKey, fallback);
+};
+const isRedisEnvConfig = (config) => config.env === true;
+const requireRedisHost = (host, context) => {
+    if (!host) {
+        throw ErrorFactory.createConfigError(`${context}.host is required`);
+    }
+    return host;
+};
+const resolveRedisFallbacks = () => {
+    const queueRedis = queueConfig.drivers.redis;
+    return {
+        host: queueRedis?.driver === 'redis' ? queueRedis.host : Env.get('REDIS_HOST', '127.0.0.1'),
+        port: queueRedis?.driver === 'redis'
+            ? queueRedis.port
+            : Env.getInt('REDIS_PORT', ZintrustLang.REDIS_DEFAULT_PORT),
+        db: queueRedis?.driver === 'redis'
+            ? queueRedis.database
+            : Env.getInt('REDIS_QUEUE_DB', ZintrustLang.REDIS_DEFAULT_DB),
+        password: queueRedis?.driver === 'redis' ? (queueRedis.password ?? '') : Env.get('REDIS_PASSWORD', ''),
+    };
+};
+const resolveRedisConfigFromEnv = (config, context) => {
+    const fallback = resolveRedisFallbacks();
+    const host = requireRedisHost(resolveEnvString(config.host ?? 'REDIS_HOST', fallback.host), context);
+    const port = resolveEnvInt(String(config.port ?? 'REDIS_PORT'), fallback.port);
+    const db = resolveEnvInt(config.db ?? 'REDIS_QUEUE_DB', fallback.db);
+    const password = resolveEnvString(config.password ?? 'REDIS_PASSWORD', fallback.password);
+    return {
+        host,
+        port,
+        db,
+        password: password || undefined,
+    };
+};
+const resolveRedisConfigFromDirect = (config, context) => {
+    const fallbackDb = Env.getInt('REDIS_QUEUE_DB', ZintrustLang.REDIS_DEFAULT_DB);
+    let normalizedDb = fallbackDb;
+    if (typeof config.db === 'number') {
+        normalizedDb = config.db;
+    }
+    else if (typeof config.database === 'number') {
+        normalizedDb = config.database;
+    }
+    return {
+        host: requireRedisHost(config.host, context),
+        port: config.port,
+        db: normalizedDb,
+        password: config.password ?? Env.get('REDIS_PASSWORD', undefined),
+    };
+};
+const resolveRedisConfig = (config, context) => isRedisEnvConfig(config)
+    ? resolveRedisConfigFromEnv(config, context)
+    : resolveRedisConfigFromDirect(config, context);
+const resolveRedisConfigWithFallback = (primary, fallback, errorMessage, context) => {
+    const selected = primary ?? fallback;
+    if (!selected) {
+        throw ErrorFactory.createConfigError(errorMessage);
+    }
+    return resolveRedisConfig(selected, context);
+};
+const logRedisPersistenceConfig = (redisConfig, key_prefix, source) => {
+    Logger.debug('Worker persistence redis config', {
+        source,
+        host: redisConfig.host,
+        port: redisConfig.port,
+        db: redisConfig.db,
+        key_prefix,
+    });
+};
+const normalizeEnvValue = (value) => {
+    if (!value)
+        return undefined;
+    const trimmed = value.trim();
+    return trimmed.length > 0 ? trimmed : undefined;
+};
+const resolveDefaultPersistenceTable = () => normalizeEnvValue(Env.get('WORKER_PERSISTENCE_TABLE', 'zintrust_workers')) ?? 'zintrust_workers';
+const resolveDefaultPersistenceConnection = () => normalizeEnvValue(Env.get('WORKER_PERSISTENCE_DB_CONNECTION', 'default')) ?? 'default';
+const resolveAutoStart = (config) => {
+    // If explicitly set in config (not null/undefined), use that
+    if (config.autoStart !== undefined && config.autoStart !== null) {
+        return config.autoStart;
+    }
+    // Otherwise, use environment variable
+    return Env.getBool('WORKER_AUTO_START', false);
+};
+const normalizeExplicitPersistence = (persistence) => {
+    if (persistence.driver === 'memory')
+        return { driver: 'memory' };
+    if (persistence.driver === 'redis') {
+        return {
+            driver: 'redis',
+            redis: persistence.redis,
+            keyPrefix: keyPrefix(),
+        };
+    }
+    const clientIsConnection = typeof persistence.client === 'string';
+    const clientConnection = clientIsConnection ? persistence.client : undefined;
+    const resolvedConnection = persistence.connection ??
+        clientConnection ??
+        normalizeEnvValue(Env.get('WORKER_PERSISTENCE_DB_CONNECTION', 'default')) ??
+        resolveDefaultPersistenceConnection();
+    return {
+        driver: 'database',
+        client: clientIsConnection ? undefined : persistence.client,
+        connection: resolvedConnection,
+        table: persistence.table ??
+            normalizeEnvValue(Env.get('WORKER_PERSISTENCE_TABLE', 'zintrust_workers')) ??
+            resolveDefaultPersistenceTable(),
+    };
+};
+const resolvePersistenceConfig = (config) => {
+    const explicit = config.infrastructure?.persistence;
+    if (explicit)
+        return normalizeExplicitPersistence(explicit);
+    const driver = normalizeEnvValue(Env.get('WORKER_PERSISTENCE_DRIVER', ''))?.toLowerCase();
+    if (!driver)
+        return undefined;
+    if (driver === 'memory')
+        return { driver: 'memory' };
+    if (driver === 'redis') {
+        const persistenceDbOverride = normalizeEnvValue(Env.get('WORKER_PERSISTENCE_REDIS_DB', ''));
+        return {
+            driver: 'redis',
+            // Optional override; otherwise defaults to REDIS_QUEUE_DB.
+            redis: {
+                env: true,
+                db: persistenceDbOverride ? 'WORKER_PERSISTENCE_REDIS_DB' : 'REDIS_QUEUE_DB',
+            },
+            keyPrefix: keyPrefix(),
+        };
+    }
+    if (driver === 'db' || driver === 'database') {
+        return {
+            driver: 'database',
+            connection: resolveDefaultPersistenceConnection(),
+            table: resolveDefaultPersistenceTable(),
+        };
+    }
+    throw ErrorFactory.createConfigError('WORKER_PERSISTENCE_DRIVER must be one of memory, redis, or database');
+};
+const resolveDbClientFromEnv = async (connectionName = 'default') => {
+    const connect = async () => await useEnsureDbConnected(undefined, connectionName);
+    try {
+        return await connect();
+    }
+    catch (error) {
+        Logger.error('Worker persistence failed to resolve database connection', error);
+    }
+    try {
+        registerDatabasesFromRuntimeConfig(databaseConfig);
+        return await connect();
+    }
+    catch (error) {
+        Logger.error('Worker persistence failed after registering runtime databases', error);
+        throw ErrorFactory.createConfigError(`Worker persistence requires a database client. Register connection '${connectionName}' or pass infrastructure.persistence.client.`);
+    }
+};
+const resolveWorkerStore = async (config) => {
+    const persistence = resolvePersistenceConfig(config);
+    if (!persistence)
+        return workerStore;
+    let next;
+    if (persistence.driver === 'memory') {
+        next = InMemoryWorkerStore.create();
+    }
+    else if (persistence.driver === 'redis') {
+        const redisConfig = resolveRedisConfigWithFallback(persistence.redis, config.infrastructure?.redis, 'Worker persistence requires redis config (persistence.redis or infrastructure.redis)', 'infrastructure.persistence.redis');
+        const key_prefix = persistence.keyPrefix ?? keyPrefix();
+        logRedisPersistenceConfig(redisConfig, key_prefix, 'resolveWorkerStore');
+        const client = createRedisConnection(redisConfig);
+        next = RedisWorkerStore.create(client, key_prefix);
+    }
+    else if (persistence.driver === 'database') {
+        const explicitConnection = typeof persistence.client === 'string' ? persistence.client : persistence.connection;
+        const client = typeof persistence.client === 'string'
+            ? await resolveDbClientFromEnv(explicitConnection)
+            : (persistence.client ?? (await resolveDbClientFromEnv(explicitConnection)));
+        next = DbWorkerStore.create(client, persistence.table);
+    }
+    else {
+        next = InMemoryWorkerStore.create();
+    }
+    await next.init();
+    return next;
+};
+// Store instance cache to reuse connections
+const storeInstanceCache = new Map();
+/**
+ * Generate cache key for persistence configuration
+ */
+const generateCacheKey = (persistence) => {
+    return JSON.stringify({
+        driver: persistence.driver,
+        redis: 'redis' in persistence ? persistence.redis : undefined,
+        keyPrefix: 'keyPrefix' in persistence ? persistence.keyPrefix : undefined,
+        connection: 'connection' in persistence ? persistence.connection : undefined,
+        table: 'table' in persistence ? persistence.table : undefined,
+    });
+};
+/**
+ * Create new store instance based on persistence configuration
+ */
+const createWorkerStore = async (persistence) => {
+    if (persistence.driver === 'memory') {
+        if (workerStoreConfigured && workerStoreConfig?.driver === 'memory') {
+            return workerStore;
+        }
+        return InMemoryWorkerStore.create();
+    }
+    if (persistence.driver === 'redis') {
+        const redisConfig = resolveRedisConfigWithFallback(persistence.redis ?? { env: true }, undefined, 'Worker persistence requires redis config (persistence.redis or REDIS_* env values)', 'persistence.redis');
+        const key_prefix = persistence.keyPrefix ?? keyPrefix();
+        logRedisPersistenceConfig(redisConfig, key_prefix, 'createWorkerStore');
+        const client = createRedisConnection(redisConfig);
+        return RedisWorkerStore.create(client, key_prefix);
+    }
+    // Database driver
+    const explicitConnection = typeof persistence.client === 'string' ? persistence.client : persistence.connection;
+    const client = typeof persistence.client === 'string'
+        ? await resolveDbClientFromEnv(explicitConnection)
+        : (persistence.client ?? (await resolveDbClientFromEnv(explicitConnection)));
+    return DbWorkerStore.create(client, persistence.table);
+};
+const resolveWorkerStoreForPersistence = async (persistence) => {
+    const cacheKey = generateCacheKey(persistence);
+    const isCloudflare = Cloudflare.getWorkersEnv() !== null;
+    // Return cached instance if available (disable cache for Cloudflare to assume cleanup)
+    // Or handle cleanup differently. For now, disable cache for Cloudflare to allow per-request connections.
+    const cached = storeInstanceCache.get(cacheKey);
+    if (cached && !isCloudflare) {
+        return cached;
+    }
+    // Create new store instance
+    const store = await createWorkerStore(persistence);
+    await store.init();
+    // Cache the store instance for reuse only if not Cloudflare
+    if (!isCloudflare) {
+        storeInstanceCache.set(cacheKey, store);
+    }
+    return store;
+};
+const getPersistedRecord = async (name, persistenceOverride) => {
+    if (!persistenceOverride) {
+        if (!isCloudflareRuntime()) {
+            await ensureWorkerStoreConfigured();
+            return workerStore.get(name);
+        }
+        const store = await getDefaultStoreForRuntime();
+        try {
+            return await store.get(name);
+        }
+        finally {
+            if (store.close) {
+                await store.close();
+            }
+        }
+    }
+    const store = await resolveWorkerStoreForPersistence(persistenceOverride);
+    return store.get(name);
+};
+const ensureWorkerStoreConfigured = async () => {
+    if (workerStoreConfigured)
+        return;
+    const bootstrapConfig = buildPersistenceBootstrapConfig();
+    const persistence = resolvePersistenceConfig(bootstrapConfig);
+    if (!persistence)
+        return;
+    workerStore = await resolveWorkerStore(bootstrapConfig);
+    workerStoreConfigured = true;
+    workerStoreConfig = persistence;
+};
+const buildWorkerRecord = (config, status) => {
+    const now = new Date();
+    const normalizedProcessorSpec = config.processorSpec
+        ? normalizeProcessorSpec(config.processorSpec)
+        : null;
+    return {
+        name: config.name,
+        queueName: config.queueName,
+        version: config.version ?? '1.0.0',
+        status,
+        autoStart: resolveAutoStart(config),
+        concurrency: config.options?.concurrency ?? 1,
+        region: config.datacenter?.primaryRegion ?? null,
+        processorSpec: normalizedProcessorSpec ?? null,
+        activeStatus: config.activeStatus ?? true,
+        features: config.features ? { ...config.features } : null,
+        infrastructure: config.infrastructure ? { ...config.infrastructure } : null,
+        datacenter: config.datacenter ? { ...config.datacenter } : null,
+        createdAt: now,
+        updatedAt: now,
+        lastHealthCheck: undefined,
+        lastError: undefined,
+        connectionState: undefined,
+    };
+};
+const buildDefaultAutoScalerConfig = () => ({
+    enabled: workersConfig.autoScaling.enabled,
+    checkInterval: workersConfig.autoScaling.interval,
+    scalingPolicies: new Map(),
+    costOptimization: {
+        enabled: workersConfig.costOptimization.enabled,
+        maxCostPerHour: 0,
+        preferSpotInstances: workersConfig.costOptimization.spotInstances,
+        offPeakSchedule: {
+            start: workersConfig.autoScaling.offPeakSchedule.split('-')[0] ?? '22:00',
+            end: workersConfig.autoScaling.offPeakSchedule.split('-')[1] ?? '06:00',
+            timezone: 'UTC',
+            reductionPercentage: Math.round(workersConfig.autoScaling.offPeakReduction * 100),
+        },
+        budgetAlerts: {
+            dailyLimit: 0,
+            weeklyLimit: 0,
+            monthlyLimit: 0,
+        },
+    },
+});
+const resolveOffPeakSchedule = (input, defaults) => {
+    const fallback = defaults.costOptimization.offPeakSchedule ?? {
+        start: '22:00',
+        end: '06:00',
+        timezone: 'UTC',
+        reductionPercentage: 0,
+    };
+    const override = input?.costOptimization?.offPeakSchedule;
+    const schedule = { ...fallback };
+    if (override) {
+        Object.assign(schedule, override);
+    }
+    return schedule;
+};
+const resolveCostOptimization = (input, defaults) => ({
+    ...defaults.costOptimization,
+    ...input?.costOptimization,
+    offPeakSchedule: resolveOffPeakSchedule(input, defaults),
+    budgetAlerts: {
+        ...defaults.costOptimization.budgetAlerts,
+        ...input?.costOptimization?.budgetAlerts,
+    },
+});
+const resolveAutoScalerConfig = (input) => {
+    const defaults = buildDefaultAutoScalerConfig();
+    if (!input)
+        return defaults;
+    return {
+        ...defaults,
+        ...input,
+        costOptimization: resolveCostOptimization(input, defaults),
+    };
+};
+const resolveWorkerOptions = (config, autoStart) => {
+    const options = config.options ? { ...config.options } : {};
+    if (options.prefix === undefined) {
+        options.prefix = getBullMQSafeQueueName();
+    }
+    if (options.autorun === undefined) {
+        options.autorun = autoStart;
+    }
+    if (options.connection)
+        return options;
+    const redisConfig = resolveRedisConfigWithFallback(config.infrastructure?.redis, undefined, 'Worker requires a connection. Provide options.connection or infrastructure.redis config', 'infrastructure.redis');
+    return {
+        ...options,
+        connection: {
+            host: redisConfig.host,
+            port: redisConfig.port,
+            db: redisConfig.db,
+            password: redisConfig.password,
+        },
+    };
+};
+const buildDefaultObservabilityConfig = () => ({
+    prometheus: {
+        enabled: workersConfig.observability.prometheus.enabled,
+        port: workersConfig.observability.prometheus.port,
+    },
+    openTelemetry: {
+        enabled: workersConfig.observability.opentelemetry.enabled,
+        serviceName: 'zintrust-workers',
+        exporterUrl: workersConfig.observability.opentelemetry.endpoint,
+    },
+    datadog: {
+        enabled: workersConfig.observability.datadog.enabled,
+        tags: workersConfig.observability.datadog.apiKey
+            ? [`apiKey:${workersConfig.observability.datadog.apiKey}`]
+            : undefined,
+    },
+});
+const resolveObservabilityConfig = (input) => {
+    const defaults = buildDefaultObservabilityConfig();
+    if (!input)
+        return defaults;
+    const enabledOverride = 'enabled' in input ? input.enabled : undefined;
+    const prometheus = { ...defaults.prometheus };
+    if (input.prometheus) {
+        Object.assign(prometheus, input.prometheus);
+    }
+    const openTelemetry = { ...defaults.openTelemetry };
+    if (input.openTelemetry) {
+        Object.assign(openTelemetry, input.openTelemetry);
+    }
+    const datadog = { ...defaults.datadog };
+    if (input.datadog) {
+        Object.assign(datadog, input.datadog);
+    }
+    if (enabledOverride === false) {
+        prometheus.enabled = false;
+        openTelemetry.enabled = false;
+        datadog.enabled = false;
+    }
+    else if (enabledOverride === true) {
+        prometheus.enabled = true;
+        openTelemetry.enabled = true;
+        datadog.enabled = true;
+    }
+    if (!openTelemetry.serviceName) {
+        openTelemetry.serviceName = defaults.openTelemetry.serviceName;
+    }
+    return { prometheus, openTelemetry, datadog };
+};
+const initializeClustering = (config) => {
+    if (clusteringInitialized || !(config.features?.clustering ?? false))
+        return;
+    const redisConfig = resolveRedisConfigWithFallback(config.infrastructure?.redis, undefined, 'ClusterLock requires infrastructure.redis config', 'infrastructure.redis');
+    ClusterLock.initialize(redisConfig);
+    clusteringInitialized = true;
+};
+const initializeMetrics = (config) => {
+    if (metricsInitialized || !(config.features?.metrics ?? false))
+        return;
+    const redisConfig = resolveRedisConfigWithFallback(config.infrastructure?.redis, undefined, 'WorkerMetrics requires infrastructure.redis config', 'infrastructure.redis');
+    WorkerMetrics.initialize(redisConfig);
+    metricsInitialized = true;
+};
+const initializeAutoScaling = (config) => {
+    if (autoScalingInitialized || !(config.features?.autoScaling ?? false))
+        return;
+    const autoScalerConfig = resolveAutoScalerConfig(config.infrastructure?.autoScaler);
+    AutoScaler.initialize(autoScalerConfig);
+    autoScalingInitialized = true;
+};
+const initializeCircuitBreaker = (config, version) => {
+    if (!(config.features?.circuitBreaker ?? false))
+        return;
+    CircuitBreaker.initialize(config.name, version);
+};
+const initializeDeadLetterQueue = (config) => {
+    if (deadLetterQueueInitialized || !(config.features?.deadLetterQueue ?? false))
+        return;
+    const dlqConfig = requireInfrastructure(config.infrastructure?.deadLetterQueue, 'DeadLetterQueue requires infrastructure.deadLetterQueue config');
+    const dlqRedisConfig = resolveRedisConfigWithFallback(dlqConfig.redis, config.infrastructure?.redis, 'DeadLetterQueue requires infrastructure.deadLetterQueue.redis or infrastructure.redis config', 'infrastructure.deadLetterQueue.redis');
+    DeadLetterQueue.initialize(dlqRedisConfig, dlqConfig.policy);
+    deadLetterQueueInitialized = true;
+};
+const initializeResourceMonitoring = (config) => {
+    if (resourceMonitoringInitialized || !(config.features?.resourceMonitoring ?? false))
+        return;
+    ResourceMonitor.initialize();
+    ResourceMonitor.start();
+    resourceMonitoringInitialized = true;
+};
+const initializeCompliance = (config) => {
+    if (complianceInitialized || !(config.features?.compliance ?? false))
+        return;
+    const complianceConfig = requireInfrastructure(config.infrastructure?.compliance, 'ComplianceManager requires infrastructure.compliance config');
+    const complianceRedisConfig = resolveRedisConfigWithFallback(complianceConfig.redis, config.infrastructure?.redis, 'ComplianceManager requires infrastructure.compliance.redis or infrastructure.redis config', 'infrastructure.compliance.redis');
+    ComplianceManager.initialize(complianceRedisConfig, complianceConfig.config);
+    complianceInitialized = true;
+};
+const initializeObservability = async (config) => {
+    if (observabilityInitialized || !(config.features?.observability ?? false))
+        return;
+    const observabilityConfig = resolveObservabilityConfig(config.infrastructure?.observability);
+    await Observability.initialize(observabilityConfig);
+    observabilityInitialized = true;
+};
+const initializeVersioning = (config, version) => {
+    if (!(config.features?.versioning ?? false))
+        return;
+    WorkerVersioning.register({
+        workerName: config.name,
+        version: WorkerVersioning.parse(version),
+        changelog: 'Initial version',
+    });
+};
+const initializeDatacenter = (config) => {
+    if (!(config.features?.datacenterOrchestration ?? false) || !config.datacenter)
+        return;
+    DatacenterOrchestrator.placeWorker({
+        workerName: config.name,
+        primaryRegion: config.datacenter.primaryRegion,
+        secondaryRegions: config.datacenter.secondaryRegions ?? [],
+        replicationStrategy: 'active-passive',
+        affinityRules: {
+            preferLocal: config.datacenter.affinityRules?.preferLocal ?? true,
+            maxLatency: config.datacenter.affinityRules?.maxLatency,
+            avoidRegions: config.datacenter.affinityRules?.avoidRegions,
+        },
+    });
+};
+const setupWorkerEventListeners = (worker, workerName, workerVersion, features) => {
+    worker.on('completed', (job) => {
+        try {
+            Logger.debug(`Job completed: ${workerName}`, { jobId: job.id });
+            if (features?.observability === true) {
+                Observability.incrementCounter('worker.jobs.completed', 1, {
+                    worker: workerName,
+                    version: workerVersion,
+                });
+            }
+        }
+        catch (error) {
+            // Isolate error - don't let it bubble up
+            Logger.error(`Error in worker completed event handler: ${workerName}`, error, 'workers');
+        }
+    });
+    worker.on('failed', (job, error) => {
+        try {
+            Logger.error(`Job failed: ${workerName}`, { error, jobId: job?.id }, 'workers');
+            if (features?.observability === true) {
+                Observability.incrementCounter('worker.jobs.failed', 1, {
+                    worker: workerName,
+                    version: workerVersion,
+                });
+            }
+        }
+        catch (handlerError) {
+            // Isolate error - don't let it bubble up
+            Logger.error(`Error in worker failed event handler: ${workerName}`, handlerError, 'workers');
+        }
+    });
+    worker.on('error', (error) => {
+        try {
+            Logger.error(`Worker error: ${workerName}`, error);
+            // Check if this is a Redis connection error that should be handled gracefully
+            if (error.message.includes('ERR value is not an integer') ||
+                error.message.includes('NOAUTH') ||
+                error.message.includes('ECONNREFUSED')) {
+                Logger.warn(`Worker ${workerName} encountered Redis configuration error - worker will remain failed but server will continue running`);
+            }
+        }
+        catch (handlerError) {
+            // Isolate error - don't let it bubble up
+            Logger.error(`Error in worker error event handler: ${workerName}`, handlerError, 'workers');
+        }
+    });
+};
+const registerWorkerInstance = (params) => {
+    const { worker, config, workerVersion, queueName, options, autoStart } = params;
+    WorkerRegistry.register({
+        name: config.name,
+        config: {},
+        activeStatus: config.activeStatus ?? true,
+        version: workerVersion,
+        region: config.datacenter?.primaryRegion,
+        queues: [queueName],
+        factory: async () => {
+            await Promise.resolve();
+            return {
+                metadata: {
+                    name: config.name,
+                    status: autoStart ? 'running' : 'stopped',
+                    version: workerVersion,
+                    region: config.datacenter?.primaryRegion ?? 'unknown',
+                    queueName,
+                    concurrency: options?.concurrency ?? 1,
+                    activeStatus: config.activeStatus ?? true,
+                    startedAt: new Date(),
+                    stoppedAt: null,
+                    lastProcessedAt: null,
+                    restartCount: 0,
+                    processedCount: 0,
+                    errorCount: 0,
+                    lockKey: null,
+                    priority: 0,
+                    memoryUsage: 0,
+                    cpuUsage: 0,
+                    circuitState: 'closed',
+                    queues: [queueName],
+                    plugins: [],
+                    datacenter: config.datacenter?.primaryRegion ?? 'unknown',
+                    canaryPercentage: 0,
+                    config: {},
+                },
+                instance: worker,
+                start: () => {
+                    if (!autoStart) {
+                        worker.run().catch((error) => {
+                            Logger.error(`Failed to start worker "${config.name}"`, error);
+                        });
+                    }
+                },
+                stop: async () => worker.close(),
+                drain: async () => worker.close(),
+                sleep: async () => worker.pause(),
+                wakeup: () => {
+                    worker.resume();
+                },
+                getStatus: () => 'running',
+                getHealth: () => 'green',
+            };
+        },
+    });
+};
+const initializeWorkerFeatures = async (config, workerVersion) => {
+    initializeClustering(config);
+    initializeMetrics(config);
+    initializeAutoScaling(config);
+    initializeCircuitBreaker(config, workerVersion);
+    initializeDeadLetterQueue(config);
+    initializeResourceMonitoring(config);
+    initializeCompliance(config);
+    await initializeObservability(config);
+    initializeVersioning(config, workerVersion);
+    initializeDatacenter(config);
+};
+/**
+ * Worker Factory - Sealed namespace
+ */
+export const WorkerFactory = Object.freeze({
+    registerProcessor,
+    registerProcessors,
+    registerProcessorPaths,
+    registerProcessorResolver,
+    registerProcessorSpec,
+    resolveProcessorPath,
+    resolveProcessorSpec,
+    /**
+     * Register a new worker configuration without starting it.
+     */
+    async register(config) {
+        const { name } = config;
+        // Check in-memory first (though unlikely if we are just registering)
+        if (workers.has(name)) {
+            throw ErrorFactory.createWorkerError(`Worker "${name}" is already running locally`);
+        }
+        const store = await getStoreForWorker(config);
+        try {
+            const existing = await store.get(name);
+            if (existing) {
+                throw ErrorFactory.createWorkerError(`Worker "${name}" already exists in persistence`);
+            }
+            // Init features to validate config, but mainly we just want to save it.
+            // initializeWorkerFeatures might rely on being active or having resources, so we might skip it or do partial.
+            // For now, just save definition.
+            // Status should be STOPPED or CREATED.
+            await store.save(buildWorkerRecord(config, WorkerCreationStatus.STOPPED));
+            Logger.info(`Worker registered (persistence only): ${name}`);
+        }
+        finally {
+            // If Cloudflare environment, try to close store connection to avoid zombie connections
+            if (Cloudflare.getWorkersEnv() !== null && store.close) {
+                await store.close();
+            }
+        }
+    },
+    /**
+     * Create new worker with full setup
+     */
+    async create(config) {
+        const { name, version, queueName, features } = config;
+        const workerVersion = version ?? '1.0.0';
+        const autoStart = resolveAutoStart(config);
+        if (workers.has(name)) {
+            throw ErrorFactory.createWorkerError(`Worker "${name}" already exists`);
+        }
+        // Resolve the correct store for this worker configuration
+        const store = await getStoreForWorker(config);
+        // Save initial status as "creating"
+        await store.save(buildWorkerRecord(config, WorkerCreationStatus.CREATING));
+        try {
+            await initializeWorkerFeatures(config, workerVersion);
+            // Update status to "connecting"
+            await store.update(name, {
+                status: WorkerCreationStatus.CONNECTING,
+                updatedAt: new Date(),
+            });
+            // Create enhanced processor
+            const enhancedProcessor = createEnhancedProcessor(config);
+            // Create BullMQ worker
+            const resolvedOptions = resolveWorkerOptions(config, autoStart);
+            const worker = new Worker(queueName, enhancedProcessor, resolvedOptions);
+            setupWorkerEventListeners(worker, name, workerVersion, features);
+            // Update status to "starting"
+            await store.update(name, {
+                status: WorkerCreationStatus.STARTING,
+                updatedAt: new Date(),
+            });
+            const timeoutMs = Env.getInt('WORKER_CONNECTION_TIMEOUT', 5000);
+            // Wait for actual connection and health verification
+            await waitForWorkerConnection(worker, name, queueName, timeoutMs);
+            // Update status to "running" only after successful connection
+            await store.update(name, {
+                status: WorkerCreationStatus.RUNNING,
+                updatedAt: new Date(),
+            });
+            // Store worker instance
+            const instance = {
+                worker,
+                config,
+                startedAt: new Date(),
+                status: WorkerCreationStatus.RUNNING,
+                connectionState: 'connected',
+            };
+            workers.set(name, instance);
+            registerWorkerInstance({
+                worker,
+                config,
+                workerVersion,
+                queueName,
+                options: resolvedOptions,
+                autoStart,
+            });
+            if (autoStart) {
+                await WorkerRegistry.start(name, workerVersion);
+            }
+            // Execute afterStart hooks
+            if (features?.plugins === true) {
+                await PluginManager.executeHook('afterStart', {
+                    workerName: name,
+                    timestamp: new Date(),
+                });
+            }
+            // Start health monitoring for the worker
+            startHealthMonitoring(name, worker, queueName);
+            return worker;
+        }
+        catch (error) {
+            // Handle failure - update status to "failed"
+            // Re-resolve store in case of error to be safe
+            const failStore = await getStoreForWorker(config);
+            await failStore.update(name, {
+                status: WorkerCreationStatus.FAILED,
+                updatedAt: new Date(),
+                lastError: error.message,
+            });
+            Logger.error(`Worker creation failed: ${name}`, error);
+            throw error;
+        }
+    },
+    /**
+     * Get worker instance
+     */
+    get(name) {
+        const instance = workers.get(name);
+        return instance ? { ...instance } : null;
+    },
+    /**
+     * Update worker status directly (used by HealthMonitor)
+     */
+    async updateStatus(name, status, error) {
+        const instance = workers.get(name);
+        if (instance) {
+            instance.status = status;
+        }
+        try {
+            const store = await getStoreForWorker(instance?.config ?? {
+                name,
+                queueName: 'unknown',
+                processor: async () => {
+                    return Promise.resolve(); //NOSONAR
+                },
+            });
+            const errorMessage = typeof error === 'string' ? error : error?.message;
+            await store.update(name, {
+                status: status,
+                updatedAt: new Date(),
+                lastError: errorMessage,
+            });
+        }
+        catch (err) {
+            Logger.warn(`Failed to update status for ${name} to ${status}`, err);
+        }
+    },
+    /**
+     * Stop worker
+     */
+    async stop(name, persistenceOverride, options) {
+        const skipPersistedUpdate = options?.skipPersistedUpdate === true;
+        const instance = workers.get(name);
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (!instance) {
+            if (!skipPersistedUpdate) {
+                await store.update(name, { status: 'stopped', updatedAt: new Date() });
+                Logger.info(`Worker marked stopped (not running): ${name}`);
+            }
+            return;
+        }
+        // Execute beforeStop hooks
+        if (instance.config.features?.plugins === true) {
+            await PluginManager.executeHook('beforeStop', {
+                workerName: name,
+                timestamp: new Date(),
+            });
+        }
+        // Close worker with timeout to prevent hanging
+        const workerClosePromise = instance.worker.close();
+        let timeoutId;
+        const timeoutPromise = new Promise((_, reject) => {
+            // eslint-disable-next-line no-restricted-syntax
+            timeoutId = setTimeout(() => {
+                reject(new Error('Worker close timeout'));
+            }, 5000);
+        });
+        try {
+            await Promise.race([workerClosePromise, timeoutPromise]);
+        }
+        catch (error) {
+            Logger.warn(`Worker "${name}" close failed or timed out, continuing...`, error);
+        }
+        finally {
+            // Always clean up timeout to prevent memory leak
+            if (timeoutId) {
+                clearTimeout(timeoutId);
+                timeoutId = undefined;
+            }
+        }
+        instance.status = WorkerCreationStatus.STOPPED;
+        // Stop health monitoring for this worker
+        HealthMonitor.unregister(name);
+        if (!skipPersistedUpdate) {
+            try {
+                await store.update(name, {
+                    status: WorkerCreationStatus.STOPPED,
+                    updatedAt: new Date(),
+                });
+                Logger.info(`Worker "${name}" status updated to stopped`);
+            }
+            catch (error) {
+                Logger.error(`Failed to update worker "${name}" status`, error);
+            }
+        }
+        await WorkerRegistry.stop(name);
+        // Execute afterStop hooks
+        if (instance.config.features?.plugins === true) {
+            await PluginManager.executeHook('afterStop', {
+                workerName: name,
+                timestamp: new Date(),
+            });
+        }
+        Logger.info(`Worker stopped: ${name}`);
+    },
+    /**
+     * Restart worker
+     */
+    async restart(name, persistenceOverride) {
+        const instance = workers.get(name);
+        if (!instance) {
+            await WorkerFactory.startFromPersisted(name, persistenceOverride);
+            Logger.info(`Worker started from persistence: ${name}`);
+            return;
+        }
+        await WorkerFactory.stop(name, persistenceOverride);
+        const refreshed = workers.get(name);
+        if (!refreshed) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found`);
+        }
+        workers.delete(name);
+        const newWorker = await WorkerFactory.create(refreshed.config);
+        refreshed.worker = newWorker;
+        refreshed.status = WorkerCreationStatus.RUNNING;
+        refreshed.startedAt = new Date();
+        Logger.info(`Worker restarted: ${name}`);
+    },
+    /**
+     * Pause worker
+     */
+    async pause(name, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (instance) {
+            await instance.worker.pause();
+            instance.status = WorkerCreationStatus.STARTING; // Using STARTING as equivalent to sleeping/paused
+        }
+        await store.update(name, {
+            status: WorkerCreationStatus.STARTING,
+            updatedAt: new Date(),
+        });
+        Logger.info(`Worker paused: ${name}`);
+    },
+    /**
+     * Resume worker
+     */
+    async resume(name, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (instance) {
+            instance.worker.resume();
+            instance.status = WorkerCreationStatus.RUNNING;
+        }
+        try {
+            await store.update(name, { status: WorkerCreationStatus.RUNNING, updatedAt: new Date() });
+        }
+        catch (error) {
+            Logger.error('Failed to persist worker resume', error);
+        }
+        Logger.info(`Worker resumed: ${name}`);
+    },
+    /**
+     * Update auto-start for persisted worker
+     */
+    async setAutoStart(name, autoStart, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (instance) {
+            instance.config.autoStart = autoStart;
+        }
+        await store.update(name, { autoStart, updatedAt: new Date() });
+        if (!autoStart)
+            return;
+        const refreshed = workers.get(name);
+        if (refreshed) {
+            if (refreshed.status !== 'running') {
+                await WorkerFactory.start(name, persistenceOverride);
+            }
+            return;
+        }
+        await WorkerFactory.startFromPersisted(name, persistenceOverride);
+    },
+    /**
+     * Update active status for a worker
+     */
+    async setWorkerActiveStatus(name, activeStatus, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (instance) {
+            instance.config.activeStatus = activeStatus;
+        }
+        await store.update(name, { activeStatus, updatedAt: new Date() });
+        WorkerRegistry.setActiveStatus(name, activeStatus);
+        if (activeStatus === false && instance) {
+            await WorkerFactory.stop(name, persistenceOverride);
+        }
+    },
+    /**
+     * Get active status for a worker
+     */
+    async getWorkerActiveStatus(name, persistenceOverride) {
+        const instance = workers.get(name);
+        if (instance?.config.activeStatus !== undefined) {
+            return instance.config.activeStatus;
+        }
+        const store = await getStoreForWorker(instance?.config, persistenceOverride);
+        const record = await store.get(name);
+        if (!record)
+            return null;
+        return record.activeStatus ?? true;
+    },
+    /**
+     * Update persisted worker record and in-memory config if running.
+     */
+    async update(name, patch, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await getStoreForWorker(instance?.config, persistenceOverride);
+        const current = await store.get(name);
+        if (!current) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found in persistence store`);
+        }
+        const merged = {
+            ...current,
+            ...patch,
+            updatedAt: patch.updatedAt ?? new Date(),
+        };
+        // Use save() which will insert or update appropriately for each store
+        await store.save(merged);
+        // If the worker is running in memory, update its runtime config so restarts use the new config
+        if (instance) {
+            const cfg = instance.config;
+            instance.config = {
+                ...cfg,
+                version: merged.version ?? cfg.version,
+                queueName: merged.queueName ?? cfg.queueName,
+                options: {
+                    ...cfg.options,
+                    concurrency: merged.concurrency ?? cfg.options?.concurrency,
+                },
+                processorSpec: merged.processorSpec ?? cfg.processorSpec,
+                activeStatus: merged.activeStatus ?? cfg.activeStatus,
+                infrastructure: merged.infrastructure ?? cfg.infrastructure,
+                features: merged.features ?? cfg.features,
+                datacenter: merged.datacenter ?? cfg.datacenter,
+            };
+        }
+    },
+    /**
+     * Start worker
+     */
+    async start(name, persistenceOverride) {
+        const instance = workers.get(name);
+        // Even if instance exists, we must validate against the requested driver
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (!instance) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found`);
+        }
+        if (instance.config.activeStatus === false) {
+            throw ErrorFactory.createConfigError(`Worker "${name}" is inactive`);
+        }
+        const persisted = await store.get(name);
+        if (persisted?.activeStatus === false) {
+            throw ErrorFactory.createConfigError(`Worker "${name}" is inactive`);
+        }
+        const version = instance.config.version ?? '1.0.0';
+        await WorkerRegistry.start(name, version);
+        instance.status = WorkerCreationStatus.RUNNING;
+        instance.startedAt = new Date();
+        await store.update(name, { status: WorkerCreationStatus.RUNNING, updatedAt: new Date() });
+        Logger.info(`Worker started: ${name}`);
+    },
+    /**
+     * List all workers
+     */
+    list() {
+        return Array.from(workers.keys());
+    },
+    /**
+     * List all persisted workers
+     */
+    async listPersisted(persistenceOverride, options) {
+        const records = await WorkerFactory.listPersistedRecords(persistenceOverride, options);
+        return records.map((record) => record.name);
+    },
+    async listPersistedRecords(persistenceOverride, options) {
+        const includeInactive = options?.includeInactive === true;
+        if (!persistenceOverride) {
+            if (!isCloudflareRuntime()) {
+                await ensureWorkerStoreConfigured();
+                const records = await workerStore.list(options);
+                return includeInactive
+                    ? records
+                    : records.filter((record) => record.activeStatus !== false);
+            }
+            const store = await getDefaultStoreForRuntime();
+            try {
+                const records = await store.list(options);
+                return includeInactive
+                    ? records
+                    : records.filter((record) => record.activeStatus !== false);
+            }
+            finally {
+                if (store.close) {
+                    await store.close();
+                }
+            }
+        }
+        const store = await resolveWorkerStoreForPersistence(persistenceOverride);
+        const records = await store.list(options);
+        return includeInactive ? records : records.filter((record) => record.activeStatus !== false);
+    },
+    /**
+     * Start a worker from persisted storage when it is not registered.
+     */
+    async startFromPersisted(name, persistenceOverride) {
+        const record = await getPersistedRecord(name, persistenceOverride);
+        if (!record) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found in persistence store`);
+        }
+        if (record.activeStatus === false) {
+            throw ErrorFactory.createConfigError(`Worker "${name}" is inactive`);
+        }
+        let processor = await resolveProcessor(name);
+        const spec = record.processorSpec ?? undefined;
+        if (!processor && spec) {
+            try {
+                processor = await resolveProcessorSpec(spec);
+            }
+            catch (error) {
+                Logger.error(`Failed to resolve processor module for "${name}"`, error);
+            }
+        }
+        if (!processor) {
+            throw ErrorFactory.createConfigError(`Worker "${name}" processor is not registered or resolvable. Register the processor at startup or persist a processorSpec.`);
+        }
+        await WorkerFactory.create({
+            name: record.name,
+            queueName: record.queueName,
+            version: record.version ?? undefined,
+            processor,
+            processorSpec: record.processorSpec ?? undefined,
+            activeStatus: record.activeStatus ?? true,
+            autoStart: true, // Override to true when manually starting
+            options: { concurrency: record.concurrency },
+            infrastructure: record.infrastructure,
+            features: record.features,
+            datacenter: record.datacenter,
+        });
+    },
+    /**
+     * Get persisted worker record
+     */
+    async getPersisted(name, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await getStoreForWorker(instance?.config, persistenceOverride);
+        try {
+            const result = await store.get(name);
+            return result;
+        }
+        finally {
+            if (Cloudflare.getWorkersEnv() !== null && store.close) {
+                await store.close();
+            }
+        }
+    },
+    /**
+     * Remove worker
+     */
+    async remove(name, persistenceOverride) {
+        const instance = workers.get(name);
+        // Validate that worker exists in the store we are trying to remove from
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (instance) {
+            await WorkerFactory.stop(name, persistenceOverride);
+            const registry = WorkerRegistry;
+            registry.unregister?.(name);
+            AutoScaler.clearHistory(name);
+            ResourceMonitor.clearHistory(name);
+            CircuitBreaker.deleteWorker(name);
+            CanaryController.purge(name);
+            WorkerVersioning.clear(name);
+            DatacenterOrchestrator.removeWorker(name);
+            await Observability.clearWorkerMetrics(name);
+            // Stop health monitoring for this worker
+            HealthMonitor.unregister(name);
+            workers.delete(name);
+        }
+        await store.remove(name);
+        Logger.info(`Worker removed: ${name}`);
+    },
+    /**
+     * Get worker metrics
+     */
+    async getMetrics(name) {
+        const instance = workers.get(name);
+        if (!instance) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found`);
+        }
+        if (instance.config.features?.metrics === undefined || !instance.config.features?.metrics) {
+            return null;
+        }
+        const now = Date.now();
+        const oneHourAgo = now - 3600 * 1000;
+        const metrics = await WorkerMetrics.aggregate({
+            workerName: name,
+            metricType: 'processed',
+            granularity: 'hourly',
+            startDate: new Date(oneHourAgo),
+            endDate: new Date(now),
+        });
+        return metrics;
+    },
+    /**
+     * Get worker health
+     */
+    async getHealth(name) {
+        const instance = workers.get(name);
+        if (!instance) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found`);
+        }
+        if (!(instance.config.features?.metrics ?? false)) {
+            return { status: 'unknown' };
+        }
+        const health = await WorkerMetrics.getLatestHealth(name);
+        return health;
+    },
+    /**
+     * Shutdown all workers
+     */
+    async shutdown() {
+        Logger.info('WorkerFactory shutting down...');
+        const workerEntries = Array.from(workers.entries());
+        const workerNames = workerEntries.map(([name]) => name);
+        // Bulk-update persisted statuses before stopping workers to avoid per-worker DB updates
+        // during shutdown (which can fail if DB connections are closing).
+        const storeGroups = new Map();
+        // Parallel get stores for all workers
+        const storePromises = workerEntries.map(async ([name, instance]) => {
+            const store = await getStoreForWorker(instance.config);
+            return { name, store };
+        });
+        const storeMappings = await Promise.all(storePromises);
+        for (const { name, store } of storeMappings) {
+            const existing = storeGroups.get(store);
+            if (existing) {
+                existing.push(name);
+            }
+            else {
+                storeGroups.set(store, [name]);
+            }
+        }
+        // Parallel bulk updates for all store groups
+        const updatePromises = Array.from(storeGroups.entries()).map(async ([store, names]) => {
+            if (typeof store.updateMany === 'function') {
+                await store.updateMany(names, {
+                    status: WorkerCreationStatus.STOPPED,
+                    updatedAt: new Date(),
+                });
+            }
+        });
+        await Promise.all(updatePromises);
+        await Promise.all(workerNames.map(async (name) => WorkerFactory.stop(name, undefined, { skipPersistedUpdate: true })));
+        // Shutdown all modules
+        ResourceMonitor.stop();
+        await WorkerMetrics.shutdown();
+        await MultiQueueWorker.shutdown();
+        await ComplianceManager.shutdown();
+        await PriorityQueue.shutdown();
+        HealthMonitor.shutdown();
+        AutoScaler.stop();
+        ClusterLock.shutdown();
+        WorkerVersioning.shutdown();
+        CanaryController.shutdown();
+        DatacenterOrchestrator.shutdown();
+        PluginManager.shutdown();
+        Observability.shutdown();
+        await DeadLetterQueue.shutdown();
+        CircuitBreaker.shutdown();
+        workers.clear();
+        Logger.info('WorkerFactory shutdown complete');
+    },
+    /**
+     * Reset persistence connection state.
+     * Useful when connections become stale in long-running processes or serverless environments.
+     */
+    async resetPersistence() {
+        workerStoreConfigured = false;
+        workerStore = InMemoryWorkerStore.create();
+        storeInstanceCache.clear();
+        Logger.info('Worker persistence configuration reset');
+    },
+});
+// Graceful shutdown handled by WorkerShutdown