@zintrust/workers 0.1.27

package/dist/WorkerFactory.js

@@ -0,0 +1,1507 @@
+/**
+ * Worker Factory
+ * Central factory for creating workers with all advanced features
+ * Sealed namespace for immutability
+ */
+import { appConfig, createRedisConnection, databaseConfig, Env, ErrorFactory, getBullMQSafeQueueName, Logger, NodeSingletons, queueConfig, registerDatabasesFromRuntimeConfig, useEnsureDbConnected, workersConfig, } from '@zintrust/core';
+import { Worker } from 'bullmq';
+import { AutoScaler } from './AutoScaler';
+import { CanaryController } from './CanaryController';
+import { CircuitBreaker } from './CircuitBreaker';
+import { ClusterLock } from './ClusterLock';
+import { ComplianceManager } from './ComplianceManager';
+import { DatacenterOrchestrator } from './DatacenterOrchestrator';
+import { DeadLetterQueue } from './DeadLetterQueue';
+import { HealthMonitor } from './HealthMonitor';
+import { MultiQueueWorker } from './MultiQueueWorker';
+import { Observability } from './Observability';
+import { PluginManager } from './PluginManager';
+import { PriorityQueue } from './PriorityQueue';
+import { ResourceMonitor } from './ResourceMonitor';
+import { WorkerMetrics } from './WorkerMetrics';
+import { WorkerRegistry } from './WorkerRegistry';
+import { WorkerVersioning } from './WorkerVersioning';
+import { DbWorkerStore, InMemoryWorkerStore, RedisWorkerStore, } from './storage/WorkerStore';
+const path = NodeSingletons.path;
+const getStoreForWorker = async (config, persistenceOverride) => {
+    if (persistenceOverride) {
+        return resolveWorkerStoreForPersistence(persistenceOverride);
+    }
+    // If worker has specific configuration, use it
+    if (config) {
+        const persistence = resolvePersistenceConfig(config);
+        if (persistence) {
+            return resolveWorkerStoreForPersistence(persistence);
+        }
+    }
+    // Fallback to default/global store
+    await ensureWorkerStoreConfigured();
+    return workerStore;
+};
+const validateAndGetStore = async (name, config, persistenceOverride) => {
+    const store = await getStoreForWorker(config, persistenceOverride);
+    const record = await store.get(name);
+    if (!record) {
+        throw ErrorFactory.createNotFoundError(`Worker "${name}" not found in the specified driver. Ensure you are addressing the correct storage backend.`);
+    }
+    return store;
+};
+// Worker creation status enum for proper lifecycle management
+export const WorkerCreationStatus = {
+    CREATING: 'creating', // Initial state - worker is being created
+    CONNECTING: 'connecting', // Connecting to Redis/Queue
+    STARTING: 'starting', // Starting BullMQ worker
+    RUNNING: 'running', // Actually processing jobs
+    FAILED: 'failed', // Connection/startup failed
+    STOPPED: 'stopped', // Intentionally stopped
+};
+// Internal initialization state to prevent memory leaks and redundant calls
+let clusteringInitialized = false;
+let metricsInitialized = false;
+let autoScalingInitialized = false;
+let deadLetterQueueInitialized = false;
+let resourceMonitoringInitialized = false;
+let complianceInitialized = false;
+let observabilityInitialized = false;
+// Internal state
+const workers = new Map();
+let workerStore = InMemoryWorkerStore.create();
+let workerStoreConfigured = false;
+let workerStoreConfig = null;
+const processorRegistry = new Map();
+const processorPathRegistry = new Map();
+const processorResolvers = [];
+const buildPersistenceBootstrapConfig = () => {
+    const driver = Env.get('WORKER_PERSISTENCE_DRIVER', 'memory');
+    const config = {
+        name: '__zintrust_persistence_bootstrap__',
+        queueName: '__zintrust_bootstrap__',
+        processor: async () => undefined,
+        infrastructure: {
+            persistence: {
+                driver,
+            },
+        },
+    };
+    // Add Redis config if using Redis persistence
+    if (driver === 'redis') {
+        config.infrastructure = {
+            ...config.infrastructure,
+            redis: queueConfig.drivers.redis,
+        };
+    }
+    return config;
+};
+const registerProcessor = (name, processor) => {
+    processorRegistry.set(name, processor);
+};
+const registerProcessors = (processors) => {
+    Object.entries(processors).forEach(([name, processor]) => {
+        if (typeof processor === 'function') {
+            processorRegistry.set(name, processor);
+        }
+    });
+};
+const registerProcessorPaths = (paths) => {
+    Object.entries(paths).forEach(([name, modulePath]) => {
+        if (typeof modulePath === 'string' && modulePath.trim().length > 0) {
+            processorPathRegistry.set(name, modulePath);
+        }
+    });
+};
+const registerProcessorResolver = (resolver) => {
+    processorResolvers.push(resolver);
+};
+const decodeProcessorPathEntities = (value) => value
+    .replaceAll(///gi, '/')
+    .replaceAll('/', '/')
+    .replaceAll(///gi, '/');
+const waitForWorkerConnection = async (worker, name, _queueName, timeoutMs) => {
+    const startTime = Date.now();
+    const checkInterval = 100; // 100ms between checks
+    let timeoutId = null;
+    return new Promise((resolve, reject) => {
+        const checkConnection = async () => {
+            try {
+                // Check if worker is actually running
+                const isRunning = await worker.isRunning();
+                if (!isRunning) {
+                    throw ErrorFactory.createWorkerError('Worker not running');
+                }
+                // Check Redis connection
+                const client = await worker.client;
+                const pingResult = await client.ping();
+                if (pingResult !== 'PONG') {
+                    throw ErrorFactory.createWorkerError('Redis ping failed');
+                }
+                // Removed heavy Queue instantiation loop - relying on Redis ping for connectivity check
+                // The queue instance creation was causing memory pressure and potential connection leaks in this retry loop
+                Logger.debug(`Worker health verification passed for ${name}`, {
+                    isRunning,
+                    pingResult,
+                });
+                if (timeoutId)
+                    clearTimeout(timeoutId);
+                resolve();
+                return;
+            }
+            catch (error) {
+                Logger.debug(`Worker health verification failed for ${name}, retrying...`, error);
+                // Check timeout
+                if (Date.now() - startTime >= timeoutMs) {
+                    if (timeoutId)
+                        clearTimeout(timeoutId);
+                    reject(ErrorFactory.createWorkerError('Worker failed health verification within timeout period'));
+                    return;
+                }
+                // Schedule next check
+                timeoutId = globalThis.setTimeout(checkConnection, checkInterval);
+            }
+        };
+        // Start checking
+        checkConnection();
+    });
+};
+const startHealthMonitoring = (name, worker, queueName) => {
+    HealthMonitor.register(name, worker, queueName);
+};
+const sanitizeProcessorPath = (value) => {
+    const decoded = decodeProcessorPathEntities(value);
+    const base = decoded.split(/[?#&]/)[0]?.trim() ?? '';
+    if (!base)
+        return '';
+    const isAbsolutePath = base.startsWith('/') || /^[A-Za-z]:[\\/]/.test(base);
+    const relativePath = base.startsWith('.') ? base : `./${base}`;
+    return isAbsolutePath ? base : path.resolve(process.cwd(), relativePath);
+};
+const resolveProcessorFromPath = async (modulePath) => {
+    const trimmed = modulePath.trim();
+    if (!trimmed)
+        return undefined;
+    const resolved = sanitizeProcessorPath(trimmed);
+    if (!resolved)
+        return undefined;
+    try {
+        const mod = await import(resolved);
+        const candidate = mod?.default ?? mod?.processor ?? mod?.handler ?? mod?.handle;
+        if (typeof candidate !== 'function') {
+            Logger.warn(`Module imported from ${resolved} but no valid processor function found (exported: ${Object.keys(mod)})`);
+        }
+        return typeof candidate === 'function'
+            ? candidate
+            : undefined;
+    }
+    catch (err) {
+        Logger.error(`Failed to import processor from path: ${resolved}`, err);
+        return undefined;
+    }
+};
+const resolveProcessor = async (name) => {
+    const direct = processorRegistry.get(name);
+    if (direct)
+        return direct;
+    const pathHint = processorPathRegistry.get(name);
+    if (pathHint) {
+        try {
+            const resolved = await resolveProcessorFromPath(pathHint);
+            if (resolved)
+                return resolved;
+        }
+        catch (error) {
+            Logger.error(`Failed to resolve processor module for "${name}"`, error);
+        }
+    }
+    const resolverResults = await Promise.all(processorResolvers.map(async (resolver) => {
+        try {
+            return await resolver(name);
+        }
+        catch (error) {
+            Logger.error(`Processor resolver failed for "${name}"`, error);
+            return undefined;
+        }
+    }));
+    const resolvedFromResolvers = resolverResults.find((result) => result !== undefined);
+    if (resolvedFromResolvers)
+        return resolvedFromResolvers;
+    return undefined;
+};
+const resolveProcessorPath = async (modulePath) => resolveProcessorFromPath(modulePath);
+const recordMetricSafely = (workerName, metricType, value, metadata) => {
+    WorkerMetrics.record(workerName, metricType, value, metadata).catch((error) => {
+        Logger.error(`Failed to record worker metric: ${workerName}/${metricType}`, error);
+    });
+};
+const ensureCircuitAllowsExecution = (workerName, version, jobId, features) => {
+    if (!(features?.circuitBreaker ?? false))
+        return;
+    const canExecute = CircuitBreaker.canExecute(workerName, version);
+    if (canExecute)
+        return;
+    const state = CircuitBreaker.getState(workerName, version);
+    Logger.warn('Circuit breaker is open, rejecting job', {
+        workerName,
+        version,
+        jobId,
+        circuitState: state?.state,
+    });
+    CircuitBreaker.recordRejection(workerName, version);
+    throw ErrorFactory.createGeneralError(`Circuit breaker is open for ${workerName}@${version}`);
+};
+const runBeforeProcessHooks = async (workerName, job, features) => {
+    if (!(features?.plugins ?? false)) {
+        return { skip: false, jobData: job.data };
+    }
+    const hookResult = await PluginManager.executeHook('beforeProcess', {
+        workerName,
+        jobId: job.id ?? '',
+        jobData: job.data,
+        timestamp: new Date(),
+    });
+    if (hookResult.stopped) {
+        const errorMessage = hookResult.errors[0]?.error?.message ?? 'Stopped by plugin';
+        Logger.info('Job processing stopped by plugin', {
+            workerName,
+            jobId: job.id,
+            reason: errorMessage,
+        });
+        return { skip: true, reason: errorMessage };
+    }
+    if (hookResult.modified) {
+        return { skip: false, jobData: hookResult.context.jobData };
+    }
+    return { skip: false, jobData: job.data };
+};
+const startProcessingSpan = (workerName, version, job, queueName, features) => {
+    if (!(features?.observability ?? false))
+        return null;
+    return Observability.startSpan(`worker.${workerName}.process`, {
+        attributes: {
+            worker_name: workerName,
+            worker_version: version,
+            job_id: job.id ?? '',
+            queue_name: queueName,
+        },
+    });
+};
+const usePluginManager = async (workerName, job, result) => {
+    await PluginManager.executeHook('afterProcess', {
+        workerName,
+        jobId: job.id ?? '',
+        jobData: job.data,
+        metadata: { result },
+        timestamp: new Date(),
+    });
+    await PluginManager.executeHook('onComplete', {
+        workerName,
+        jobId: job.id ?? '',
+        jobData: job.data,
+        metadata: { result },
+        timestamp: new Date(),
+    });
+};
+const handleSuccess = async (params) => {
+    const { workerName, jobVersion, job, result, duration, spanId, features } = params;
+    if (features?.metrics ?? false) {
+        recordMetricSafely(workerName, 'processed', 1);
+        recordMetricSafely(workerName, 'duration', duration);
+    }
+    if (features?.circuitBreaker ?? false) {
+        CircuitBreaker.recordSuccess(workerName, jobVersion);
+    }
+    if (features?.observability ?? false) {
+        Observability.recordJobMetrics(workerName, job.name, {
+            processed: 1,
+            failed: 0,
+            durationMs: duration,
+        });
+        if (spanId !== null) {
+            Observability.endSpan(spanId, { success: true });
+        }
+    }
+    if (features?.plugins ?? false) {
+        await usePluginManager(workerName, { id: job.id ?? '', data: job.data }, result);
+    }
+};
+const recordFailureMetrics = (workerName, _jobVersion, duration, features) => {
+    if (features?.metrics === true) {
+        recordMetricSafely(workerName, 'errors', 1);
+        recordMetricSafely(workerName, 'duration', duration);
+    }
+};
+const recordFailureObservability = (workerName, jobName, duration, spanId, features) => {
+    if (features?.observability === true) {
+        Observability.recordJobMetrics(workerName, jobName, {
+            processed: 0,
+            failed: 1,
+            durationMs: duration,
+        });
+        if (spanId !== null) {
+            Observability.recordSpanError(spanId, ErrorFactory.createGeneralError('Job processing failed'));
+            Observability.endSpan(spanId, { success: false });
+        }
+    }
+};
+const addFailedJobToDeadLetterQueue = async (workerName, job, error, duration, jobVersion, queueName, features) => {
+    if (features?.deadLetterQueue === true) {
+        await DeadLetterQueue.addFailedJob({
+            id: job.id ?? '',
+            queueName,
+            workerName,
+            jobName: job.name,
+            data: job.data,
+            error: {
+                name: error.name,
+                message: error.message,
+                stack: error.stack,
+            },
+            attemptsMade: job.attemptsMade ?? 0,
+            maxAttempts: job.opts.attempts ?? 0,
+            failedAt: new Date(),
+            firstAttemptAt: new Date(job.timestamp ?? Date.now()),
+            lastAttemptAt: new Date(),
+            processingTime: duration,
+            metadata: {
+                version: jobVersion,
+            },
+            complianceFlags: {
+                containsPII: false,
+                containsPHI: false,
+                dataClassification: 'public',
+            },
+        });
+    }
+};
+const executeFailurePlugins = async (workerName, job, error, features) => {
+    if (features?.plugins === true) {
+        await PluginManager.executeHook('onError', {
+            workerName,
+            jobId: job.id ?? '',
+            jobData: job.data,
+            error,
+            timestamp: new Date(),
+        });
+    }
+};
+const recordCircuitBreakerFailure = (workerName, jobVersion, error, features) => {
+    if (features?.circuitBreaker === true) {
+        CircuitBreaker.recordFailure(workerName, jobVersion, error);
+    }
+};
+const logAndRecordFailure = (workerName, jobVersion, job, error, features) => {
+    Logger.error(`Worker job failed: ${workerName}`, { error, jobId: job.id, version: jobVersion }, 'workers');
+    recordCircuitBreakerFailure(workerName, jobVersion, error, features);
+};
+const recordFailureObservabilityAndMetrics = (params) => {
+    const { workerName, jobVersion, jobName, duration, spanId, features } = params;
+    recordFailureMetrics(workerName, jobVersion, duration, features);
+    recordFailureObservability(workerName, jobName, duration, spanId, features);
+};
+const executeAllFailureHandlers = async (params) => {
+    const { workerName, jobVersion, job, error, duration, spanId, features, queueName } = params;
+    recordFailureObservabilityAndMetrics({
+        workerName,
+        jobVersion,
+        jobName: job.name,
+        duration,
+        spanId,
+        features,
+    });
+    if (features?.deadLetterQueue === true) {
+        await addFailedJobToDeadLetterQueue(workerName, job, error, duration, jobVersion, queueName, features);
+    }
+};
+const handleFailure = async (params) => {
+    const { workerName, jobVersion, job, error, features } = params;
+    logAndRecordFailure(workerName, jobVersion, job, error, features);
+    await executeAllFailureHandlers(params);
+    await executeFailurePlugins(workerName, job, error, features);
+};
+/**
+ * Helper: Create enhanced processor with all features
+ */
+const createEnhancedProcessor = (config) => {
+    return async (job) => {
+        const { name, version, processor, features } = config;
+        const jobVersion = version ?? '1.0.0';
+        ensureCircuitAllowsExecution(name, jobVersion, job.id, features);
+        const beforeOutcome = await runBeforeProcessHooks(name, job, features);
+        if (beforeOutcome.skip) {
+            return { skipped: true, reason: beforeOutcome.reason };
+        }
+        if (beforeOutcome.jobData !== undefined) {
+            job.data = beforeOutcome.jobData;
+        }
+        const startTime = Date.now();
+        let result;
+        let spanId = null;
+        try {
+            spanId = startProcessingSpan(name, jobVersion, job, config.queueName, features);
+            // Process the job
+            result = await processor(job);
+            const duration = Date.now() - startTime;
+            await handleSuccess({
+                workerName: name,
+                jobVersion,
+                job,
+                result,
+                duration,
+                spanId,
+                features,
+            });
+            return result;
+        }
+        catch (err) {
+            const error = err;
+            const duration = Date.now() - startTime;
+            await handleFailure({
+                workerName: name,
+                jobVersion,
+                job,
+                error,
+                duration,
+                spanId,
+                features,
+                queueName: config.queueName,
+            });
+            throw error;
+        }
+    };
+};
+const requireInfrastructure = (value, message) => {
+    if (value === null || value === undefined) {
+        throw ErrorFactory.createConfigError(message);
+    }
+    return value;
+};
+const resolveEnvString = (envKey, fallback) => {
+    if (!envKey)
+        return fallback;
+    return Env.get(envKey, fallback);
+};
+const resolveEnvInt = (envKey, fallback) => {
+    if (!envKey)
+        return fallback;
+    return Env.getInt(envKey, fallback);
+};
+const isRedisEnvConfig = (config) => config.env === true;
+const requireRedisHost = (host, context) => {
+    if (!host) {
+        throw ErrorFactory.createConfigError(`${context}.host is required`);
+    }
+    return host;
+};
+const resolveRedisFallbacks = () => {
+    const queueRedis = queueConfig.drivers.redis;
+    return {
+        host: queueRedis?.driver === 'redis' ? queueRedis.host : Env.get('REDIS_HOST', '127.0.0.1'),
+        port: queueRedis?.driver === 'redis' ? queueRedis.port : Env.getInt('REDIS_PORT', 6379),
+        db: queueRedis?.driver === 'redis' ? queueRedis.database : Env.getInt('REDIS_DB', 0),
+        password: queueRedis?.driver === 'redis' ? (queueRedis.password ?? '') : Env.get('REDIS_PASSWORD', ''),
+    };
+};
+const resolveRedisConfigFromEnv = (config, context) => {
+    const fallback = resolveRedisFallbacks();
+    const host = requireRedisHost(resolveEnvString(config.host ?? 'REDIS_HOST', fallback.host), context);
+    const port = resolveEnvInt(config.port ?? 'REDIS_PORT', fallback.port);
+    const db = resolveEnvInt(config.db ?? 'REDIS_DB', fallback.db);
+    const password = resolveEnvString(config.password ?? 'REDIS_PASSWORD', fallback.password);
+    return {
+        host,
+        port,
+        db,
+        password: password || undefined,
+    };
+};
+const resolveRedisConfigFromDirect = (config, context) => ({
+    host: requireRedisHost(config.host, context),
+    port: config.port,
+    db: config.db,
+    password: config.password ?? Env.get('REDIS_PASSWORD', undefined),
+});
+const resolveRedisConfig = (config, context) => isRedisEnvConfig(config)
+    ? resolveRedisConfigFromEnv(config, context)
+    : resolveRedisConfigFromDirect(config, context);
+const resolveRedisConfigWithFallback = (primary, fallback, errorMessage, context) => {
+    const selected = primary ?? fallback;
+    if (!selected) {
+        throw ErrorFactory.createConfigError(errorMessage);
+    }
+    return resolveRedisConfig(selected, context);
+};
+const normalizeEnvValue = (value) => {
+    if (!value)
+        return undefined;
+    const trimmed = value.trim();
+    return trimmed.length > 0 ? trimmed : undefined;
+};
+const normalizeAppName = (value) => {
+    const normalized = (value ?? '').trim().replaceAll(/\s+/g, '_');
+    return normalized.length > 0 ? normalized : 'zintrust';
+};
+const resolveDefaultRedisKeyPrefix = () => 'worker_' + normalizeAppName(appConfig.prefix);
+const resolveDefaultPersistenceTable = () => normalizeEnvValue(Env.get('WORKER_PERSISTENCE_TABLE', 'zintrust_workers')) ?? 'zintrust_workers';
+const resolveDefaultPersistenceConnection = () => normalizeEnvValue(Env.get('WORKER_PERSISTENCE_DB_CONNECTION', 'default')) ?? 'default';
+const resolveAutoStart = (config) => {
+    // If explicitly set in config (not null/undefined), use that
+    if (config.autoStart !== undefined && config.autoStart !== null) {
+        return config.autoStart;
+    }
+    // Otherwise, use environment variable
+    return Env.getBool('WORKER_AUTO_START', false);
+};
+const normalizeExplicitPersistence = (persistence) => {
+    if (persistence.driver === 'memory')
+        return { driver: 'memory' };
+    if (persistence.driver === 'redis') {
+        return {
+            driver: 'redis',
+            redis: persistence.redis,
+            keyPrefix: persistence.keyPrefix ??
+                normalizeEnvValue(Env.get('WORKER_PERSISTENCE_REDIS_KEY_PREFIX', '')) ??
+                resolveDefaultRedisKeyPrefix(),
+        };
+    }
+    const clientIsConnection = typeof persistence.client === 'string';
+    const clientConnection = clientIsConnection ? persistence.client : undefined;
+    const resolvedConnection = persistence.connection ??
+        clientConnection ??
+        normalizeEnvValue(Env.get('WORKER_PERSISTENCE_DB_CONNECTION', 'default')) ??
+        resolveDefaultPersistenceConnection();
+    return {
+        driver: 'database',
+        client: clientIsConnection ? undefined : persistence.client,
+        connection: resolvedConnection,
+        table: persistence.table ??
+            normalizeEnvValue(Env.get('WORKER_PERSISTENCE_TABLE', 'zintrust_workers')) ??
+            resolveDefaultPersistenceTable(),
+    };
+};
+const resolvePersistenceConfig = (config) => {
+    const explicit = config.infrastructure?.persistence;
+    if (explicit)
+        return normalizeExplicitPersistence(explicit);
+    const driver = normalizeEnvValue(Env.get('WORKER_PERSISTENCE_DRIVER', ''))?.toLowerCase();
+    if (!driver)
+        return undefined;
+    if (driver === 'memory')
+        return { driver: 'memory' };
+    if (driver === 'redis') {
+        const keyPrefix = normalizeEnvValue(Env.get('WORKER_PERSISTENCE_REDIS_KEY_PREFIX', ''));
+        return {
+            driver: 'redis',
+            redis: { env: true },
+            keyPrefix: `${keyPrefix}_worker_${appConfig.prefix}`,
+        };
+    }
+    if (driver === 'db' || driver === 'database') {
+        return {
+            driver: 'database',
+            connection: resolveDefaultPersistenceConnection(),
+            table: resolveDefaultPersistenceTable(),
+        };
+    }
+    throw ErrorFactory.createConfigError('WORKER_PERSISTENCE_DRIVER must be one of memory, redis, or database');
+};
+const resolveDbClientFromEnv = async (connectionName = 'default') => {
+    const connect = async () => await useEnsureDbConnected(undefined, connectionName);
+    try {
+        return await connect();
+    }
+    catch (error) {
+        Logger.error('Worker persistence failed to resolve database connection', error);
+    }
+    try {
+        registerDatabasesFromRuntimeConfig(databaseConfig);
+        return await connect();
+    }
+    catch (error) {
+        Logger.error('Worker persistence failed after registering runtime databases', error);
+        throw ErrorFactory.createConfigError(`Worker persistence requires a database client. Register connection '${connectionName}' or pass infrastructure.persistence.client.`);
+    }
+};
+const resolveWorkerStore = async (config) => {
+    const persistence = resolvePersistenceConfig(config);
+    if (!persistence)
+        return workerStore;
+    let next;
+    if (persistence.driver === 'memory') {
+        next = InMemoryWorkerStore.create();
+    }
+    else if (persistence.driver === 'redis') {
+        const redisConfig = resolveRedisConfigWithFallback(persistence.redis, config.infrastructure?.redis, 'Worker persistence requires redis config (persistence.redis or infrastructure.redis)', 'infrastructure.persistence.redis');
+        const client = createRedisConnection(redisConfig);
+        next = RedisWorkerStore.create(client, persistence.keyPrefix ?? resolveDefaultRedisKeyPrefix());
+    }
+    else if (persistence.driver === 'database') {
+        const explicitConnection = typeof persistence.client === 'string' ? persistence.client : persistence.connection;
+        const client = typeof persistence.client === 'string'
+            ? await resolveDbClientFromEnv(explicitConnection)
+            : (persistence.client ?? (await resolveDbClientFromEnv(explicitConnection)));
+        next = DbWorkerStore.create(client, persistence.table);
+    }
+    else {
+        next = InMemoryWorkerStore.create();
+    }
+    await next.init();
+    return next;
+};
+// Store instance cache to reuse connections
+const storeInstanceCache = new Map();
+/**
+ * Generate cache key for persistence configuration
+ */
+const generateCacheKey = (persistence) => {
+    return JSON.stringify({
+        driver: persistence.driver,
+        redis: 'redis' in persistence ? persistence.redis : undefined,
+        keyPrefix: 'keyPrefix' in persistence ? persistence.keyPrefix : undefined,
+        connection: 'connection' in persistence ? persistence.connection : undefined,
+        table: 'table' in persistence ? persistence.table : undefined,
+    });
+};
+/**
+ * Create new store instance based on persistence configuration
+ */
+const createWorkerStore = async (persistence) => {
+    if (persistence.driver === 'memory') {
+        if (workerStoreConfigured && workerStoreConfig?.driver === 'memory') {
+            return workerStore;
+        }
+        return InMemoryWorkerStore.create();
+    }
+    if (persistence.driver === 'redis') {
+        const redisConfig = resolveRedisConfigWithFallback(persistence.redis ?? { env: true }, undefined, 'Worker persistence requires redis config (persistence.redis or REDIS_* env values)', 'persistence.redis');
+        const client = createRedisConnection(redisConfig);
+        return RedisWorkerStore.create(client, persistence.keyPrefix ?? resolveDefaultRedisKeyPrefix());
+    }
+    // Database driver
+    const explicitConnection = typeof persistence.client === 'string' ? persistence.client : persistence.connection;
+    const client = typeof persistence.client === 'string'
+        ? await resolveDbClientFromEnv(explicitConnection)
+        : (persistence.client ?? (await resolveDbClientFromEnv(explicitConnection)));
+    return DbWorkerStore.create(client, persistence.table);
+};
+const resolveWorkerStoreForPersistence = async (persistence) => {
+    const cacheKey = generateCacheKey(persistence);
+    // Return cached instance if available
+    const cached = storeInstanceCache.get(cacheKey);
+    if (cached) {
+        return cached;
+    }
+    // Create new store instance
+    const store = await createWorkerStore(persistence);
+    await store.init();
+    // Cache the store instance for reuse
+    storeInstanceCache.set(cacheKey, store);
+    return store;
+};
+const getPersistedRecord = async (name, persistenceOverride) => {
+    if (!persistenceOverride) {
+        await ensureWorkerStoreConfigured();
+        return workerStore.get(name);
+    }
+    const store = await resolveWorkerStoreForPersistence(persistenceOverride);
+    return store.get(name);
+};
+const ensureWorkerStoreConfigured = async () => {
+    if (workerStoreConfigured)
+        return;
+    const bootstrapConfig = buildPersistenceBootstrapConfig();
+    const persistence = resolvePersistenceConfig(bootstrapConfig);
+    if (!persistence)
+        return;
+    workerStore = await resolveWorkerStore(bootstrapConfig);
+    workerStoreConfigured = true;
+    workerStoreConfig = persistence;
+};
+const buildWorkerRecord = (config, status) => {
+    const now = new Date();
+    const decodedProcessorPath = config.processorPath
+        ? decodeProcessorPathEntities(config.processorPath)
+        : null;
+    return {
+        name: config.name,
+        queueName: config.queueName,
+        version: config.version ?? '1.0.0',
+        status,
+        autoStart: resolveAutoStart(config),
+        concurrency: config.options?.concurrency ?? 1,
+        region: config.datacenter?.primaryRegion ?? null,
+        processorPath: decodedProcessorPath,
+        features: config.features ? { ...config.features } : null,
+        infrastructure: config.infrastructure ? { ...config.infrastructure } : null,
+        datacenter: config.datacenter ? { ...config.datacenter } : null,
+        createdAt: now,
+        updatedAt: now,
+        lastHealthCheck: undefined,
+        lastError: undefined,
+        connectionState: undefined,
+    };
+};
+const buildDefaultAutoScalerConfig = () => ({
+    enabled: workersConfig.autoScaling.enabled,
+    checkInterval: workersConfig.autoScaling.interval,
+    scalingPolicies: new Map(),
+    costOptimization: {
+        enabled: workersConfig.costOptimization.enabled,
+        maxCostPerHour: 0,
+        preferSpotInstances: workersConfig.costOptimization.spotInstances,
+        offPeakSchedule: {
+            start: workersConfig.autoScaling.offPeakSchedule.split('-')[0] ?? '22:00',
+            end: workersConfig.autoScaling.offPeakSchedule.split('-')[1] ?? '06:00',
+            timezone: 'UTC',
+            reductionPercentage: Math.round(workersConfig.autoScaling.offPeakReduction * 100),
+        },
+        budgetAlerts: {
+            dailyLimit: 0,
+            weeklyLimit: 0,
+            monthlyLimit: 0,
+        },
+    },
+});
+const resolveOffPeakSchedule = (input, defaults) => {
+    const fallback = defaults.costOptimization.offPeakSchedule ?? {
+        start: '22:00',
+        end: '06:00',
+        timezone: 'UTC',
+        reductionPercentage: 0,
+    };
+    const override = input?.costOptimization?.offPeakSchedule;
+    const schedule = { ...fallback };
+    if (override) {
+        Object.assign(schedule, override);
+    }
+    return schedule;
+};
+const resolveCostOptimization = (input, defaults) => ({
+    ...defaults.costOptimization,
+    ...input?.costOptimization,
+    offPeakSchedule: resolveOffPeakSchedule(input, defaults),
+    budgetAlerts: {
+        ...defaults.costOptimization.budgetAlerts,
+        ...input?.costOptimization?.budgetAlerts,
+    },
+});
+const resolveAutoScalerConfig = (input) => {
+    const defaults = buildDefaultAutoScalerConfig();
+    if (!input)
+        return defaults;
+    return {
+        ...defaults,
+        ...input,
+        costOptimization: resolveCostOptimization(input, defaults),
+    };
+};
+const resolveWorkerOptions = (config, autoStart) => {
+    const options = config.options ? { ...config.options } : {};
+    if (options.prefix === undefined) {
+        options.prefix = getBullMQSafeQueueName();
+    }
+    if (options.autorun === undefined) {
+        options.autorun = autoStart;
+    }
+    if (options.connection)
+        return options;
+    const redisConfig = resolveRedisConfigWithFallback(config.infrastructure?.redis, undefined, 'Worker requires a connection. Provide options.connection or infrastructure.redis config', 'infrastructure.redis');
+    return {
+        ...options,
+        connection: {
+            host: redisConfig.host,
+            port: redisConfig.port,
+            db: redisConfig.db,
+            password: redisConfig.password,
+        },
+    };
+};
+const buildDefaultObservabilityConfig = () => ({
+    prometheus: {
+        enabled: workersConfig.observability.prometheus.enabled,
+        port: workersConfig.observability.prometheus.port,
+    },
+    openTelemetry: {
+        enabled: workersConfig.observability.opentelemetry.enabled,
+        serviceName: 'zintrust-workers',
+        exporterUrl: workersConfig.observability.opentelemetry.endpoint,
+    },
+    datadog: {
+        enabled: workersConfig.observability.datadog.enabled,
+        tags: workersConfig.observability.datadog.apiKey
+            ? [`apiKey:${workersConfig.observability.datadog.apiKey}`]
+            : undefined,
+    },
+});
+const resolveObservabilityConfig = (input) => {
+    const defaults = buildDefaultObservabilityConfig();
+    if (!input)
+        return defaults;
+    const enabledOverride = 'enabled' in input ? input.enabled : undefined;
+    const prometheus = { ...defaults.prometheus };
+    if (input.prometheus) {
+        Object.assign(prometheus, input.prometheus);
+    }
+    const openTelemetry = { ...defaults.openTelemetry };
+    if (input.openTelemetry) {
+        Object.assign(openTelemetry, input.openTelemetry);
+    }
+    const datadog = { ...defaults.datadog };
+    if (input.datadog) {
+        Object.assign(datadog, input.datadog);
+    }
+    if (enabledOverride === false) {
+        prometheus.enabled = false;
+        openTelemetry.enabled = false;
+        datadog.enabled = false;
+    }
+    else if (enabledOverride === true) {
+        prometheus.enabled = true;
+        openTelemetry.enabled = true;
+        datadog.enabled = true;
+    }
+    if (!openTelemetry.serviceName) {
+        openTelemetry.serviceName = defaults.openTelemetry.serviceName;
+    }
+    return { prometheus, openTelemetry, datadog };
+};
+const initializeClustering = (config) => {
+    if (clusteringInitialized || !(config.features?.clustering ?? false))
+        return;
+    const redisConfig = resolveRedisConfigWithFallback(config.infrastructure?.redis, undefined, 'ClusterLock requires infrastructure.redis config', 'infrastructure.redis');
+    ClusterLock.initialize(redisConfig);
+    clusteringInitialized = true;
+};
+const initializeMetrics = (config) => {
+    if (metricsInitialized || !(config.features?.metrics ?? false))
+        return;
+    const redisConfig = resolveRedisConfigWithFallback(config.infrastructure?.redis, undefined, 'WorkerMetrics requires infrastructure.redis config', 'infrastructure.redis');
+    WorkerMetrics.initialize(redisConfig);
+    metricsInitialized = true;
+};
+const initializeAutoScaling = (config) => {
+    if (autoScalingInitialized || !(config.features?.autoScaling ?? false))
+        return;
+    const autoScalerConfig = resolveAutoScalerConfig(config.infrastructure?.autoScaler);
+    AutoScaler.initialize(autoScalerConfig);
+    autoScalingInitialized = true;
+};
+const initializeCircuitBreaker = (config, version) => {
+    if (!(config.features?.circuitBreaker ?? false))
+        return;
+    CircuitBreaker.initialize(config.name, version);
+};
+const initializeDeadLetterQueue = (config) => {
+    if (deadLetterQueueInitialized || !(config.features?.deadLetterQueue ?? false))
+        return;
+    const dlqConfig = requireInfrastructure(config.infrastructure?.deadLetterQueue, 'DeadLetterQueue requires infrastructure.deadLetterQueue config');
+    const dlqRedisConfig = resolveRedisConfigWithFallback(dlqConfig.redis, config.infrastructure?.redis, 'DeadLetterQueue requires infrastructure.deadLetterQueue.redis or infrastructure.redis config', 'infrastructure.deadLetterQueue.redis');
+    DeadLetterQueue.initialize(dlqRedisConfig, dlqConfig.policy);
+    deadLetterQueueInitialized = true;
+};
+const initializeResourceMonitoring = (config) => {
+    if (resourceMonitoringInitialized || !(config.features?.resourceMonitoring ?? false))
+        return;
+    ResourceMonitor.initialize();
+    ResourceMonitor.start();
+    resourceMonitoringInitialized = true;
+};
+const initializeCompliance = (config) => {
+    if (complianceInitialized || !(config.features?.compliance ?? false))
+        return;
+    const complianceConfig = requireInfrastructure(config.infrastructure?.compliance, 'ComplianceManager requires infrastructure.compliance config');
+    const complianceRedisConfig = resolveRedisConfigWithFallback(complianceConfig.redis, config.infrastructure?.redis, 'ComplianceManager requires infrastructure.compliance.redis or infrastructure.redis config', 'infrastructure.compliance.redis');
+    ComplianceManager.initialize(complianceRedisConfig, complianceConfig.config);
+    complianceInitialized = true;
+};
+const initializeObservability = async (config) => {
+    if (observabilityInitialized || !(config.features?.observability ?? false))
+        return;
+    const observabilityConfig = resolveObservabilityConfig(config.infrastructure?.observability);
+    await Observability.initialize(observabilityConfig);
+    observabilityInitialized = true;
+};
+const initializeVersioning = (config, version) => {
+    if (!(config.features?.versioning ?? false))
+        return;
+    WorkerVersioning.register({
+        workerName: config.name,
+        version: WorkerVersioning.parse(version),
+        changelog: 'Initial version',
+    });
+};
+const initializeDatacenter = (config) => {
+    if (!(config.features?.datacenterOrchestration ?? false) || !config.datacenter)
+        return;
+    DatacenterOrchestrator.placeWorker({
+        workerName: config.name,
+        primaryRegion: config.datacenter.primaryRegion,
+        secondaryRegions: config.datacenter.secondaryRegions ?? [],
+        replicationStrategy: 'active-passive',
+        affinityRules: {
+            preferLocal: config.datacenter.affinityRules?.preferLocal ?? true,
+            maxLatency: config.datacenter.affinityRules?.maxLatency,
+            avoidRegions: config.datacenter.affinityRules?.avoidRegions,
+        },
+    });
+};
+const setupWorkerEventListeners = (worker, workerName, workerVersion, features) => {
+    worker.on('completed', (job) => {
+        Logger.debug(`Job completed: ${workerName}`, { jobId: job.id });
+        if (features?.observability === true) {
+            Observability.incrementCounter('worker.jobs.completed', 1, {
+                worker: workerName,
+                version: workerVersion,
+            });
+        }
+    });
+    worker.on('failed', (job, error) => {
+        Logger.error(`Job failed: ${workerName}`, { error, jobId: job?.id }, 'workers');
+        if (features?.observability === true) {
+            Observability.incrementCounter('worker.jobs.failed', 1, {
+                worker: workerName,
+                version: workerVersion,
+            });
+        }
+    });
+    worker.on('error', (error) => {
+        Logger.error(`Worker error: ${workerName}`, error);
+    });
+};
+const registerWorkerInstance = (params) => {
+    const { worker, config, workerVersion, queueName, options, autoStart } = params;
+    WorkerRegistry.register({
+        name: config.name,
+        config: {},
+        version: workerVersion,
+        region: config.datacenter?.primaryRegion,
+        queues: [queueName],
+        factory: async () => {
+            await Promise.resolve();
+            return {
+                metadata: {
+                    name: config.name,
+                    status: autoStart ? 'running' : 'stopped',
+                    version: workerVersion,
+                    region: config.datacenter?.primaryRegion ?? 'unknown',
+                    queueName,
+                    concurrency: options?.concurrency ?? 1,
+                    startedAt: new Date(),
+                    stoppedAt: null,
+                    lastProcessedAt: null,
+                    restartCount: 0,
+                    processedCount: 0,
+                    errorCount: 0,
+                    lockKey: null,
+                    priority: 0,
+                    memoryUsage: 0,
+                    cpuUsage: 0,
+                    circuitState: 'closed',
+                    queues: [queueName],
+                    plugins: [],
+                    datacenter: config.datacenter?.primaryRegion ?? 'unknown',
+                    canaryPercentage: 0,
+                    config: {},
+                },
+                instance: worker,
+                start: () => {
+                    if (!autoStart) {
+                        worker.run().catch((error) => {
+                            Logger.error(`Failed to start worker "${config.name}"`, error);
+                        });
+                    }
+                },
+                stop: async () => worker.close(),
+                drain: async () => worker.close(),
+                sleep: async () => worker.pause(),
+                wakeup: () => {
+                    worker.resume();
+                },
+                getStatus: () => 'running',
+                getHealth: () => 'green',
+            };
+        },
+    });
+};
+const initializeWorkerFeatures = async (config, workerVersion) => {
+    initializeClustering(config);
+    initializeMetrics(config);
+    initializeAutoScaling(config);
+    initializeCircuitBreaker(config, workerVersion);
+    initializeDeadLetterQueue(config);
+    initializeResourceMonitoring(config);
+    initializeCompliance(config);
+    await initializeObservability(config);
+    initializeVersioning(config, workerVersion);
+    initializeDatacenter(config);
+};
+/**
+ * Worker Factory - Sealed namespace
+ */
+export const WorkerFactory = Object.freeze({
+    registerProcessor,
+    registerProcessors,
+    registerProcessorPaths,
+    registerProcessorResolver,
+    resolveProcessorPath,
+    /**
+     * Create new worker with full setup
+     */
+    async create(config) {
+        const { name, version, queueName, features } = config;
+        const workerVersion = version ?? '1.0.0';
+        const autoStart = resolveAutoStart(config);
+        if (workers.has(name)) {
+            throw ErrorFactory.createWorkerError(`Worker "${name}" already exists`);
+        }
+        // Resolve the correct store for this worker configuration
+        const store = await getStoreForWorker(config);
+        // Save initial status as "creating"
+        await store.save(buildWorkerRecord(config, WorkerCreationStatus.CREATING));
+        try {
+            await initializeWorkerFeatures(config, workerVersion);
+            // Update status to "connecting"
+            await store.update(name, {
+                status: WorkerCreationStatus.CONNECTING,
+                updatedAt: new Date(),
+            });
+            // Create enhanced processor
+            const enhancedProcessor = createEnhancedProcessor(config);
+            // Create BullMQ worker
+            const resolvedOptions = resolveWorkerOptions(config, autoStart);
+            const worker = new Worker(queueName, enhancedProcessor, resolvedOptions);
+            setupWorkerEventListeners(worker, name, workerVersion, features);
+            // Update status to "starting"
+            await store.update(name, {
+                status: WorkerCreationStatus.STARTING,
+                updatedAt: new Date(),
+            });
+            const timeoutMs = Env.getInt('WORKER_CONNECTION_TIMEOUT', 5000);
+            // Wait for actual connection and health verification
+            await waitForWorkerConnection(worker, name, queueName, timeoutMs);
+            // Update status to "running" only after successful connection
+            await store.update(name, {
+                status: WorkerCreationStatus.RUNNING,
+                updatedAt: new Date(),
+            });
+            // Store worker instance
+            const instance = {
+                worker,
+                config,
+                startedAt: new Date(),
+                status: WorkerCreationStatus.RUNNING,
+                connectionState: 'connected',
+            };
+            workers.set(name, instance);
+            registerWorkerInstance({
+                worker,
+                config,
+                workerVersion,
+                queueName,
+                options: resolvedOptions,
+                autoStart,
+            });
+            if (autoStart) {
+                await WorkerRegistry.start(name, workerVersion);
+            }
+            // Execute afterStart hooks
+            if (features?.plugins === true) {
+                await PluginManager.executeHook('afterStart', {
+                    workerName: name,
+                    timestamp: new Date(),
+                });
+            }
+            // Start health monitoring for the worker
+            startHealthMonitoring(name, worker, queueName);
+            Logger.info(`Worker created: ${name}@${workerVersion}`, {
+                queueName,
+                features: Object.keys(features ?? {}).filter((k) => features?.[k] === true),
+            });
+            return worker;
+        }
+        catch (error) {
+            // Handle failure - update status to "failed"
+            // Re-resolve store in case of error to be safe
+            const failStore = await getStoreForWorker(config);
+            await failStore.update(name, {
+                status: WorkerCreationStatus.FAILED,
+                updatedAt: new Date(),
+                lastError: error.message,
+            });
+            Logger.error(`Worker creation failed: ${name}`, error);
+            throw error;
+        }
+    },
+    /**
+     * Get worker instance
+     */
+    get(name) {
+        const instance = workers.get(name);
+        return instance ? { ...instance } : null;
+    },
+    /**
+     * Update worker status directly (used by HealthMonitor)
+     */
+    async updateStatus(name, status, error) {
+        const instance = workers.get(name);
+        if (instance) {
+            instance.status = status;
+        }
+        try {
+            const store = await getStoreForWorker(instance?.config ?? {
+                name,
+                queueName: 'unknown',
+                processor: async () => {
+                    return Promise.resolve(); //NOSONAR
+                },
+            });
+            const errorMessage = typeof error === 'string' ? error : error?.message;
+            await store.update(name, {
+                status: status,
+                updatedAt: new Date(),
+                lastError: errorMessage,
+            });
+        }
+        catch (err) {
+            Logger.warn(`Failed to update status for ${name} to ${status}`, err);
+        }
+    },
+    /**
+     * Stop worker
+     */
+    async stop(name, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (!instance) {
+            await store.update(name, { status: 'stopped', updatedAt: new Date() });
+            Logger.info(`Worker marked stopped (not running): ${name}`);
+            return;
+        }
+        // Execute beforeStop hooks
+        if (instance.config.features?.plugins === true) {
+            await PluginManager.executeHook('beforeStop', {
+                workerName: name,
+                timestamp: new Date(),
+            });
+        }
+        // Close worker with timeout to prevent hanging
+        const workerClosePromise = instance.worker.close();
+        let timeoutId;
+        const timeoutPromise = new Promise((_, reject) => {
+            // eslint-disable-next-line no-restricted-syntax
+            timeoutId = setTimeout(() => {
+                reject(new Error('Worker close timeout'));
+            }, 5000);
+        });
+        try {
+            await Promise.race([workerClosePromise, timeoutPromise]);
+        }
+        catch (error) {
+            Logger.warn(`Worker "${name}" close failed or timed out, continuing...`, error);
+        }
+        finally {
+            // Always clean up timeout to prevent memory leak
+            if (timeoutId) {
+                clearTimeout(timeoutId);
+                timeoutId = undefined;
+            }
+        }
+        instance.status = WorkerCreationStatus.STOPPED;
+        // Stop health monitoring for this worker
+        HealthMonitor.unregister(name);
+        try {
+            await store.update(name, {
+                status: WorkerCreationStatus.STOPPED,
+                updatedAt: new Date(),
+            });
+            Logger.info(`Worker "${name}" status updated to stopped`);
+        }
+        catch (error) {
+            Logger.error(`Failed to update worker "${name}" status`, error);
+        }
+        await WorkerRegistry.stop(name);
+        // Execute afterStop hooks
+        if (instance.config.features?.plugins === true) {
+            await PluginManager.executeHook('afterStop', {
+                workerName: name,
+                timestamp: new Date(),
+            });
+        }
+        Logger.info(`Worker stopped: ${name}`);
+    },
+    /**
+     * Restart worker
+     */
+    async restart(name, persistenceOverride) {
+        const instance = workers.get(name);
+        if (!instance) {
+            await WorkerFactory.startFromPersisted(name, persistenceOverride);
+            Logger.info(`Worker started from persistence: ${name}`);
+            return;
+        }
+        await WorkerFactory.stop(name, persistenceOverride);
+        const refreshed = workers.get(name);
+        if (!refreshed) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found`);
+        }
+        workers.delete(name);
+        const newWorker = await WorkerFactory.create(refreshed.config);
+        refreshed.worker = newWorker;
+        refreshed.status = WorkerCreationStatus.RUNNING;
+        refreshed.startedAt = new Date();
+        Logger.info(`Worker restarted: ${name}`);
+    },
+    /**
+     * Pause worker
+     */
+    async pause(name, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (instance) {
+            await instance.worker.pause();
+            instance.status = WorkerCreationStatus.STARTING; // Using STARTING as equivalent to sleeping/paused
+        }
+        await store.update(name, {
+            status: WorkerCreationStatus.STARTING,
+            updatedAt: new Date(),
+        });
+        Logger.info(`Worker paused: ${name}`);
+    },
+    /**
+     * Resume worker
+     */
+    async resume(name, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (instance) {
+            instance.worker.resume();
+            instance.status = WorkerCreationStatus.RUNNING;
+        }
+        try {
+            await store.update(name, { status: WorkerCreationStatus.RUNNING, updatedAt: new Date() });
+        }
+        catch (error) {
+            Logger.error('Failed to persist worker resume', error);
+        }
+        Logger.info(`Worker resumed: ${name}`);
+    },
+    /**
+     * Update auto-start for persisted worker
+     */
+    async setAutoStart(name, autoStart, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (instance) {
+            instance.config.autoStart = autoStart;
+        }
+        await store.update(name, { autoStart, updatedAt: new Date() });
+        if (!autoStart)
+            return;
+        const refreshed = workers.get(name);
+        if (refreshed) {
+            if (refreshed.status !== 'running') {
+                await WorkerFactory.start(name, persistenceOverride);
+            }
+            return;
+        }
+        await WorkerFactory.startFromPersisted(name, persistenceOverride);
+    },
+    /**
+     * Update persisted worker record and in-memory config if running.
+     */
+    async update(name, patch, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await getStoreForWorker(instance?.config, persistenceOverride);
+        const current = await store.get(name);
+        if (!current) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found in persistence store`);
+        }
+        const merged = {
+            ...current,
+            ...patch,
+            updatedAt: patch.updatedAt ?? new Date(),
+        };
+        // Use save() which will insert or update appropriately for each store
+        await store.save(merged);
+        // If the worker is running in memory, update its runtime config so restarts use the new config
+        if (instance) {
+            const cfg = instance.config;
+            instance.config = {
+                ...cfg,
+                version: merged.version ?? cfg.version,
+                queueName: merged.queueName ?? cfg.queueName,
+                options: {
+                    ...cfg.options,
+                    concurrency: merged.concurrency ?? cfg.options?.concurrency,
+                },
+                infrastructure: merged.infrastructure ?? cfg.infrastructure,
+                features: merged.features ?? cfg.features,
+                datacenter: merged.datacenter ?? cfg.datacenter,
+            };
+        }
+    },
+    /**
+     * Start worker
+     */
+    async start(name, persistenceOverride) {
+        const instance = workers.get(name);
+        // Even if instance exists, we must validate against the requested driver
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (!instance) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found`);
+        }
+        const version = instance.config.version ?? '1.0.0';
+        await WorkerRegistry.start(name, version);
+        instance.status = WorkerCreationStatus.RUNNING;
+        instance.startedAt = new Date();
+        await store.update(name, { status: WorkerCreationStatus.RUNNING, updatedAt: new Date() });
+        Logger.info(`Worker started: ${name}`);
+    },
+    /**
+     * List all workers
+     */
+    list() {
+        return Array.from(workers.keys());
+    },
+    /**
+     * List all persisted workers
+     */
+    async listPersisted(persistenceOverride, options) {
+        const records = await WorkerFactory.listPersistedRecords(persistenceOverride, options);
+        return records.map((record) => record.name);
+    },
+    async listPersistedRecords(persistenceOverride, options) {
+        if (!persistenceOverride) {
+            await ensureWorkerStoreConfigured();
+            return workerStore.list(options);
+        }
+        const store = await resolveWorkerStoreForPersistence(persistenceOverride);
+        return store.list(options);
+    },
+    /**
+     * Start a worker from persisted storage when it is not registered.
+     */
+    async startFromPersisted(name, persistenceOverride) {
+        const record = await getPersistedRecord(name, persistenceOverride);
+        if (!record) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found in persistence store`);
+        }
+        let processor = await resolveProcessor(name);
+        if (!processor && record.processorPath) {
+            try {
+                processor = await resolveProcessorFromPath(record.processorPath);
+            }
+            catch (error) {
+                Logger.error(`Failed to resolve processor module for "${name}"`, error);
+            }
+        }
+        if (!processor) {
+            throw ErrorFactory.createConfigError(`Worker "${name}" processor is not registered or resolvable. Register the processor at startup or persist a processorPath.`);
+        }
+        await WorkerFactory.create({
+            name: record.name,
+            queueName: record.queueName,
+            version: record.version ?? undefined,
+            processor,
+            processorPath: record.processorPath ?? undefined,
+            autoStart: true, // Override to true when manually starting
+            options: { concurrency: record.concurrency },
+            infrastructure: record.infrastructure,
+            features: record.features,
+            datacenter: record.datacenter,
+        });
+    },
+    /**
+     * Get persisted worker record
+     */
+    async getPersisted(name, persistenceOverride) {
+        const instance = workers.get(name);
+        const store = await getStoreForWorker(instance?.config, persistenceOverride);
+        return store.get(name);
+    },
+    /**
+     * Remove worker
+     */
+    async remove(name, persistenceOverride) {
+        const instance = workers.get(name);
+        // Validate that worker exists in the store we are trying to remove from
+        const store = await validateAndGetStore(name, instance?.config, persistenceOverride);
+        if (instance) {
+            await WorkerFactory.stop(name, persistenceOverride);
+            const registry = WorkerRegistry;
+            registry.unregister?.(name);
+            AutoScaler.clearHistory(name);
+            ResourceMonitor.clearHistory(name);
+            CircuitBreaker.deleteWorker(name);
+            CanaryController.purge(name);
+            WorkerVersioning.clear(name);
+            DatacenterOrchestrator.removeWorker(name);
+            await Observability.clearWorkerMetrics(name);
+            // Stop health monitoring for this worker
+            HealthMonitor.unregister(name);
+            workers.delete(name);
+        }
+        await store.remove(name);
+        Logger.info(`Worker removed: ${name}`);
+    },
+    /**
+     * Get worker metrics
+     */
+    async getMetrics(name) {
+        const instance = workers.get(name);
+        if (!instance) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found`);
+        }
+        if (instance.config.features?.metrics === undefined || !instance.config.features?.metrics) {
+            return null;
+        }
+        const now = Date.now();
+        const oneHourAgo = now - 3600 * 1000;
+        const metrics = await WorkerMetrics.aggregate({
+            workerName: name,
+            metricType: 'processed',
+            granularity: 'hourly',
+            startDate: new Date(oneHourAgo),
+            endDate: new Date(now),
+        });
+        return metrics;
+    },
+    /**
+     * Get worker health
+     */
+    async getHealth(name) {
+        const instance = workers.get(name);
+        if (!instance) {
+            throw ErrorFactory.createNotFoundError(`Worker "${name}" not found`);
+        }
+        if (!(instance.config.features?.metrics ?? false)) {
+            return { status: 'unknown' };
+        }
+        const health = await WorkerMetrics.getLatestHealth(name);
+        return health;
+    },
+    /**
+     * Shutdown all workers
+     */
+    async shutdown() {
+        Logger.info('WorkerFactory shutting down...');
+        const workerNames = Array.from(workers.keys());
+        await Promise.all(workerNames.map(async (name) => WorkerFactory.stop(name)));
+        // Shutdown all modules
+        ResourceMonitor.stop();
+        await WorkerMetrics.shutdown();
+        await MultiQueueWorker.shutdown();
+        await ComplianceManager.shutdown();
+        await PriorityQueue.shutdown();
+        HealthMonitor.shutdown();
+        AutoScaler.stop();
+        ClusterLock.shutdown();
+        WorkerVersioning.shutdown();
+        CanaryController.shutdown();
+        DatacenterOrchestrator.shutdown();
+        PluginManager.shutdown();
+        Observability.shutdown();
+        await DeadLetterQueue.shutdown();
+        CircuitBreaker.shutdown();
+        workers.clear();
+        Logger.info('WorkerFactory shutdown complete');
+    },
+});
+// Graceful shutdown handled by WorkerShutdown