@zintrust/workers 0.1.27

package/dist/CanaryController.js

@@ -0,0 +1,380 @@
+/**
+ * Canary Deployment Controller
+ * Gradual rollout with traffic percentage control and automatic rollback
+ * Sealed namespace for immutability
+ */
+import { ErrorFactory, Logger } from '@zintrust/core';
+import { CircuitBreaker } from './CircuitBreaker';
+// Internal state
+const canaryDeployments = new Map();
+const canaryTimers = new Map();
+const MAX_HISTORY = 1000;
+/**
+ * Helper: Calculate error rate
+ */
+const calculateErrorRate = (processed, errors) => {
+    if (processed === 0)
+        return 0;
+    return errors / processed;
+};
+/**
+ * Helper: Calculate success rate
+ */
+const calculateSuccessRate = (processed, errors) => {
+    if (processed === 0)
+        return 1;
+    return (processed - errors) / processed;
+};
+/**
+ * Helper: Should rollback based on metrics
+ */
+const shouldRollback = (deployment) => {
+    const { config, metrics } = deployment;
+    const { canaryVersion } = metrics;
+    // Check error threshold
+    const errorRate = calculateErrorRate(canaryVersion.processed, canaryVersion.errors);
+    if (errorRate > config.errorThreshold) {
+        return {
+            should: true,
+            reason: `Error rate ${(errorRate * 100).toFixed(2)}% exceeds threshold ${(config.errorThreshold * 100).toFixed(2)}%`,
+        };
+    }
+    // Check success rate
+    const successRate = calculateSuccessRate(canaryVersion.processed, canaryVersion.errors);
+    if (successRate < config.minSuccessRate) {
+        return {
+            should: true,
+            reason: `Success rate ${(successRate * 100).toFixed(2)}% below minimum ${(config.minSuccessRate * 100).toFixed(2)}%`,
+        };
+    }
+    // Check latency threshold
+    if (canaryVersion.avgLatency > config.latencyThreshold) {
+        return {
+            should: true,
+            reason: `P95 latency ${canaryVersion.avgLatency}ms exceeds threshold ${config.latencyThreshold}ms`,
+        };
+    }
+    return { should: false };
+};
+/**
+ * Helper: Increment traffic
+ */
+const incrementTraffic = (workerName) => {
+    const deployment = canaryDeployments.get(workerName);
+    if (!deployment) {
+        Logger.error('Canary deployment not found', { workerName });
+        return;
+    }
+    const { config } = deployment;
+    // Check if we should rollback
+    const rollbackCheck = shouldRollback(deployment);
+    if (rollbackCheck.should && config.autoRollback) {
+        Logger.warn('Auto-rollback triggered', {
+            workerName,
+            reason: rollbackCheck.reason,
+        });
+        CanaryController.rollback(workerName, rollbackCheck.reason ?? '');
+        return;
+    }
+    // Increment traffic
+    const newTrafficPercent = Math.min(deployment.currentTrafficPercent + config.incrementPercent, config.targetTrafficPercent);
+    deployment.currentTrafficPercent = newTrafficPercent;
+    // Record history
+    appendHistory(deployment, {
+        timestamp: new Date(),
+        trafficPercent: newTrafficPercent,
+        stage: deployment.stage,
+        metrics: { ...deployment.metrics },
+        decision: `Traffic increased to ${newTrafficPercent}%`,
+    });
+    Logger.info('Canary traffic incremented', {
+        workerName,
+        trafficPercent: newTrafficPercent,
+        targetPercent: config.targetTrafficPercent,
+    });
+    // Check if we've reached the target
+    if (newTrafficPercent >= config.targetTrafficPercent) {
+        deployment.stage = 'monitoring';
+        // Wait for final monitoring period
+        const existingCompleteTimer = canaryTimers.get(`${workerName}:complete`);
+        if (existingCompleteTimer) {
+            clearTimeout(existingCompleteTimer);
+            canaryTimers.delete(`${workerName}:complete`);
+        }
+        // eslint-disable-next-line no-restricted-syntax
+        const timer = setTimeout(() => {
+            CanaryController.complete(workerName);
+        }, config.monitoringDuration * 1000);
+        canaryTimers.set(`${workerName}:complete`, timer);
+    }
+    else {
+        // Schedule next increment
+        const existingTimer = canaryTimers.get(workerName);
+        if (existingTimer) {
+            clearTimeout(existingTimer);
+            canaryTimers.delete(workerName);
+        }
+        // eslint-disable-next-line no-restricted-syntax
+        const timer = setTimeout(() => {
+            incrementTraffic(workerName);
+        }, config.incrementInterval * 1000);
+        canaryTimers.set(workerName, timer);
+    }
+};
+const appendHistory = (deployment, entry) => {
+    deployment.history.push(entry);
+    if (deployment.history.length > MAX_HISTORY) {
+        deployment.history.shift();
+    }
+};
+/**
+ * Canary Deployment Controller - Sealed namespace
+ */
+export const CanaryController = Object.freeze({
+    /**
+     * Start canary deployment
+     */
+    start(config) {
+        const { workerName } = config;
+        if (canaryDeployments.has(workerName)) {
+            throw ErrorFactory.createGeneralError(`Canary deployment already in progress for "${workerName}"`);
+        }
+        // Validate config
+        if (config.initialTrafficPercent < 0 || config.initialTrafficPercent > 100) {
+            throw ErrorFactory.createValidationError('Initial traffic percent must be between 0 and 100');
+        }
+        if (config.targetTrafficPercent < config.initialTrafficPercent ||
+            config.targetTrafficPercent > 100) {
+            throw ErrorFactory.createValidationError('Target traffic percent must be >= initial and <= 100');
+        }
+        // Create deployment
+        const deployment = {
+            config,
+            currentTrafficPercent: config.initialTrafficPercent,
+            stage: 'initial',
+            startedAt: new Date(),
+            metrics: {
+                currentVersion: { processed: 0, errors: 0, avgLatency: 0 },
+                canaryVersion: { processed: 0, errors: 0, avgLatency: 0 },
+            },
+            history: [],
+        };
+        canaryDeployments.set(workerName, deployment);
+        Logger.info('Canary deployment started', {
+            workerName,
+            currentVersion: config.currentVersion,
+            canaryVersion: config.canaryVersion,
+            initialTraffic: config.initialTrafficPercent,
+        });
+        // Start ramping up
+        deployment.stage = 'ramping';
+        // Schedule first increment
+        const existingTimer = canaryTimers.get(workerName);
+        if (existingTimer) {
+            clearTimeout(existingTimer);
+            canaryTimers.delete(workerName);
+        }
+        // eslint-disable-next-line no-restricted-syntax
+        const timer = setTimeout(() => {
+            incrementTraffic(workerName);
+        }, config.monitoringDuration * 1000);
+        canaryTimers.set(workerName, timer);
+    },
+    /**
+     * Pause canary deployment
+     */
+    pause(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            throw ErrorFactory.createNotFoundError(`Canary deployment not found for "${workerName}"`);
+        }
+        // Clear timer
+        const timer = canaryTimers.get(workerName);
+        if (timer) {
+            clearTimeout(timer);
+            canaryTimers.delete(workerName);
+        }
+        Logger.info('Canary deployment paused', { workerName });
+    },
+    /**
+     * Resume canary deployment
+     */
+    resume(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            throw ErrorFactory.createNotFoundError(`Canary deployment not found for "${workerName}"`);
+        }
+        // Resume incrementing
+        const existingTimer = canaryTimers.get(workerName);
+        if (existingTimer) {
+            clearTimeout(existingTimer);
+            canaryTimers.delete(workerName);
+        }
+        // eslint-disable-next-line no-restricted-syntax
+        const timer = setTimeout(() => {
+            incrementTraffic(workerName);
+        }, deployment.config.incrementInterval * 1000);
+        canaryTimers.set(workerName, timer);
+        Logger.info('Canary deployment resumed', { workerName });
+    },
+    /**
+     * Complete canary deployment
+     */
+    complete(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            throw ErrorFactory.createNotFoundError(`Canary deployment not found for "${workerName}"`);
+        }
+        deployment.stage = 'completed';
+        deployment.completedAt = new Date();
+        // Clear timers
+        const timer = canaryTimers.get(workerName);
+        if (timer) {
+            clearTimeout(timer);
+            canaryTimers.delete(workerName);
+        }
+        const completeTimer = canaryTimers.get(`${workerName}:complete`);
+        if (completeTimer) {
+            clearTimeout(completeTimer);
+            canaryTimers.delete(`${workerName}:complete`);
+        }
+        // Record completion in history
+        appendHistory(deployment, {
+            timestamp: new Date(),
+            trafficPercent: deployment.currentTrafficPercent,
+            stage: 'completed',
+            metrics: { ...deployment.metrics },
+            decision: 'Deployment completed successfully',
+        });
+        Logger.info('Canary deployment completed', {
+            workerName,
+            duration: deployment.completedAt.getTime() - deployment.startedAt.getTime(),
+        });
+    },
+    /**
+     * Rollback canary deployment
+     */
+    rollback(workerName, reason) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            throw ErrorFactory.createNotFoundError(`Canary deployment not found for "${workerName}"`);
+        }
+        deployment.stage = 'rolling-back';
+        // Clear timers
+        const timer = canaryTimers.get(workerName);
+        if (timer) {
+            clearTimeout(timer);
+            canaryTimers.delete(workerName);
+        }
+        // Roll back traffic to 0%
+        deployment.currentTrafficPercent = 0;
+        // Record rollback in history
+        appendHistory(deployment, {
+            timestamp: new Date(),
+            trafficPercent: 0,
+            stage: 'rolling-back',
+            metrics: { ...deployment.metrics },
+            decision: `Rollback initiated: ${reason}`,
+        });
+        deployment.stage = 'failed';
+        deployment.completedAt = new Date();
+        Logger.error('Canary deployment rolled back', { workerName, reason });
+        // Optional: Open circuit breaker for canary version
+        CircuitBreaker.forceOpen(workerName, deployment.config.canaryVersion, reason);
+    },
+    /**
+     * Get canary deployment status
+     */
+    getStatus(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        return deployment ? { ...deployment } : null;
+    },
+    /**
+     * Update metrics for canary deployment
+     */
+    updateMetrics(workerName, version, processed, errors, avgLatency) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            return;
+        }
+        if (version === deployment.config.currentVersion) {
+            deployment.metrics.currentVersion = { processed, errors, avgLatency };
+        }
+        else if (version === deployment.config.canaryVersion) {
+            deployment.metrics.canaryVersion = { processed, errors, avgLatency };
+        }
+    },
+    /**
+     * Route job to version based on traffic percentage
+     */
+    routeJob(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment || deployment.stage === 'completed' || deployment.stage === 'failed') {
+            return null; // No active canary
+        }
+        // Random routing based on traffic percentage
+        const random = Math.random() * 100; // NOSONAR
+        if (random < deployment.currentTrafficPercent) {
+            return deployment.config.canaryVersion;
+        }
+        return deployment.config.currentVersion;
+    },
+    /**
+     * List all canary deployments
+     */
+    listDeployments() {
+        return Array.from(canaryDeployments.keys());
+    },
+    /**
+     * Get deployment history
+     */
+    getHistory(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        return deployment ? [...deployment.history] : null;
+    },
+    /**
+     * Remove completed/failed deployment
+     */
+    remove(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            throw ErrorFactory.createNotFoundError(`Canary deployment not found for "${workerName}"`);
+        }
+        if (deployment.stage !== 'completed' && deployment.stage !== 'failed') {
+            throw ErrorFactory.createValidationError('Cannot remove active deployment. Pause or complete it first.');
+        }
+        canaryDeployments.delete(workerName);
+        Logger.info('Canary deployment removed', { workerName });
+    },
+    /**
+     * Purge deployment data (force cleanup)
+     */
+    purge(workerName) {
+        const timer = canaryTimers.get(workerName);
+        if (timer) {
+            clearTimeout(timer);
+            canaryTimers.delete(workerName);
+        }
+        const completionTimer = canaryTimers.get(`${workerName}:complete`);
+        if (completionTimer) {
+            clearTimeout(completionTimer);
+            canaryTimers.delete(`${workerName}:complete`);
+        }
+        canaryDeployments.delete(workerName);
+        Logger.info('Canary deployment purged', { workerName });
+    },
+    /**
+     * Shutdown all canary deployments
+     */
+    shutdown() {
+        Logger.info('CanaryController shutting down...');
+        // Clear all timers
+        for (const timer of canaryTimers.values()) {
+            clearTimeout(timer);
+        }
+        canaryTimers.clear();
+        canaryDeployments.clear();
+        Logger.info('CanaryController shutdown complete');
+    },
+});
+// Graceful shutdown handled by WorkerShutdown

package/dist/ChaosEngineering.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Chaos Engineering
+ * Controlled fault injection experiments for worker resilience testing
+ * Sealed namespace for immutability
+ */
+export interface IChaosExperiment {
+    name: string;
+    description: string;
+    target: {
+        workers?: string[];
+        percentage?: number;
+    };
+    failure: {
+        type: 'crash' | 'latency' | 'error' | 'resource';
+        config: unknown;
+    };
+    duration: number;
+    safetyLimits: {
+        maxConcurrent: number;
+        circuitBreaker: boolean;
+        rollbackOn: string[];
+    };
+}
+export type IChaosStatus = {
+    id: string;
+    name: string;
+    state: 'defined' | 'running' | 'completed' | 'stopped' | 'failed';
+    startedAt: Date | null;
+    endedAt: Date | null;
+    targetedWorkers: string[];
+    failureType: IChaosExperiment['failure']['type'];
+};
+export type IChaosReport = {
+    experimentId: string;
+    summary: string;
+    durationMs: number;
+    targetedWorkers: string[];
+    completedAt: Date | null;
+};
+export type IChaosComparison = {
+    comparedIds: string[];
+    running: number;
+    completed: number;
+    stopped: number;
+};
+/**
+ * Chaos Engineering - Sealed namespace
+ */
+export declare const ChaosEngineering: Readonly<{
+    /**
+     * Define a chaos experiment
+     */
+    defineExperiment(config: IChaosExperiment): string;
+    /**
+     * Start a chaos experiment
+     */
+    startExperiment(experimentId: string): Promise<void>;
+    /**
+     * Stop a chaos experiment
+     */
+    stopExperiment(experimentId: string): Promise<void>;
+    /**
+     * Get experiment status
+     */
+    getExperimentStatus(experimentId: string): IChaosStatus;
+    /**
+     * Failure injection helpers
+     */
+    injectCrash(workerName: string): void;
+    injectLatency(workerName: string, delayMs: number): void;
+    injectError(workerName: string, errorRate: number): void;
+    injectResourceExhaustion(workerName: string, type: "cpu" | "memory"): void;
+    /**
+     * Analysis helpers
+     */
+    analyzeResilience(experimentId: string): IChaosReport;
+    compareExperiments(ids: string[]): IChaosComparison;
+}>;
+export default ChaosEngineering;

package/dist/ChaosEngineering.js

@@ -0,0 +1,216 @@
+/**
+ * Chaos Engineering
+ * Controlled fault injection experiments for worker resilience testing
+ * Sealed namespace for immutability
+ */
+import { ErrorFactory, Logger, generateUuid } from '@zintrust/core';
+import { CircuitBreaker } from './CircuitBreaker';
+import { Observability } from './Observability';
+import { ResourceMonitor } from './ResourceMonitor';
+import { WorkerRegistry } from './WorkerRegistry';
+const experiments = new Map();
+const getTargetWorkers = (config) => {
+    const candidates = config.target.workers ?? WorkerRegistry.listRunning();
+    if (candidates.length === 0)
+        return [];
+    const percentage = config.target.percentage ?? 100;
+    if (percentage >= 100)
+        return [...candidates];
+    const count = Math.max(1, Math.floor((candidates.length * percentage) / 100));
+    return candidates.slice(0, count);
+};
+const applyFailure = async (config, workers) => {
+    switch (config.failure.type) {
+        case 'crash':
+            await Promise.all(workers.map(async (workerName) => {
+                try {
+                    await WorkerRegistry.stop(workerName);
+                }
+                catch (error) {
+                    Logger.error(`Failed to inject crash for ${workerName}`, error);
+                }
+            }));
+            break;
+        case 'latency':
+            workers.forEach((workerName) => {
+                Logger.warn(`Injected latency for ${workerName}`, { config: config.failure.config });
+            });
+            break;
+        case 'error':
+            workers.forEach((workerName) => {
+                const status = WorkerRegistry.status(workerName);
+                if (status) {
+                    CircuitBreaker.forceOpen(workerName, status.version, 'Chaos experiment error injection');
+                }
+            });
+            break;
+        case 'resource':
+            workers.forEach((workerName) => {
+                const usage = ResourceMonitor.getCurrentUsage(workerName);
+                Logger.warn(`Resource pressure simulated for ${workerName}`, {
+                    cpu: usage.cpu,
+                    memory: usage.memory.percent,
+                    config: config.failure.config,
+                });
+            });
+            break;
+    }
+    if (Observability.isEnabled()) {
+        workers.forEach((workerName) => {
+            Observability.recordJobMetrics(workerName, 'chaos', {
+                processed: 0,
+                failed: 0,
+            });
+        });
+    }
+};
+const rollbackFailure = (config, workers) => {
+    if (!config.safetyLimits.circuitBreaker)
+        return;
+    if (config.failure.type === 'error') {
+        workers.forEach((workerName) => {
+            const status = WorkerRegistry.status(workerName);
+            if (status) {
+                CircuitBreaker.reset(workerName, status.version);
+            }
+        });
+    }
+};
+/**
+ * Chaos Engineering - Sealed namespace
+ */
+export const ChaosEngineering = Object.freeze({
+    /**
+     * Define a chaos experiment
+     */
+    defineExperiment(config) {
+        const id = generateUuid();
+        const status = {
+            id,
+            name: config.name,
+            state: 'defined',
+            startedAt: null,
+            endedAt: null,
+            targetedWorkers: [],
+            failureType: config.failure.type,
+        };
+        experiments.set(id, { config, status, timer: null });
+        Logger.info(`Chaos experiment defined: ${config.name}`, { id });
+        return id;
+    },
+    /**
+     * Start a chaos experiment
+     */
+    async startExperiment(experimentId) {
+        const record = experiments.get(experimentId);
+        if (!record) {
+            throw ErrorFactory.createNotFoundError(`Chaos experiment not found: ${experimentId}`);
+        }
+        if (record.status.state === 'running') {
+            throw ErrorFactory.createWorkerError(`Chaos experiment already running: ${experimentId}`);
+        }
+        const runningCount = Array.from(experiments.values()).filter((exp) => exp.status.state === 'running').length;
+        if (runningCount >= record.config.safetyLimits.maxConcurrent) {
+            throw ErrorFactory.createWorkerError('Maximum concurrent chaos experiments reached');
+        }
+        const targets = getTargetWorkers(record.config);
+        record.status.state = 'running';
+        record.status.startedAt = new Date();
+        record.status.targetedWorkers = targets;
+        Logger.warn(`Chaos experiment started: ${record.config.name}`, {
+            id: experimentId,
+            targets,
+        });
+        await applyFailure(record.config, targets);
+        record.timer = globalThis.setTimeout(() => {
+            const currentTimer = record.timer;
+            record.timer = null;
+            ChaosEngineering.stopExperiment(experimentId).catch((error) => {
+                Logger.error('Failed to stop chaos experiment after duration', error);
+                if (currentTimer)
+                    clearTimeout(currentTimer);
+            });
+        }, record.config.duration);
+    },
+    /**
+     * Stop a chaos experiment
+     */
+    async stopExperiment(experimentId) {
+        const record = experiments.get(experimentId);
+        if (!record) {
+            throw ErrorFactory.createNotFoundError(`Chaos experiment not found: ${experimentId}`);
+        }
+        if (record.timer) {
+            clearTimeout(record.timer);
+            record.timer = null;
+        }
+        rollbackFailure(record.config, record.status.targetedWorkers);
+        record.status.state = 'completed';
+        record.status.endedAt = new Date();
+        Logger.info(`Chaos experiment completed: ${record.config.name}`, {
+            id: experimentId,
+            duration: record.config.duration,
+        });
+    },
+    /**
+     * Get experiment status
+     */
+    getExperimentStatus(experimentId) {
+        const record = experiments.get(experimentId);
+        if (!record) {
+            throw ErrorFactory.createNotFoundError(`Chaos experiment not found: ${experimentId}`);
+        }
+        return { ...record.status, targetedWorkers: [...record.status.targetedWorkers] };
+    },
+    /**
+     * Failure injection helpers
+     */
+    injectCrash(workerName) {
+        WorkerRegistry.stop(workerName).catch((error) => {
+            Logger.error(`Failed to inject crash for ${workerName}`, error);
+        });
+    },
+    injectLatency(workerName, delayMs) {
+        Logger.warn(`Injected latency ${delayMs}ms for ${workerName}`);
+    },
+    injectError(workerName, errorRate) {
+        const status = WorkerRegistry.status(workerName);
+        if (!status) {
+            throw ErrorFactory.createNotFoundError(`Worker not found: ${workerName}`);
+        }
+        CircuitBreaker.forceOpen(workerName, status.version, `Chaos error rate ${errorRate}`);
+    },
+    injectResourceExhaustion(workerName, type) {
+        const usage = ResourceMonitor.getCurrentUsage(workerName);
+        Logger.warn(`Injected resource exhaustion (${type}) for ${workerName}`, {
+            cpu: usage.cpu,
+            memory: usage.memory.percent,
+        });
+    },
+    /**
+     * Analysis helpers
+     */
+    analyzeResilience(experimentId) {
+        const record = experiments.get(experimentId);
+        if (!record) {
+            throw ErrorFactory.createNotFoundError(`Chaos experiment not found: ${experimentId}`);
+        }
+        return {
+            experimentId,
+            summary: `Experiment ${record.config.name} finished with ${record.status.state}`,
+            durationMs: record.config.duration,
+            targetedWorkers: [...record.status.targetedWorkers],
+            completedAt: record.status.endedAt,
+        };
+    },
+    compareExperiments(ids) {
+        const statuses = ids.map((id) => experiments.get(id)?.status).filter(Boolean);
+        return {
+            comparedIds: ids,
+            running: statuses.filter((s) => s.state === 'running').length,
+            completed: statuses.filter((s) => s.state === 'completed').length,
+            stopped: statuses.filter((s) => s.state === 'stopped').length,
+        };
+    },
+});
+export default ChaosEngineering;