@zintrust/core 0.1.41 → 0.1.43

package/packages/workers/src/CanaryController.js

@@ -0,0 +1,424 @@
+/**
+ * Canary Deployment Controller
+ * Gradual rollout with traffic percentage control and automatic rollback
+ * Sealed namespace for immutability
+ */
+import { ErrorFactory, Logger } from '../../../src/index.js';
+import { CircuitBreaker } from './CircuitBreaker.js';
+// Internal state
+const canaryDeployments = new Map();
+const canaryTimers = new Map();
+const MAX_HISTORY = 1000;
+/**
+ * Helper: Calculate error rate
+ */
+const calculateErrorRate = (processed, errors) => {
+    if (processed === 0)
+        return 0;
+    return errors / processed;
+};
+/**
+ * Helper: Calculate success rate
+ */
+const calculateSuccessRate = (processed, errors) => {
+    if (processed === 0)
+        return 1;
+    return (processed - errors) / processed;
+};
+/**
+ * Helper: Should rollback based on metrics
+ */
+const shouldRollback = (deployment) => {
+    const { config, metrics } = deployment;
+    const { canaryVersion } = metrics;
+    // Check error threshold
+    const errorRate = calculateErrorRate(canaryVersion.processed, canaryVersion.errors);
+    if (errorRate > config.errorThreshold) {
+        return {
+            should: true,
+            reason: `Error rate ${(errorRate * 100).toFixed(2)}% exceeds threshold ${(config.errorThreshold * 100).toFixed(2)}%`,
+        };
+    }
+    // Check success rate
+    const successRate = calculateSuccessRate(canaryVersion.processed, canaryVersion.errors);
+    if (successRate < config.minSuccessRate) {
+        return {
+            should: true,
+            reason: `Success rate ${(successRate * 100).toFixed(2)}% below minimum ${(config.minSuccessRate * 100).toFixed(2)}%`,
+        };
+    }
+    // Check latency threshold
+    if (canaryVersion.avgLatency > config.latencyThreshold) {
+        return {
+            should: true,
+            reason: `P95 latency ${canaryVersion.avgLatency}ms exceeds threshold ${config.latencyThreshold}ms`,
+        };
+    }
+    return { should: false };
+};
+/**
+ * Helper: Increment traffic
+ */
+const incrementTraffic = (workerName) => {
+    const deployment = canaryDeployments.get(workerName);
+    if (!deployment) {
+        Logger.error('Canary deployment not found', { workerName });
+        return;
+    }
+    const { config } = deployment;
+    // Check if we should rollback
+    const rollbackCheck = shouldRollback(deployment);
+    if (rollbackCheck.should && config.autoRollback) {
+        Logger.warn('Auto-rollback triggered', {
+            workerName,
+            reason: rollbackCheck.reason,
+        });
+        CanaryController.rollback(workerName, rollbackCheck.reason ?? '');
+        return;
+    }
+    // Increment traffic
+    const newTrafficPercent = Math.min(deployment.currentTrafficPercent + config.incrementPercent, config.targetTrafficPercent);
+    deployment.currentTrafficPercent = newTrafficPercent;
+    // Record history
+    appendHistory(deployment, {
+        timestamp: new Date(),
+        trafficPercent: newTrafficPercent,
+        stage: deployment.stage,
+        metrics: { ...deployment.metrics },
+        decision: `Traffic increased to ${newTrafficPercent}%`,
+    });
+    Logger.info('Canary traffic incremented', {
+        workerName,
+        trafficPercent: newTrafficPercent,
+        targetPercent: config.targetTrafficPercent,
+    });
+    // Check if we've reached the target
+    if (newTrafficPercent >= config.targetTrafficPercent) {
+        deployment.stage = 'monitoring';
+        // Wait for final monitoring period
+        const existingCompleteTimer = canaryTimers.get(`${workerName}:complete`);
+        if (existingCompleteTimer) {
+            clearTimeout(existingCompleteTimer);
+            canaryTimers.delete(`${workerName}:complete`);
+        }
+        // eslint-disable-next-line no-restricted-syntax
+        const timer = setTimeout(() => {
+            try {
+                CanaryController.complete(workerName);
+            }
+            catch (error) {
+                Logger.error('Error during canary completion callback', error);
+            }
+            finally {
+                const current = canaryTimers.get(`${workerName}:complete`);
+                if (current === timer) {
+                    canaryTimers.delete(`${workerName}:complete`);
+                }
+            }
+        }, config.monitoringDuration * 1000);
+        canaryTimers.set(`${workerName}:complete`, timer);
+    }
+    else {
+        // Schedule next increment
+        const existingTimer = canaryTimers.get(workerName);
+        if (existingTimer) {
+            clearTimeout(existingTimer);
+            canaryTimers.delete(workerName);
+        }
+        // eslint-disable-next-line no-restricted-syntax
+        const timer = setTimeout(() => {
+            try {
+                incrementTraffic(workerName);
+            }
+            catch (error) {
+                Logger.error('Error during canary increment callback', error);
+            }
+            finally {
+                const current = canaryTimers.get(workerName);
+                if (current === timer) {
+                    canaryTimers.delete(workerName);
+                }
+            }
+        }, config.incrementInterval * 1000);
+        canaryTimers.set(workerName, timer);
+    }
+};
+const appendHistory = (deployment, entry) => {
+    deployment.history.push(entry);
+    if (deployment.history.length > MAX_HISTORY) {
+        deployment.history.splice(0, deployment.history.length - MAX_HISTORY);
+    }
+};
+/**
+ * Canary Deployment Controller - Sealed namespace
+ */
+export const CanaryController = Object.freeze({
+    /**
+     * Start canary deployment
+     */
+    start(config) {
+        const { workerName } = config;
+        if (canaryDeployments.has(workerName)) {
+            throw ErrorFactory.createGeneralError(`Canary deployment already in progress for "${workerName}"`);
+        }
+        // Validate config
+        if (config.initialTrafficPercent < 0 || config.initialTrafficPercent > 100) {
+            throw ErrorFactory.createValidationError('Initial traffic percent must be between 0 and 100');
+        }
+        if (config.targetTrafficPercent < config.initialTrafficPercent ||
+            config.targetTrafficPercent > 100) {
+            throw ErrorFactory.createValidationError('Target traffic percent must be >= initial and <= 100');
+        }
+        // Create deployment
+        const deployment = {
+            config,
+            currentTrafficPercent: config.initialTrafficPercent,
+            stage: 'initial',
+            startedAt: new Date(),
+            metrics: {
+                currentVersion: { processed: 0, errors: 0, avgLatency: 0 },
+                canaryVersion: { processed: 0, errors: 0, avgLatency: 0 },
+            },
+            history: [],
+        };
+        canaryDeployments.set(workerName, deployment);
+        Logger.info('Canary deployment started', {
+            workerName,
+            currentVersion: config.currentVersion,
+            canaryVersion: config.canaryVersion,
+            initialTraffic: config.initialTrafficPercent,
+        });
+        // Start ramping up
+        deployment.stage = 'ramping';
+        // Schedule first increment
+        const existingTimer = canaryTimers.get(workerName);
+        if (existingTimer) {
+            clearTimeout(existingTimer);
+            canaryTimers.delete(workerName);
+        }
+        // eslint-disable-next-line no-restricted-syntax
+        const timer = setTimeout(() => {
+            try {
+                incrementTraffic(workerName);
+            }
+            catch (error) {
+                Logger.error('Error during canary start callback', error);
+            }
+            finally {
+                const current = canaryTimers.get(workerName);
+                if (current === timer) {
+                    canaryTimers.delete(workerName);
+                }
+            }
+        }, config.monitoringDuration * 1000);
+        canaryTimers.set(workerName, timer);
+    },
+    /**
+     * Pause canary deployment
+     */
+    pause(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            throw ErrorFactory.createNotFoundError(`Canary deployment not found for "${workerName}"`);
+        }
+        // Clear timer
+        const timer = canaryTimers.get(workerName);
+        if (timer) {
+            clearTimeout(timer);
+            canaryTimers.delete(workerName);
+        }
+        Logger.info('Canary deployment paused', { workerName });
+    },
+    /**
+     * Resume canary deployment
+     */
+    resume(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            throw ErrorFactory.createNotFoundError(`Canary deployment not found for "${workerName}"`);
+        }
+        // Resume incrementing
+        const existingTimer = canaryTimers.get(workerName);
+        if (existingTimer) {
+            clearTimeout(existingTimer);
+            canaryTimers.delete(workerName);
+        }
+        // eslint-disable-next-line no-restricted-syntax
+        const timer = setTimeout(() => {
+            try {
+                incrementTraffic(workerName);
+            }
+            catch (error) {
+                Logger.error('Error during canary resume callback', error);
+            }
+            finally {
+                const current = canaryTimers.get(workerName);
+                if (current === timer) {
+                    canaryTimers.delete(workerName);
+                }
+            }
+        }, deployment.config.incrementInterval * 1000);
+        canaryTimers.set(workerName, timer);
+        Logger.info('Canary deployment resumed', { workerName });
+    },
+    /**
+     * Complete canary deployment
+     */
+    complete(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            throw ErrorFactory.createNotFoundError(`Canary deployment not found for "${workerName}"`);
+        }
+        deployment.stage = 'completed';
+        deployment.completedAt = new Date();
+        // Clear timers
+        const timer = canaryTimers.get(workerName);
+        if (timer) {
+            clearTimeout(timer);
+            canaryTimers.delete(workerName);
+        }
+        const completeTimer = canaryTimers.get(`${workerName}:complete`);
+        if (completeTimer) {
+            clearTimeout(completeTimer);
+            canaryTimers.delete(`${workerName}:complete`);
+        }
+        // Record completion in history
+        appendHistory(deployment, {
+            timestamp: new Date(),
+            trafficPercent: deployment.currentTrafficPercent,
+            stage: 'completed',
+            metrics: { ...deployment.metrics },
+            decision: 'Deployment completed successfully',
+        });
+        Logger.info('Canary deployment completed', {
+            workerName,
+            duration: deployment.completedAt.getTime() - deployment.startedAt.getTime(),
+        });
+    },
+    /**
+     * Rollback canary deployment
+     */
+    rollback(workerName, reason) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            throw ErrorFactory.createNotFoundError(`Canary deployment not found for "${workerName}"`);
+        }
+        deployment.stage = 'rolling-back';
+        // Clear timers
+        const timer = canaryTimers.get(workerName);
+        if (timer) {
+            clearTimeout(timer);
+            canaryTimers.delete(workerName);
+        }
+        // Roll back traffic to 0%
+        deployment.currentTrafficPercent = 0;
+        // Record rollback in history
+        appendHistory(deployment, {
+            timestamp: new Date(),
+            trafficPercent: 0,
+            stage: 'rolling-back',
+            metrics: { ...deployment.metrics },
+            decision: `Rollback initiated: ${reason}`,
+        });
+        deployment.stage = 'failed';
+        deployment.completedAt = new Date();
+        Logger.error('Canary deployment rolled back', { workerName, reason });
+        // Optional: Open circuit breaker for canary version
+        CircuitBreaker.forceOpen(workerName, deployment.config.canaryVersion, reason);
+    },
+    /**
+     * Get canary deployment status
+     */
+    getStatus(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        return deployment ? { ...deployment } : null;
+    },
+    /**
+     * Update metrics for canary deployment
+     */
+    updateMetrics(workerName, version, processed, errors, avgLatency) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            return;
+        }
+        if (version === deployment.config.currentVersion) {
+            deployment.metrics.currentVersion = { processed, errors, avgLatency };
+        }
+        else if (version === deployment.config.canaryVersion) {
+            deployment.metrics.canaryVersion = { processed, errors, avgLatency };
+        }
+    },
+    /**
+     * Route job to version based on traffic percentage
+     */
+    routeJob(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment || deployment.stage === 'completed' || deployment.stage === 'failed') {
+            return null; // No active canary
+        }
+        // Random routing based on traffic percentage
+        const random = Math.random() * 100; // NOSONAR
+        if (random < deployment.currentTrafficPercent) {
+            return deployment.config.canaryVersion;
+        }
+        return deployment.config.currentVersion;
+    },
+    /**
+     * List all canary deployments
+     */
+    listDeployments() {
+        return Array.from(canaryDeployments.keys());
+    },
+    /**
+     * Get deployment history
+     */
+    getHistory(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        return deployment ? [...deployment.history] : null;
+    },
+    /**
+     * Remove completed/failed deployment
+     */
+    remove(workerName) {
+        const deployment = canaryDeployments.get(workerName);
+        if (!deployment) {
+            throw ErrorFactory.createNotFoundError(`Canary deployment not found for "${workerName}"`);
+        }
+        if (deployment.stage !== 'completed' && deployment.stage !== 'failed') {
+            throw ErrorFactory.createValidationError('Cannot remove active deployment. Pause or complete it first.');
+        }
+        canaryDeployments.delete(workerName);
+        Logger.info('Canary deployment removed', { workerName });
+    },
+    /**
+     * Purge deployment data (force cleanup)
+     */
+    purge(workerName) {
+        const timer = canaryTimers.get(workerName);
+        if (timer) {
+            clearTimeout(timer);
+            canaryTimers.delete(workerName);
+        }
+        const completionTimer = canaryTimers.get(`${workerName}:complete`);
+        if (completionTimer) {
+            clearTimeout(completionTimer);
+            canaryTimers.delete(`${workerName}:complete`);
+        }
+        canaryDeployments.delete(workerName);
+        Logger.info('Canary deployment purged', { workerName });
+    },
+    /**
+     * Shutdown all canary deployments
+     */
+    shutdown() {
+        Logger.info('CanaryController shutting down...');
+        // Clear all timers
+        for (const timer of canaryTimers.values()) {
+            clearTimeout(timer);
+        }
+        canaryTimers.clear();
+        canaryDeployments.clear();
+        Logger.info('CanaryController shutdown complete');
+    },
+});
+// Graceful shutdown handled by WorkerShutdown

package/packages/workers/src/ChaosEngineering.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Chaos Engineering
+ * Controlled fault injection experiments for worker resilience testing
+ * Sealed namespace for immutability
+ */
+export interface IChaosExperiment {
+    name: string;
+    description: string;
+    target: {
+        workers?: string[];
+        percentage?: number;
+    };
+    failure: {
+        type: 'crash' | 'latency' | 'error' | 'resource';
+        config: unknown;
+    };
+    duration: number;
+    safetyLimits: {
+        maxConcurrent: number;
+        circuitBreaker: boolean;
+        rollbackOn: string[];
+    };
+}
+export type IChaosStatus = {
+    id: string;
+    name: string;
+    state: 'defined' | 'running' | 'completed' | 'stopped' | 'failed';
+    startedAt: Date | null;
+    endedAt: Date | null;
+    targetedWorkers: string[];
+    failureType: IChaosExperiment['failure']['type'];
+};
+export type IChaosReport = {
+    experimentId: string;
+    summary: string;
+    durationMs: number;
+    targetedWorkers: string[];
+    completedAt: Date | null;
+};
+export type IChaosComparison = {
+    comparedIds: string[];
+    running: number;
+    completed: number;
+    stopped: number;
+};
+/**
+ * Chaos Engineering - Sealed namespace
+ */
+export declare const ChaosEngineering: Readonly<{
+    /**
+     * Define a chaos experiment
+     */
+    defineExperiment(config: IChaosExperiment): string;
+    /**
+     * Start a chaos experiment
+     */
+    startExperiment(experimentId: string): Promise<void>;
+    /**
+     * Stop a chaos experiment
+     */
+    stopExperiment(experimentId: string): Promise<void>;
+    /**
+     * Get experiment status
+     */
+    getExperimentStatus(experimentId: string): IChaosStatus;
+    /**
+     * Failure injection helpers
+     */
+    injectCrash(workerName: string): void;
+    injectLatency(workerName: string, delayMs: number): void;
+    injectError(workerName: string, errorRate: number): void;
+    injectResourceExhaustion(workerName: string, type: "cpu" | "memory"): void;
+    /**
+     * Analysis helpers
+     */
+    analyzeResilience(experimentId: string): IChaosReport;
+    compareExperiments(ids: string[]): IChaosComparison;
+}>;
+export default ChaosEngineering;
+//# sourceMappingURL=ChaosEngineering.d.ts.map

package/packages/workers/src/ChaosEngineering.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChaosEngineering.d.ts","sourceRoot":"","sources":["../../../../packages/workers/src/ChaosEngineering.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAQH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE;QACN,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,OAAO,EAAE;QACP,IAAI,EAAE,OAAO,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,CAAC;QACjD,MAAM,EAAE,OAAO,CAAC;KACjB,CAAC;IACF,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE;QACZ,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,OAAO,CAAC;QACxB,UAAU,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACH;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,QAAQ,CAAC;IAClE,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,OAAO,EAAE,IAAI,GAAG,IAAI,CAAC;IACrB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,WAAW,EAAE,gBAAgB,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC;CAClD,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,WAAW,EAAE,IAAI,GAAG,IAAI,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AA8FF;;GAEG;AACH,eAAO,MAAM,gBAAgB;IAC3B;;OAEG;6BACsB,gBAAgB,GAAG,MAAM;IAiBlD;;OAEG;kCACiC,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwC1D;;OAEG;iCACgC,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBzD;;OAEG;sCAC+B,MAAM,GAAG,YAAY;IASvD;;OAEG;4BACqB,MAAM,GAAG,IAAI;8BAMX,MAAM,WAAW,MAAM,GAAG,IAAI;4BAIhC,MAAM,aAAa,MAAM,GAAG,IAAI;yCASnB,MAAM,QAAQ,KAAK,GAAG,QAAQ,GAAG,IAAI;IAQ1E;;OAEG;oCAC6B,MAAM,GAAG,YAAY;4BAe7B,MAAM,EAAE,GAAG,gBAAgB;EAUnD,CAAC;AAEH,eAAe,gBAAgB,CAAC"}