@zintrust/core 0.1.41 → 0.1.43

package/packages/workers/src/ChaosEngineering.js

@@ -0,0 +1,229 @@
+/**
+ * Chaos Engineering
+ * Controlled fault injection experiments for worker resilience testing
+ * Sealed namespace for immutability
+ */
+import { ErrorFactory, Logger, generateUuid } from '../../../src/index.js';
+import { CircuitBreaker } from './CircuitBreaker.js';
+import { Observability } from './Observability.js';
+import { ResourceMonitor } from './ResourceMonitor.js';
+import { WorkerRegistry } from './WorkerRegistry.js';
+const experiments = new Map();
+const EXPERIMENT_RETENTION_MS = 24 * 60 * 60 * 1000;
+const cleanupExpiredExperiments = () => {
+    const cutoff = Date.now() - EXPERIMENT_RETENTION_MS;
+    for (const [id, record] of experiments.entries()) {
+        if (record.status.state !== 'completed')
+            continue;
+        const endedAt = record.status.endedAt?.getTime() ?? 0;
+        if (endedAt > 0 && endedAt < cutoff) {
+            experiments.delete(id);
+        }
+    }
+};
+const getTargetWorkers = (config) => {
+    const candidates = config.target.workers ?? WorkerRegistry.listRunning();
+    if (candidates.length === 0)
+        return [];
+    const percentage = config.target.percentage ?? 100;
+    if (percentage >= 100)
+        return [...candidates];
+    const count = Math.max(1, Math.floor((candidates.length * percentage) / 100));
+    return candidates.slice(0, count);
+};
+const applyFailure = async (config, workers) => {
+    switch (config.failure.type) {
+        case 'crash':
+            await Promise.all(workers.map(async (workerName) => {
+                try {
+                    await WorkerRegistry.stop(workerName);
+                }
+                catch (error) {
+                    Logger.error(`Failed to inject crash for ${workerName}`, error);
+                }
+            }));
+            break;
+        case 'latency':
+            workers.forEach((workerName) => {
+                Logger.warn(`Injected latency for ${workerName}`, { config: config.failure.config });
+            });
+            break;
+        case 'error':
+            workers.forEach((workerName) => {
+                const status = WorkerRegistry.status(workerName);
+                if (status) {
+                    CircuitBreaker.forceOpen(workerName, status.version, 'Chaos experiment error injection');
+                }
+            });
+            break;
+        case 'resource':
+            workers.forEach((workerName) => {
+                const usage = ResourceMonitor.getCurrentUsage(workerName);
+                Logger.warn(`Resource pressure simulated for ${workerName}`, {
+                    cpu: usage.cpu,
+                    memory: usage.memory.percent,
+                    config: config.failure.config,
+                });
+            });
+            break;
+    }
+    if (Observability.isEnabled()) {
+        workers.forEach((workerName) => {
+            Observability.recordJobMetrics(workerName, 'chaos', {
+                processed: 0,
+                failed: 0,
+            });
+        });
+    }
+};
+const rollbackFailure = (config, workers) => {
+    if (!config.safetyLimits.circuitBreaker)
+        return;
+    if (config.failure.type === 'error') {
+        workers.forEach((workerName) => {
+            const status = WorkerRegistry.status(workerName);
+            if (status) {
+                CircuitBreaker.reset(workerName, status.version);
+            }
+        });
+    }
+};
+/**
+ * Chaos Engineering - Sealed namespace
+ */
+export const ChaosEngineering = Object.freeze({
+    /**
+     * Define a chaos experiment
+     */
+    defineExperiment(config) {
+        const id = generateUuid();
+        const status = {
+            id,
+            name: config.name,
+            state: 'defined',
+            startedAt: null,
+            endedAt: null,
+            targetedWorkers: [],
+            failureType: config.failure.type,
+        };
+        experiments.set(id, { config, status, timer: null });
+        Logger.info(`Chaos experiment defined: ${config.name}`, { id });
+        return id;
+    },
+    /**
+     * Start a chaos experiment
+     */
+    async startExperiment(experimentId) {
+        const record = experiments.get(experimentId);
+        if (!record) {
+            throw ErrorFactory.createNotFoundError(`Chaos experiment not found: ${experimentId}`);
+        }
+        if (record.status.state === 'running') {
+            throw ErrorFactory.createWorkerError(`Chaos experiment already running: ${experimentId}`);
+        }
+        const runningCount = Array.from(experiments.values()).filter((exp) => exp.status.state === 'running').length;
+        if (runningCount >= record.config.safetyLimits.maxConcurrent) {
+            throw ErrorFactory.createWorkerError('Maximum concurrent chaos experiments reached');
+        }
+        const targets = getTargetWorkers(record.config);
+        record.status.state = 'running';
+        record.status.startedAt = new Date();
+        record.status.targetedWorkers = targets;
+        Logger.warn(`Chaos experiment started: ${record.config.name}`, {
+            id: experimentId,
+            targets,
+        });
+        await applyFailure(record.config, targets);
+        record.timer = globalThis.setTimeout(() => {
+            const currentTimer = record.timer;
+            record.timer = null;
+            ChaosEngineering.stopExperiment(experimentId).catch((error) => {
+                Logger.error('Failed to stop chaos experiment after duration', error);
+                if (currentTimer)
+                    clearTimeout(currentTimer);
+            });
+        }, record.config.duration);
+    },
+    /**
+     * Stop a chaos experiment
+     */
+    async stopExperiment(experimentId) {
+        const record = experiments.get(experimentId);
+        if (!record) {
+            throw ErrorFactory.createNotFoundError(`Chaos experiment not found: ${experimentId}`);
+        }
+        if (record.timer) {
+            clearTimeout(record.timer);
+            record.timer = null;
+        }
+        rollbackFailure(record.config, record.status.targetedWorkers);
+        record.status.state = 'completed';
+        record.status.endedAt = new Date();
+        Logger.info(`Chaos experiment completed: ${record.config.name}`, {
+            id: experimentId,
+            duration: record.config.duration,
+        });
+        cleanupExpiredExperiments();
+    },
+    /**
+     * Get experiment status
+     */
+    getExperimentStatus(experimentId) {
+        const record = experiments.get(experimentId);
+        if (!record) {
+            throw ErrorFactory.createNotFoundError(`Chaos experiment not found: ${experimentId}`);
+        }
+        return { ...record.status, targetedWorkers: [...record.status.targetedWorkers] };
+    },
+    /**
+     * Failure injection helpers
+     */
+    injectCrash(workerName) {
+        WorkerRegistry.stop(workerName).catch((error) => {
+            Logger.error(`Failed to inject crash for ${workerName}`, error);
+        });
+    },
+    injectLatency(workerName, delayMs) {
+        Logger.warn(`Injected latency ${delayMs}ms for ${workerName}`);
+    },
+    injectError(workerName, errorRate) {
+        const status = WorkerRegistry.status(workerName);
+        if (!status) {
+            throw ErrorFactory.createNotFoundError(`Worker not found: ${workerName}`);
+        }
+        CircuitBreaker.forceOpen(workerName, status.version, `Chaos error rate ${errorRate}`);
+    },
+    injectResourceExhaustion(workerName, type) {
+        const usage = ResourceMonitor.getCurrentUsage(workerName);
+        Logger.warn(`Injected resource exhaustion (${type}) for ${workerName}`, {
+            cpu: usage.cpu,
+            memory: usage.memory.percent,
+        });
+    },
+    /**
+     * Analysis helpers
+     */
+    analyzeResilience(experimentId) {
+        const record = experiments.get(experimentId);
+        if (!record) {
+            throw ErrorFactory.createNotFoundError(`Chaos experiment not found: ${experimentId}`);
+        }
+        return {
+            experimentId,
+            summary: `Experiment ${record.config.name} finished with ${record.status.state}`,
+            durationMs: record.config.duration,
+            targetedWorkers: [...record.status.targetedWorkers],
+            completedAt: record.status.endedAt,
+        };
+    },
+    compareExperiments(ids) {
+        const statuses = ids.map((id) => experiments.get(id)?.status).filter(Boolean);
+        return {
+            comparedIds: ids,
+            running: statuses.filter((s) => s.state === 'running').length,
+            completed: statuses.filter((s) => s.state === 'completed').length,
+            stopped: statuses.filter((s) => s.state === 'stopped').length,
+        };
+    },
+});
+export default ChaosEngineering;

package/packages/workers/src/CircuitBreaker.d.ts

@@ -0,0 +1,107 @@
+/**
+ * Circuit Breaker Pattern
+ * Fault tolerance with version tracking and automatic recovery
+ * Sealed namespace for immutability
+ */
+export type CircuitState = 'closed' | 'open' | 'half-open';
+export type CircuitBreakerConfig = {
+    failureThreshold: number;
+    successThreshold: number;
+    timeout: number;
+    resetTimeout: number;
+    volumeThreshold: number;
+};
+export type CircuitBreakerState = {
+    workerName: string;
+    version: string;
+    state: CircuitState;
+    failureCount: number;
+    successCount: number;
+    totalRequests: number;
+    lastFailureTime: Date | null;
+    lastSuccessTime: Date | null;
+    lastStateChange: Date;
+    nextRetryTime: Date | null;
+    errorRate: number;
+};
+export type CircuitBreakerEvent = {
+    workerName: string;
+    version: string;
+    event: 'opened' | 'closed' | 'half-open' | 'success' | 'failure' | 'rejected';
+    state: CircuitState;
+    timestamp: Date;
+    reason?: string;
+    error?: Error;
+};
+/**
+ * Circuit Breaker - Sealed namespace
+ */
+export declare const CircuitBreaker: Readonly<{
+    /**
+     * Initialize circuit breaker for a worker version
+     */
+    initialize(workerName: string, version: string): void;
+    /**
+     * Check if circuit allows execution
+     */
+    canExecute(workerName: string, version: string): boolean;
+    /**
+     * Record successful execution
+     */
+    recordSuccess(workerName: string, version: string): void;
+    /**
+     * Record failed execution
+     */
+    recordFailure(workerName: string, version: string, error: Error): void;
+    /**
+     * Record rejected execution (when circuit is open)
+     */
+    recordRejection(workerName: string, version: string): void;
+    /**
+     * Get circuit state
+     */
+    getState(workerName: string, version: string): CircuitBreakerState | null;
+    /**
+     * Get all circuit states
+     */
+    getAllStates(): ReadonlyArray<CircuitBreakerState>;
+    /**
+     * Get circuit states by worker name (all versions)
+     */
+    getStatesByWorker(workerName: string): ReadonlyArray<CircuitBreakerState>;
+    /**
+     * Get event history
+     */
+    getEventHistory(workerName: string, version: string, limit?: number): ReadonlyArray<CircuitBreakerEvent>;
+    /**
+     * Manually reset circuit to closed state
+     */
+    reset(workerName: string, version: string): void;
+    /**
+     * Manually force circuit to open state
+     */
+    forceOpen(workerName: string, version: string, reason: string): void;
+    /**
+     * Delete circuit breaker
+     */
+    delete(workerName: string, version: string): void;
+    /**
+     * Delete all circuit breakers for a worker (all versions)
+     */
+    deleteWorker(workerName: string): void;
+    /**
+     * Get summary statistics
+     */
+    getSummary(): {
+        totalCircuits: number;
+        openCircuits: number;
+        halfOpenCircuits: number;
+        closedCircuits: number;
+        circuitsByWorker: Record<string, number>;
+    };
+    /**
+     * Shutdown and clear all circuits
+     */
+    shutdown(): void;
+}>;
+//# sourceMappingURL=CircuitBreaker.d.ts.map

package/packages/workers/src/CircuitBreaker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CircuitBreaker.d.ts","sourceRoot":"","sources":["../../../../packages/workers/src/CircuitBreaker.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAE3D,MAAM,MAAM,oBAAoB,GAAG;IACjC,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,IAAI,GAAG,IAAI,CAAC;IAC7B,eAAe,EAAE,IAAI,GAAG,IAAI,CAAC;IAC7B,eAAe,EAAE,IAAI,CAAC;IACtB,aAAa,EAAE,IAAI,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,QAAQ,GAAG,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,SAAS,GAAG,UAAU,CAAC;IAC9E,KAAK,EAAE,YAAY,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,KAAK,CAAC;CACf,CAAC;AA6GF;;GAEG;AACH,eAAO,MAAM,cAAc;IACzB;;OAEG;2BACoB,MAAM,WAAW,MAAM,GAAG,IAAI;IA2BrD;;OAEG;2BACoB,MAAM,WAAW,MAAM,GAAG,OAAO;IAqCxD;;OAEG;8BACuB,MAAM,WAAW,MAAM,GAAG,IAAI;IA8BxD;;OAEG;8BACuB,MAAM,WAAW,MAAM,SAAS,KAAK,GAAG,IAAI;IAyCtE;;OAEG;gCACyB,MAAM,WAAW,MAAM,GAAG,IAAI;IAkB1D;;OAEG;yBACkB,MAAM,WAAW,MAAM,GAAG,mBAAmB,GAAG,IAAI;IAOzE;;OAEG;oBACa,aAAa,CAAC,mBAAmB,CAAC;IAIlD;;OAEG;kCAC2B,MAAM,GAAG,aAAa,CAAC,mBAAmB,CAAC;IAYzE;;OAEG;gCAEW,MAAM,WACT,MAAM,mBAEd,aAAa,CAAC,mBAAmB,CAAC;IAOrC;;OAEG;sBACe,MAAM,WAAW,MAAM,GAAG,IAAI;IAkBhD;;OAEG;0BACmB,MAAM,WAAW,MAAM,UAAU,MAAM,GAAG,IAAI;IAmBpE;;OAEG;uBACgB,MAAM,WAAW,MAAM,GAAG,IAAI;IAQjD;;OAEG;6BACsB,MAAM,GAAG,IAAI;IAmBtC;;OAEG;kBACW;QACZ,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC1C;IA6BD;;OAEG;gBACS,IAAI;EAQhB,CAAC"}