@zintrust/workers 0.1.27

package/dist/SLAMonitor.js

@@ -0,0 +1,274 @@
+/**
+ * SLA Monitor
+ * SLA compliance checks and violation tracking for workers
+ * Sealed namespace for immutability
+ */
+import { ErrorFactory, Logger, generateUuid } from '@zintrust/core';
+import { HealthMonitor } from './HealthMonitor';
+import { WorkerMetrics } from './WorkerMetrics';
+const slaConfigs = new Map();
+const violationHistory = new Map();
+const lastAlertAt = new Map();
+const DEFAULT_LOOKBACK_MS = 60 * 60 * 1000;
+const quantile = (points, percentile) => {
+    if (points.length === 0)
+        return 0;
+    const sorted = [...points].sort((a, b) => a.value - b.value);
+    const index = Math.min(sorted.length - 1, Math.max(0, Math.floor(percentile * sorted.length)));
+    return sorted[index]?.value ?? 0;
+};
+const warnOnMaxThreshold = (value, threshold) => value >= threshold * 0.9 && value <= threshold;
+const warnOnMinThreshold = (value, threshold) => value <= threshold * 1.1 && value >= threshold;
+const evaluateMaxThreshold = (value, threshold) => {
+    if (value > threshold)
+        return 'fail';
+    if (warnOnMaxThreshold(value, threshold))
+        return 'warn';
+    return 'pass';
+};
+const evaluateMinThreshold = (value, threshold) => {
+    if (value < threshold)
+        return 'fail';
+    if (warnOnMinThreshold(value, threshold))
+        return 'warn';
+    return 'pass';
+};
+const storeViolation = (violation) => {
+    const history = violationHistory.get(violation.workerName) ?? [];
+    history.push(violation);
+    if (history.length > 1000) {
+        history.shift();
+    }
+    violationHistory.set(violation.workerName, history);
+};
+const calculateAvailability = (checks) => {
+    if (checks.length === 0)
+        return 0;
+    const upCount = checks.filter((check) => check.status === 'healthy' || check.status === 'degraded').length;
+    return (upCount / checks.length) * 100;
+};
+const calculateRecoveryTime = (checks) => {
+    if (checks.length < 2)
+        return 0;
+    const sorted = [...checks].sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
+    let currentDownAt = null;
+    let maxRecoverySeconds = 0;
+    for (const check of sorted) {
+        const isDown = check.status === 'critical';
+        if (isDown && currentDownAt === null) {
+            currentDownAt = check.timestamp;
+        }
+        if (!isDown && currentDownAt !== null) {
+            const recoverySeconds = (check.timestamp.getTime() - currentDownAt.getTime()) / 1000;
+            maxRecoverySeconds = Math.max(maxRecoverySeconds, recoverySeconds);
+            currentDownAt = null;
+        }
+    }
+    return Math.round(maxRecoverySeconds);
+};
+const loadHealthChecks = async (workerName, range) => {
+    const history = HealthMonitor.getHealthHistory(workerName);
+    const filtered = history.filter((check) => check.timestamp.getTime() >= range.start.getTime() &&
+        check.timestamp.getTime() <= range.end.getTime());
+    if (filtered.length > 0)
+        return filtered;
+    try {
+        const current = HealthMonitor.getCurrentHealth(workerName);
+        return current ? [current] : [];
+    }
+    catch (error) {
+        Logger.debug('Failed to get current health for SLA check', error);
+        return [];
+    }
+};
+const getMetricPoints = async (workerName, metricType, range) => {
+    const entry = await WorkerMetrics.query({
+        workerName,
+        metricType,
+        granularity: 'hourly',
+        startDate: range.start,
+        endDate: range.end,
+    });
+    return entry.points;
+};
+const collectSlaMetrics = async (workerName, range) => {
+    const [durationPoints, processedAgg, errorAgg, healthChecks] = await Promise.all([
+        getMetricPoints(workerName, 'duration', range),
+        WorkerMetrics.aggregate({
+            workerName,
+            metricType: 'processed',
+            granularity: 'hourly',
+            startDate: range.start,
+            endDate: range.end,
+        }),
+        WorkerMetrics.aggregate({
+            workerName,
+            metricType: 'errors',
+            granularity: 'hourly',
+            startDate: range.start,
+            endDate: range.end,
+        }),
+        loadHealthChecks(workerName, range),
+    ]);
+    const latencyP99 = quantile(durationPoints, 0.99);
+    const minutes = Math.max(1, (range.end.getTime() - range.start.getTime()) / 60000);
+    const throughput = processedAgg.total / minutes;
+    const errorRate = processedAgg.total > 0 ? (errorAgg.total / processedAgg.total) * 100 : 0;
+    const availability = calculateAvailability(healthChecks);
+    const recoveryTime = calculateRecoveryTime(healthChecks);
+    return { latencyP99, throughput, availability, errorRate, recoveryTime };
+};
+const buildChecks = (metrics, config) => ({
+    latencyP99: {
+        value: metrics.latencyP99,
+        threshold: config.metrics.maxLatencyP99,
+        status: evaluateMaxThreshold(metrics.latencyP99, config.metrics.maxLatencyP99),
+    },
+    throughput: {
+        value: metrics.throughput,
+        threshold: config.metrics.minThroughput,
+        status: evaluateMinThreshold(metrics.throughput, config.metrics.minThroughput),
+    },
+    availability: {
+        value: metrics.availability,
+        threshold: config.metrics.minAvailability,
+        status: evaluateMinThreshold(metrics.availability, config.metrics.minAvailability),
+    },
+    errorRate: {
+        value: metrics.errorRate,
+        threshold: config.metrics.maxErrorRate,
+        status: evaluateMaxThreshold(metrics.errorRate, config.metrics.maxErrorRate),
+    },
+    recoveryTime: {
+        value: metrics.recoveryTime,
+        threshold: config.metrics.maxRecoveryTime,
+        status: evaluateMaxThreshold(metrics.recoveryTime, config.metrics.maxRecoveryTime),
+    },
+});
+const buildSlaStatus = (checks) => {
+    const hasFailures = Object.values(checks).some((check) => check.status === 'fail');
+    if (hasFailures)
+        return 'breach';
+    const hasWarnings = Object.values(checks).some((check) => check.status === 'warn');
+    return hasWarnings ? 'warning' : 'compliant';
+};
+const buildViolation = (params) => ({
+    id: generateUuid(),
+    workerName: params.workerName,
+    metric: params.metric,
+    expected: params.expected,
+    actual: params.actual,
+    timestamp: new Date(),
+    severity: params.severity,
+    message: params.message,
+});
+/**
+ * SLA Monitor - Sealed namespace
+ */
+export const SLAMonitor = Object.freeze({
+    /**
+     * Define SLA for a worker
+     */
+    defineSLA(config) {
+        slaConfigs.set(config.workerName, { ...config });
+        Logger.info(`SLA defined for worker "${config.workerName}"`);
+    },
+    /**
+     * Check SLA compliance for a worker
+     */
+    async checkCompliance(workerName) {
+        const config = slaConfigs.get(workerName);
+        if (!config) {
+            throw ErrorFactory.createNotFoundError(`SLA config not found for worker "${workerName}"`);
+        }
+        const range = {
+            start: new Date(Date.now() - DEFAULT_LOOKBACK_MS),
+            end: new Date(),
+        };
+        const metrics = await collectSlaMetrics(workerName, range);
+        const checks = buildChecks(metrics, config);
+        const status = buildSlaStatus(checks);
+        Object.entries(checks)
+            .filter(([, detail]) => detail.status === 'fail')
+            .forEach(([metricKey, detail]) => {
+            const violation = buildViolation({
+                workerName,
+                metric: metricKey,
+                expected: detail.threshold,
+                actual: detail.value,
+                severity: 'critical',
+                message: `SLA breach for ${metricKey}: ${detail.value} (expected ${detail.threshold})`,
+            });
+            storeViolation(violation);
+            SLAMonitor.alertOnViolation(violation);
+        });
+        return {
+            workerName,
+            status,
+            evaluatedAt: new Date(),
+            checks,
+        };
+    },
+    /**
+     * Get SLA violations for a worker
+     */
+    getViolations(workerName, timeRange) {
+        const history = violationHistory.get(workerName) ?? [];
+        return history.filter((violation) => violation.timestamp.getTime() >= timeRange.start.getTime() &&
+            violation.timestamp.getTime() <= timeRange.end.getTime());
+    },
+    /**
+     * Get SLA compliance report
+     */
+    getComplianceReport(timeRange) {
+        const violations = [];
+        const perWorker = [];
+        for (const [workerName] of slaConfigs.entries()) {
+            const workerViolations = SLAMonitor.getViolations(workerName, timeRange);
+            violations.push(...workerViolations);
+            const totalChecks = slaConfigs.size * 5;
+            const breachCount = workerViolations.length;
+            const complianceRate = totalChecks > 0 ? (totalChecks - breachCount) / totalChecks : 1;
+            perWorker.push({
+                workerName,
+                violations: breachCount,
+                complianceRate,
+            });
+        }
+        const totalChecks = slaConfigs.size * 5;
+        const totalViolations = violations.length;
+        const complianceRate = totalChecks > 0 ? (totalChecks - totalViolations) / totalChecks : 1;
+        return {
+            generatedAt: new Date(),
+            period: timeRange,
+            totalWorkers: slaConfigs.size,
+            totalChecks,
+            totalViolations,
+            complianceRate,
+            violations,
+            perWorker,
+        };
+    },
+    /**
+     * Alert on SLA violation
+     */
+    alertOnViolation(violation) {
+        const config = slaConfigs.get(violation.workerName);
+        if (!config)
+            return;
+        const lastAlert = lastAlertAt.get(violation.workerName);
+        const cooldownMs = config.alerting.cooldown * 60 * 1000;
+        if (lastAlert && Date.now() - lastAlert.getTime() < cooldownMs) {
+            return;
+        }
+        lastAlertAt.set(violation.workerName, new Date());
+        Logger.warn(`SLA violation for ${violation.workerName}`, {
+            metric: violation.metric,
+            expected: violation.expected,
+            actual: violation.actual,
+            severity: violation.severity,
+            channels: config.alerting.channels,
+        });
+    },
+});
+export default SLAMonitor;

package/dist/WorkerFactory.d.ts

@@ -0,0 +1,193 @@
+/**
+ * Worker Factory
+ * Central factory for creating workers with all advanced features
+ * Sealed namespace for immutability
+ */
+import { type IDatabase, type RedisConfig } from '@zintrust/core';
+import { Worker, type Job, type WorkerOptions } from 'bullmq';
+import { type AutoScalerConfig } from './AutoScaler';
+import { type ComplianceConfig } from './ComplianceManager';
+import { type RetentionPolicy } from './DeadLetterQueue';
+import { type ObservabilityConfig } from './Observability';
+import { type WorkerRecord } from './storage/WorkerStore';
+export declare const WorkerCreationStatus: {
+    readonly CREATING: "creating";
+    readonly CONNECTING: "connecting";
+    readonly STARTING: "starting";
+    readonly RUNNING: "running";
+    readonly FAILED: "failed";
+    readonly STOPPED: "stopped";
+};
+export type WorkerCreationStatus = (typeof WorkerCreationStatus)[keyof typeof WorkerCreationStatus];
+export type WorkerFactoryConfig = {
+    name: string;
+    version?: string;
+    queueName: string;
+    processor: (job: Job) => Promise<unknown>;
+    processorPath?: string;
+    options?: WorkerOptions;
+    autoStart?: boolean;
+    infrastructure?: {
+        redis?: RedisConfigInput;
+        persistence?: WorkerPersistenceConfig;
+        deadLetterQueue?: {
+            redis?: RedisConfigInput;
+            policy: RetentionPolicy;
+        };
+        compliance?: {
+            redis?: RedisConfigInput;
+            config?: Partial<ComplianceConfig>;
+        };
+        observability?: ObservabilityConfigInput;
+        autoScaler?: AutoScalerConfig;
+    };
+    features?: {
+        clustering?: boolean;
+        metrics?: boolean;
+        autoScaling?: boolean;
+        circuitBreaker?: boolean;
+        deadLetterQueue?: boolean;
+        resourceMonitoring?: boolean;
+        compliance?: boolean;
+        observability?: boolean;
+        plugins?: boolean;
+        versioning?: boolean;
+        datacenterOrchestration?: boolean;
+    };
+    datacenter?: {
+        primaryRegion: string;
+        secondaryRegions?: string[];
+        affinityRules?: {
+            preferLocal?: boolean;
+            maxLatency?: number;
+            avoidRegions?: string[];
+        };
+    };
+};
+export type WorkerInstance = {
+    worker: Worker;
+    config: WorkerFactoryConfig;
+    startedAt: Date;
+    status: WorkerCreationStatus;
+    lastHealthCheck?: Date;
+    connectionState?: 'disconnected' | 'connecting' | 'connected' | 'error';
+};
+type RedisEnvConfig = {
+    env: true;
+    host?: string;
+    port?: string;
+    password?: string;
+    db?: string;
+};
+type RedisConfigInput = RedisConfig | RedisEnvConfig;
+export type WorkerPersistenceConfig = {
+    driver: 'memory';
+} | {
+    driver: 'redis';
+    redis?: RedisConfigInput;
+    keyPrefix?: string;
+} | {
+    driver: 'database';
+    client?: IDatabase | string;
+    connection?: string;
+    table?: string;
+};
+type ObservabilityConfigInput = ObservabilityConfig | {
+    enabled?: boolean;
+    prometheus?: Partial<ObservabilityConfig['prometheus']>;
+    openTelemetry?: Partial<ObservabilityConfig['openTelemetry']>;
+    datadog?: Partial<ObservabilityConfig['datadog']>;
+};
+type ProcessorResolver = (name: string) => WorkerFactoryConfig['processor'] | undefined | Promise<WorkerFactoryConfig['processor'] | undefined>;
+/**
+ * Worker Factory - Sealed namespace
+ */
+export declare const WorkerFactory: Readonly<{
+    registerProcessor: (name: string, processor: WorkerFactoryConfig["processor"]) => void;
+    registerProcessors: (processors: Record<string, WorkerFactoryConfig["processor"]>) => void;
+    registerProcessorPaths: (paths: Record<string, string>) => void;
+    registerProcessorResolver: (resolver: ProcessorResolver) => void;
+    resolveProcessorPath: (modulePath: string) => Promise<WorkerFactoryConfig["processor"] | undefined>;
+    /**
+     * Create new worker with full setup
+     */
+    create(config: WorkerFactoryConfig): Promise<Worker>;
+    /**
+     * Get worker instance
+     */
+    get(name: string): WorkerInstance | null;
+    /**
+     * Update worker status directly (used by HealthMonitor)
+     */
+    updateStatus(name: string, status: string, error?: Error | string): Promise<void>;
+    /**
+     * Stop worker
+     */
+    stop(name: string, persistenceOverride?: WorkerPersistenceConfig): Promise<void>;
+    /**
+     * Restart worker
+     */
+    restart(name: string, persistenceOverride?: WorkerPersistenceConfig): Promise<void>;
+    /**
+     * Pause worker
+     */
+    pause(name: string, persistenceOverride?: WorkerPersistenceConfig): Promise<void>;
+    /**
+     * Resume worker
+     */
+    resume(name: string, persistenceOverride?: WorkerPersistenceConfig): Promise<void>;
+    /**
+     * Update auto-start for persisted worker
+     */
+    setAutoStart(name: string, autoStart: boolean, persistenceOverride?: WorkerPersistenceConfig): Promise<void>;
+    /**
+     * Update persisted worker record and in-memory config if running.
+     */
+    update(name: string, patch: Partial<WorkerRecord> | WorkerRecord, persistenceOverride?: WorkerPersistenceConfig): Promise<void>;
+    /**
+     * Start worker
+     */
+    start(name: string, persistenceOverride?: WorkerPersistenceConfig): Promise<void>;
+    /**
+     * List all workers
+     */
+    list(): string[];
+    /**
+     * List all persisted workers
+     */
+    listPersisted(persistenceOverride?: WorkerPersistenceConfig, options?: {
+        offset?: number;
+        limit?: number;
+        search?: string;
+    }): Promise<string[]>;
+    listPersistedRecords(persistenceOverride?: WorkerPersistenceConfig, options?: {
+        offset?: number;
+        limit?: number;
+        search?: string;
+    }): Promise<WorkerRecord[]>;
+    /**
+     * Start a worker from persisted storage when it is not registered.
+     */
+    startFromPersisted(name: string, persistenceOverride?: WorkerPersistenceConfig): Promise<void>;
+    /**
+     * Get persisted worker record
+     */
+    getPersisted(name: string, persistenceOverride?: WorkerPersistenceConfig): Promise<WorkerRecord | null>;
+    /**
+     * Remove worker
+     */
+    remove(name: string, persistenceOverride?: WorkerPersistenceConfig): Promise<void>;
+    /**
+     * Get worker metrics
+     */
+    getMetrics(name: string): Promise<unknown>;
+    /**
+     * Get worker health
+     */
+    getHealth(name: string): Promise<unknown>;
+    /**
+     * Shutdown all workers
+     */
+    shutdown(): Promise<void>;
+}>;
+export {};