@poolzin/pool-bot 2026.3.9 → 2026.3.10

package/dist/telemetry/alert-engine.js

@@ -0,0 +1,258 @@
+import { createSubsystemLogger } from "../logging/subsystem.js";
+const log = createSubsystemLogger("telemetry:alerts");
+export class AlertEngine {
+    rules = new Map();
+    states = new Map();
+    onAlertCallback;
+    constructor(rules = []) {
+        for (const rule of rules) {
+            this.addRule(rule);
+        }
+    }
+    addRule(rule) {
+        this.rules.set(rule.id, rule);
+        this.states.set(rule.id, { consecutiveCount: 0 });
+        log.debug(`Added alert rule: ${rule.id} (${rule.name})`);
+    }
+    removeRule(ruleId) {
+        this.states.delete(ruleId);
+        return this.rules.delete(ruleId);
+    }
+    getRules() {
+        return Array.from(this.rules.values());
+    }
+    onAlert(callback) {
+        this.onAlertCallback = callback;
+    }
+    evaluate(snapshot) {
+        const alerts = [];
+        const now = Date.now();
+        for (const rule of this.rules.values()) {
+            if (!rule.enabled)
+                continue;
+            const alert = this.evaluateRule(rule, snapshot, now);
+            if (alert) {
+                alerts.push(alert);
+                this.onAlertCallback?.(alert);
+            }
+        }
+        return alerts;
+    }
+    evaluateRule(rule, snapshot, now) {
+        const state = this.states.get(rule.id);
+        if (!state)
+            return null;
+        // Find matching metrics
+        const matchingMetrics = snapshot.metrics.filter((m) => {
+            if (m.name !== rule.threshold.metric)
+                return false;
+            if (rule.attributes) {
+                for (const [key, value] of Object.entries(rule.attributes)) {
+                    if (m.attributes?.[key] !== value)
+                        return false;
+                }
+            }
+            return true;
+        });
+        if (matchingMetrics.length === 0) {
+            // Reset state if no matching metrics
+            state.consecutiveCount = 0;
+            state.triggeredAt = undefined;
+            return null;
+        }
+        // Check if any metric violates threshold
+        let violated = false;
+        let violatingMetric = null;
+        for (const metric of matchingMetrics) {
+            if (this.checkThreshold(metric.value, rule.threshold)) {
+                violated = true;
+                violatingMetric = metric;
+                break;
+            }
+        }
+        if (!violated) {
+            // Reset state if threshold no longer violated
+            state.consecutiveCount = 0;
+            state.triggeredAt = undefined;
+            return null;
+        }
+        // Track consecutive violations
+        state.consecutiveCount++;
+        if (!state.triggeredAt) {
+            state.triggeredAt = now;
+        }
+        // Check duration requirement
+        const durationMs = rule.threshold.durationMs ?? 0;
+        if (now - state.triggeredAt < durationMs) {
+            return null; // Haven't met duration requirement yet
+        }
+        // Check cooldown
+        const cooldownMs = rule.threshold.cooldownMs ?? 60000;
+        if (state.lastAlertAt && now - state.lastAlertAt < cooldownMs) {
+            return null; // Still in cooldown
+        }
+        // Trigger alert
+        state.lastAlertAt = now;
+        return {
+            id: `${rule.id}-${now}`,
+            ruleId: rule.id,
+            name: rule.name,
+            severity: rule.severity,
+            metric: rule.threshold.metric,
+            value: violatingMetric.value,
+            threshold: rule.threshold.value,
+            operator: rule.threshold.operator,
+            timestamp: now,
+            message: this.buildAlertMessage(rule, violatingMetric.value),
+            attributes: violatingMetric.attributes,
+        };
+    }
+    checkThreshold(value, threshold) {
+        switch (threshold.operator) {
+            case "gt":
+                return value > threshold.value;
+            case "gte":
+                return value >= threshold.value;
+            case "lt":
+                return value < threshold.value;
+            case "lte":
+                return value <= threshold.value;
+            case "eq":
+                return value === threshold.value;
+            default:
+                return false;
+        }
+    }
+    buildAlertMessage(rule, value) {
+        const opMap = {
+            gt: ">",
+            gte: ">=",
+            lt: "<",
+            lte: "<=",
+            eq: "=",
+        };
+        return `${rule.name}: ${rule.threshold.metric} = ${value} ${opMap[rule.threshold.operator]} ${rule.threshold.value}`;
+    }
+}
+// Predefined alert rules for common Pool Bot scenarios
+export const defaultAlertRules = [
+    // Cron Job Alerts
+    {
+        id: "cron-high-failure-rate",
+        name: "High Cron Job Failure Rate",
+        description: "Triggers when cron job failures exceed 3 in 5 minutes",
+        enabled: true,
+        severity: "warning",
+        threshold: {
+            metric: "poolbot.cron.job_failures",
+            operator: "gt",
+            value: 3,
+            durationMs: 300000, // 5 minutes
+            cooldownMs: 600000, // 10 minutes
+        },
+    },
+    {
+        id: "cron-slow-jobs",
+        name: "Slow Cron Job Execution",
+        description: "Triggers when cron job duration exceeds 60 seconds",
+        enabled: true,
+        severity: "info",
+        threshold: {
+            metric: "poolbot.cron.job_duration_ms",
+            operator: "gt",
+            value: 60000,
+            durationMs: 60000, // 1 minute
+            cooldownMs: 300000, // 5 minutes
+        },
+    },
+    {
+        id: "cron-job-stuck",
+        name: "Cron Job Possibly Stuck",
+        description: "Triggers when cron job duration exceeds 5 minutes",
+        enabled: true,
+        severity: "critical",
+        threshold: {
+            metric: "poolbot.cron.job_duration_ms",
+            operator: "gt",
+            value: 300000,
+            durationMs: 60000,
+            cooldownMs: 300000,
+        },
+    },
+    // Security Scan Alerts (HexStrike)
+    {
+        id: "security-scan-failures",
+        name: "Security Scan Failures",
+        description: "Triggers when security scans fail",
+        enabled: true,
+        severity: "warning",
+        threshold: {
+            metric: "poolbot.security.scan_failures",
+            operator: "gt",
+            value: 2,
+            durationMs: 300000,
+            cooldownMs: 600000,
+        },
+    },
+    {
+        id: "security-high-latency",
+        name: "HexStrike API High Latency",
+        description: "Triggers when HexStrike API latency exceeds 5 seconds",
+        enabled: true,
+        severity: "info",
+        threshold: {
+            metric: "poolbot.security.api_latency_ms",
+            operator: "gt",
+            value: 5000,
+            durationMs: 120000,
+            cooldownMs: 300000,
+        },
+    },
+    // Swarm Alerts
+    {
+        id: "swarm-high-failure-rate",
+        name: "Swarm Task Failure Rate",
+        description: "Triggers when swarm tasks fail frequently",
+        enabled: true,
+        severity: "warning",
+        threshold: {
+            metric: "poolbot.swarm.task_failures",
+            operator: "gt",
+            value: 5,
+            durationMs: 600000,
+            cooldownMs: 600000,
+        },
+    },
+    // Gateway Health Alerts
+    {
+        id: "gateway-high-memory",
+        name: "High Gateway Memory Usage",
+        description: "Triggers when gateway memory usage exceeds 512MB",
+        enabled: true,
+        severity: "critical",
+        threshold: {
+            metric: "poolbot.gateway.memory_usage_bytes",
+            operator: "gt",
+            value: 536870912, // 512MB
+            durationMs: 120000,
+            cooldownMs: 300000,
+        },
+    },
+    {
+        id: "gateway-high-cpu",
+        name: "High Gateway CPU Usage",
+        description: "Triggers when gateway CPU usage exceeds 80%",
+        enabled: true,
+        severity: "warning",
+        threshold: {
+            metric: "poolbot.gateway.cpu_usage_percent",
+            operator: "gt",
+            value: 80,
+            durationMs: 180000,
+            cooldownMs: 300000,
+        },
+    },
+];
+export function createAlertEngine(customRules) {
+    return new AlertEngine([...defaultAlertRules, ...(customRules ?? [])]);
+}

package/dist/telemetry/cron-instrumentation.js

@@ -0,0 +1,49 @@
+import { recordCounter, recordGauge, recordHistogram, withInstrumentation, } from "../telemetry/instrumentation.js";
+export function instrumentCronOperation(operation, fn, attributes) {
+    return withInstrumentation({
+        operation,
+        component: "cron",
+        attributes,
+    }, fn);
+}
+export function recordCronJobExecution(metrics) {
+    recordCounter("poolbot.cron.jobs_executed", 1, {
+        job_id: metrics.jobId,
+        job_name: metrics.jobName,
+        schedule: metrics.schedule,
+        status: metrics.success ? "success" : "failure",
+    });
+    recordHistogram("poolbot.cron.job_duration_ms", metrics.durationMs, {
+        job_id: metrics.jobId,
+        job_name: metrics.jobName,
+    });
+    if (!metrics.success) {
+        recordCounter("poolbot.cron.job_failures", 1, {
+            job_id: metrics.jobId,
+            job_name: metrics.jobName,
+            error_type: metrics.errorType ?? "unknown",
+        });
+    }
+}
+export function recordCronJobScheduled(jobId, jobName, schedule) {
+    recordCounter("poolbot.cron.jobs_scheduled", 1, {
+        job_id: jobId,
+        job_name: jobName,
+        schedule,
+    });
+}
+export function recordCronJobCancelled(jobId, jobName) {
+    recordCounter("poolbot.cron.jobs_cancelled", 1, {
+        job_id: jobId,
+        job_name: jobName,
+    });
+}
+export function updateActiveCronJobsGauge(count) {
+    recordGauge("poolbot.cron.active_jobs", count);
+}
+export function recordCronQueueDepth(depth) {
+    recordGauge("poolbot.cron.queue_depth", depth);
+}
+export function recordCronOverdueJobs(count) {
+    recordGauge("poolbot.cron.overdue_jobs", count);
+}

package/dist/telemetry/gateway-instrumentation.js

@@ -0,0 +1,80 @@
+import { recordCounter, recordHistogram, withInstrumentation, } from "../telemetry/instrumentation.js";
+export function instrumentGatewayRequest(operation, fn, attributes) {
+    return withInstrumentation({
+        operation,
+        component: "gateway",
+        attributes,
+    }, fn);
+}
+export function recordGatewayRequest(metrics) {
+    recordCounter("poolbot.gateway.requests", 1, {
+        method: metrics.method,
+        path: metrics.path,
+        status_code: String(metrics.statusCode),
+    });
+    recordHistogram("poolbot.gateway.request_duration_ms", metrics.durationMs, {
+        method: metrics.method,
+        path: metrics.path,
+    });
+    if (metrics.statusCode >= 500) {
+        recordCounter("poolbot.gateway.server_errors", 1, {
+            path: metrics.path,
+        });
+    }
+    else if (metrics.statusCode >= 400) {
+        recordCounter("poolbot.gateway.client_errors", 1, {
+            path: metrics.path,
+            status_code: String(metrics.statusCode),
+        });
+    }
+}
+export function createTelemetryMiddleware() {
+    return (req, res, next) => {
+        const startTime = Date.now();
+        const path = req.url ?? "/unknown";
+        const method = req.method ?? "UNKNOWN";
+        recordCounter("poolbot.gateway.active_requests", 1, {
+            method,
+            path,
+        });
+        const originalEnd = res.end.bind(res);
+        res.end = function (...args) {
+            const duration = Date.now() - startTime;
+            recordCounter("poolbot.gateway.active_requests", -1, {
+                method,
+                path,
+            });
+            recordGatewayRequest({
+                method,
+                path,
+                statusCode: res.statusCode,
+                durationMs: duration,
+                userAgent: req.headers["user-agent"],
+            });
+            return originalEnd(...args);
+        };
+        next();
+    };
+}
+export function instrumentWebSocketConnection(connectionId) {
+    recordCounter("poolbot.gateway.ws.connections", 1, {
+        connection_id: connectionId,
+    });
+}
+export function recordWebSocketMessage(connectionId, messageType, sizeBytes) {
+    recordCounter("poolbot.gateway.ws.messages", 1, {
+        connection_id: connectionId,
+        message_type: messageType,
+    });
+    recordHistogram("poolbot.gateway.ws.message_size_bytes", sizeBytes, {
+        message_type: messageType,
+    });
+}
+export function recordWebSocketDisconnection(connectionId, durationMs) {
+    recordCounter("poolbot.gateway.ws.disconnections", 1, {
+        connection_id: connectionId,
+    });
+    recordHistogram("poolbot.gateway.ws.connection_duration_ms", durationMs, {
+        connection_id: connectionId,
+    });
+}

package/dist/telemetry/instrumentation.js

@@ -0,0 +1,66 @@
+import { getGlobalTelemetryService } from "../telemetry/service.js";
+export function withInstrumentation(options, fn) {
+    const telemetry = getGlobalTelemetryService();
+    if (!telemetry?.isEnabled()) {
+        return Promise.resolve(fn());
+    }
+    const spanName = `${options.component}.${options.operation}`;
+    const attributes = {
+        component: options.component,
+        operation: options.operation,
+        ...options.attributes,
+    };
+    telemetry.recordCounter(`poolbot.${options.component}.operations`, 1, {
+        operation: options.operation,
+    });
+    const startTime = Date.now();
+    return telemetry.withSpan(spanName, async () => {
+        try {
+            const result = await fn();
+            const duration = Date.now() - startTime;
+            telemetry.recordHistogram(`poolbot.${options.component}.duration_ms`, duration, {
+                operation: options.operation,
+            });
+            telemetry.recordCounter(`poolbot.${options.component}.success`, 1, {
+                operation: options.operation,
+            });
+            return result;
+        }
+        catch (error) {
+            const duration = Date.now() - startTime;
+            telemetry.recordHistogram(`poolbot.${options.component}.duration_ms`, duration, {
+                operation: options.operation,
+                error: "true",
+            });
+            telemetry.recordCounter(`poolbot.${options.component}.errors`, 1, {
+                operation: options.operation,
+                error_type: error instanceof Error ? error.name : "unknown",
+            });
+            throw error;
+        }
+    }, attributes);
+}
+export function recordMetric(name, value, attributes) {
+    const telemetry = getGlobalTelemetryService();
+    if (!telemetry?.isEnabled())
+        return;
+    telemetry.recordHistogram(name, value, attributes);
+}
+export function recordCounter(name, value = 1, attributes) {
+    const telemetry = getGlobalTelemetryService();
+    if (!telemetry?.isEnabled())
+        return;
+    telemetry.recordCounter(name, value, attributes);
+}
+export function recordGauge(name, value, attributes) {
+    const telemetry = getGlobalTelemetryService();
+    if (!telemetry?.isEnabled())
+        return;
+    telemetry.recordGauge(name, value, attributes);
+}
+export function recordHistogram(name, value, attributes) {
+    const telemetry = getGlobalTelemetryService();
+    if (!telemetry?.isEnabled())
+        return;
+    telemetry.recordHistogram(name, value, attributes);
+}