opensentinel 3.1.1 → 3.7.0

package/dist/alerting-LK7VVYTX.js

@@ -0,0 +1,699 @@
+import {
+  db
+} from "./chunk-5BTVJR7R.js";
+import "./chunk-4KIHDIXZ.js";
+import {
+  errorLogs,
+  metrics
+} from "./chunk-ZIBRVA3Y.js";
+import "./chunk-UP2VWCW5.js";
+
+// src/core/observability/alerting.ts
+import { eq, and, gte, lte, desc } from "drizzle-orm";
+import { EventEmitter } from "events";
+var TOKEN_COST_PER_MILLION_INPUT = 3;
+var TOKEN_COST_PER_MILLION_OUTPUT = 15;
+var activeAlerts = /* @__PURE__ */ new Map();
+var alertRules = /* @__PURE__ */ new Map();
+var healthCheckResults = /* @__PURE__ */ new Map();
+var alertHistory = [];
+var alertEmitter = new EventEmitter();
+var channelConfig = {
+  console: { enabled: true },
+  telegram: { enabled: false },
+  webhook: { enabled: false },
+  email: { enabled: false },
+  database: { enabled: true }
+};
+var historyLoaded = false;
+async function loadAlertHistoryFromDb() {
+  if (historyLoaded) return;
+  try {
+    const rows = await db.select().from(errorLogs).where(eq(errorLogs.source, "alerting")).orderBy(desc(errorLogs.createdAt)).limit(500);
+    for (const row of rows.reverse()) {
+      const alertType = row.errorType?.replace("Alert:", "") || "system";
+      const severity = row.errorCode || "info";
+      const [title, ...msgParts] = (row.message || "").split(": ");
+      const alert = {
+        id: String(row.id),
+        type: alertType,
+        severity,
+        title: title || "Alert",
+        message: msgParts.join(": ") || row.message || "",
+        source: "alert_rule",
+        timestamp: new Date(row.createdAt),
+        metadata: row.context || {},
+        acknowledged: false,
+        resolved: true
+        // historical alerts are resolved
+      };
+      alertHistory.push(alert);
+    }
+    historyLoaded = true;
+    if (alertHistory.length > 0) {
+      console.log(`[Alerting] Loaded ${alertHistory.length} historical alerts from DB`);
+    }
+  } catch {
+  }
+}
+function initializeDefaultRules() {
+  addAlertRule({
+    id: "error-spike",
+    name: "Error Spike Detection",
+    description: "Triggers when error rate exceeds normal levels",
+    enabled: true,
+    type: "error_spike",
+    condition: {
+      metric: "error_count",
+      operator: "gt",
+      threshold: 10,
+      windowMinutes: 5,
+      aggregation: "count"
+    },
+    severity: "error",
+    cooldownMinutes: 15,
+    channels: ["console", "database"],
+    triggerCount: 0
+  });
+  addAlertRule({
+    id: "high-latency",
+    name: "High Response Latency",
+    description: "Triggers when average response time exceeds threshold",
+    enabled: true,
+    type: "performance",
+    condition: {
+      metric: "response_latency",
+      operator: "gt",
+      threshold: 1e4,
+      // 10 seconds
+      windowMinutes: 5,
+      aggregation: "avg"
+    },
+    severity: "warning",
+    cooldownMinutes: 10,
+    channels: ["console", "database"],
+    triggerCount: 0
+  });
+  addAlertRule({
+    id: "daily-cost-threshold",
+    name: "Daily Cost Threshold",
+    description: "Triggers when daily API costs exceed threshold",
+    enabled: true,
+    type: "cost_threshold",
+    condition: {
+      metric: "cost",
+      operator: "gt",
+      threshold: 50,
+      // $50
+      windowMinutes: 1440
+      // 24 hours
+    },
+    severity: "warning",
+    cooldownMinutes: 60,
+    channels: ["console", "database"],
+    triggerCount: 0
+  });
+  addAlertRule({
+    id: "memory-anomaly",
+    name: "Memory Usage Anomaly",
+    description: "Triggers when memory usage is abnormally high",
+    enabled: true,
+    type: "anomaly",
+    condition: {
+      metric: "memory_usage",
+      operator: "anomaly",
+      windowMinutes: 60
+    },
+    severity: "warning",
+    cooldownMinutes: 30,
+    channels: ["console", "database"],
+    triggerCount: 0
+  });
+  addAlertRule({
+    id: "quota-80-percent",
+    name: "80% Quota Usage",
+    description: "Triggers when usage reaches 80% of quota",
+    enabled: true,
+    type: "quota_warning",
+    condition: {
+      metric: "quota_usage",
+      operator: "gte",
+      threshold: 80
+    },
+    severity: "warning",
+    cooldownMinutes: 60,
+    channels: ["console", "database"],
+    triggerCount: 0
+  });
+  addAlertRule({
+    id: "tool-failure-rate",
+    name: "High Tool Failure Rate",
+    description: "Triggers when tool failure rate is high",
+    enabled: true,
+    type: "performance",
+    condition: {
+      metric: "tool_failure",
+      operator: "gt",
+      threshold: 5,
+      windowMinutes: 10,
+      aggregation: "count"
+    },
+    severity: "warning",
+    cooldownMinutes: 15,
+    channels: ["console", "database"],
+    triggerCount: 0
+  });
+}
+function addAlertRule(rule) {
+  alertRules.set(rule.id, rule);
+}
+function removeAlertRule(ruleId) {
+  return alertRules.delete(ruleId);
+}
+function getAlertRules() {
+  return Array.from(alertRules.values());
+}
+function setRuleEnabled(ruleId, enabled) {
+  const rule = alertRules.get(ruleId);
+  if (rule) {
+    rule.enabled = enabled;
+    return true;
+  }
+  return false;
+}
+function configureChannels(config) {
+  channelConfig = { ...channelConfig, ...config };
+}
+async function createAlert(type, severity, title, message, source, metadata) {
+  const alert = {
+    id: crypto.randomUUID(),
+    type,
+    severity,
+    title,
+    message,
+    source,
+    timestamp: /* @__PURE__ */ new Date(),
+    metadata,
+    acknowledged: false,
+    resolved: false
+  };
+  activeAlerts.set(alert.id, alert);
+  alertHistory.push(alert);
+  await dispatchAlert(alert);
+  alertEmitter.emit("alert", alert);
+  return alert;
+}
+async function dispatchAlert(alert) {
+  const promises = [];
+  if (channelConfig.console.enabled) {
+    promises.push(dispatchToConsole(alert));
+  }
+  if (channelConfig.database.enabled) {
+    promises.push(dispatchToDatabase(alert));
+  }
+  if (channelConfig.webhook.enabled && channelConfig.webhook.url) {
+    promises.push(dispatchToWebhook(alert, channelConfig.webhook.url, channelConfig.webhook.headers));
+  }
+  await Promise.allSettled(promises);
+}
+async function dispatchToConsole(alert) {
+  const severityColors = {
+    info: "\x1B[36m",
+    // Cyan
+    warning: "\x1B[33m",
+    // Yellow
+    error: "\x1B[31m",
+    // Red
+    critical: "\x1B[35m"
+    // Magenta
+  };
+  const reset = "\x1B[0m";
+  const color = severityColors[alert.severity];
+  console.log(`${color}[ALERT ${alert.severity.toUpperCase()}]${reset} ${alert.title}`);
+  console.log(`  Source: ${alert.source}`);
+  console.log(`  Message: ${alert.message}`);
+  if (alert.metadata) {
+    console.log(`  Metadata: ${JSON.stringify(alert.metadata)}`);
+  }
+}
+async function dispatchToDatabase(alert) {
+  await db.insert(errorLogs).values({
+    source: "alerting",
+    errorType: `Alert:${alert.type}`,
+    errorCode: alert.severity,
+    message: `${alert.title}: ${alert.message}`,
+    context: alert.metadata
+  });
+}
+async function dispatchToWebhook(alert, url, headers) {
+  try {
+    await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        ...headers
+      },
+      body: JSON.stringify(alert)
+    });
+  } catch (error) {
+    console.error("[Alerting] Failed to dispatch to webhook:", error);
+  }
+}
+function acknowledgeAlert(alertId, acknowledgedBy) {
+  const alert = activeAlerts.get(alertId);
+  if (alert) {
+    alert.acknowledged = true;
+    alert.acknowledgedAt = /* @__PURE__ */ new Date();
+    alert.acknowledgedBy = acknowledgedBy;
+    alertEmitter.emit("alert:acknowledged", alert);
+    return true;
+  }
+  return false;
+}
+function resolveAlert(alertId, resolvedBy) {
+  const alert = activeAlerts.get(alertId);
+  if (alert) {
+    alert.resolved = true;
+    alert.resolvedAt = /* @__PURE__ */ new Date();
+    alert.resolvedBy = resolvedBy;
+    activeAlerts.delete(alertId);
+    alertEmitter.emit("alert:resolved", alert);
+    return true;
+  }
+  return false;
+}
+function getActiveAlerts(type, severity) {
+  let alerts = Array.from(activeAlerts.values());
+  if (type) {
+    alerts = alerts.filter((a) => a.type === type);
+  }
+  if (severity) {
+    alerts = alerts.filter((a) => a.severity === severity);
+  }
+  return alerts.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+}
+function getAlertHistory(limit = 100) {
+  return alertHistory.slice(-limit).reverse();
+}
+function clearAlertHistory() {
+  alertHistory.length = 0;
+  activeAlerts.clear();
+  db.delete(errorLogs).where(eq(errorLogs.source, "alerting")).catch(() => {
+  });
+}
+async function evaluateAlertRules() {
+  const triggeredAlerts = [];
+  for (const rule of alertRules.values()) {
+    if (!rule.enabled) continue;
+    if (rule.lastTriggered) {
+      const cooldownMs = rule.cooldownMinutes * 60 * 1e3;
+      if (Date.now() - rule.lastTriggered.getTime() < cooldownMs) {
+        continue;
+      }
+    }
+    try {
+      const shouldTrigger = await evaluateCondition(rule.condition);
+      if (shouldTrigger) {
+        rule.lastTriggered = /* @__PURE__ */ new Date();
+        rule.triggerCount++;
+        const alert = await createAlert(
+          rule.type,
+          rule.severity,
+          rule.name,
+          rule.description,
+          "alert_rule",
+          { ruleId: rule.id, triggerCount: rule.triggerCount }
+        );
+        triggeredAlerts.push(alert);
+      }
+    } catch (error) {
+      console.error(`[Alerting] Error evaluating rule ${rule.id}:`, error);
+    }
+  }
+  return triggeredAlerts;
+}
+async function evaluateCondition(condition) {
+  if (condition.customCheck) {
+    return condition.customCheck();
+  }
+  if (condition.metric && condition.operator !== "anomaly") {
+    const windowMs = (condition.windowMinutes || 5) * 60 * 1e3;
+    const startDate = new Date(Date.now() - windowMs);
+    const results = await db.select().from(metrics).where(
+      and(
+        eq(metrics.name, condition.metric),
+        gte(metrics.timestamp, startDate)
+      )
+    );
+    if (results.length === 0) return false;
+    const values = results.map((r) => r.value);
+    let aggregatedValue;
+    switch (condition.aggregation || "avg") {
+      case "sum":
+        aggregatedValue = values.reduce((a, b) => a + b, 0);
+        break;
+      case "max":
+        aggregatedValue = Math.max(...values);
+        break;
+      case "min":
+        aggregatedValue = Math.min(...values);
+        break;
+      case "count":
+        aggregatedValue = values.length;
+        break;
+      case "avg":
+      default:
+        aggregatedValue = values.reduce((a, b) => a + b, 0) / values.length;
+    }
+    const threshold = condition.threshold || 0;
+    switch (condition.operator) {
+      case "gt":
+        return aggregatedValue > threshold;
+      case "lt":
+        return aggregatedValue < threshold;
+      case "eq":
+        return aggregatedValue === threshold;
+      case "gte":
+        return aggregatedValue >= threshold;
+      case "lte":
+        return aggregatedValue <= threshold;
+      default:
+        return false;
+    }
+  }
+  if (condition.operator === "anomaly" && condition.metric) {
+    const anomaly = await detectAnomaly(condition.metric, condition.windowMinutes || 60);
+    return anomaly.isAnomaly;
+  }
+  return false;
+}
+async function detectAnomaly(metricName, windowMinutes = 60) {
+  const windowMs = windowMinutes * 60 * 1e3;
+  const now = Date.now();
+  const baselineStart = new Date(now - 7 * 24 * 60 * 60 * 1e3);
+  const baselineEnd = new Date(now - windowMs);
+  const baselineResults = await db.select().from(metrics).where(
+    and(
+      eq(metrics.name, metricName),
+      gte(metrics.timestamp, baselineStart),
+      lte(metrics.timestamp, baselineEnd)
+    )
+  );
+  const currentStart = new Date(now - windowMs);
+  const currentResults = await db.select().from(metrics).where(
+    and(
+      eq(metrics.name, metricName),
+      gte(metrics.timestamp, currentStart)
+    )
+  );
+  if (baselineResults.length < 10 || currentResults.length === 0) {
+    return {
+      metric: metricName,
+      currentValue: currentResults.length > 0 ? currentResults.reduce((a, b) => a + b.value, 0) / currentResults.length : 0,
+      expectedValue: 0,
+      deviation: 0,
+      deviationPercent: 0,
+      isAnomaly: false,
+      confidence: 0,
+      timestamp: /* @__PURE__ */ new Date()
+    };
+  }
+  const baselineValues = baselineResults.map((r) => r.value);
+  const baselineMean = baselineValues.reduce((a, b) => a + b, 0) / baselineValues.length;
+  const baselineStdDev = Math.sqrt(
+    baselineValues.reduce((sum, val) => sum + Math.pow(val - baselineMean, 2), 0) / baselineValues.length
+  );
+  const currentValue = currentResults.reduce((a, b) => a + b.value, 0) / currentResults.length;
+  const zScore = baselineStdDev > 0 ? (currentValue - baselineMean) / baselineStdDev : 0;
+  const deviation = currentValue - baselineMean;
+  const deviationPercent = baselineMean !== 0 ? deviation / baselineMean * 100 : 0;
+  const isAnomaly = Math.abs(zScore) > 2;
+  const confidence = Math.min(
+    baselineResults.length / 100,
+    // More data = higher confidence
+    1 - 1 / (1 + Math.abs(zScore))
+    // Higher z-score = higher confidence
+  );
+  return {
+    metric: metricName,
+    currentValue,
+    expectedValue: baselineMean,
+    deviation,
+    deviationPercent,
+    isAnomaly,
+    confidence,
+    timestamp: /* @__PURE__ */ new Date()
+  };
+}
+async function calculateCostMetrics(period, startDate) {
+  const now = /* @__PURE__ */ new Date();
+  let periodStart;
+  switch (period) {
+    case "hourly":
+      periodStart = startDate || new Date(now.getTime() - 60 * 60 * 1e3);
+      break;
+    case "daily":
+      periodStart = startDate || new Date(now.getTime() - 24 * 60 * 60 * 1e3);
+      break;
+    case "weekly":
+      periodStart = startDate || new Date(now.getTime() - 7 * 24 * 60 * 60 * 1e3);
+      break;
+    case "monthly":
+      periodStart = startDate || new Date(now.getTime() - 30 * 24 * 60 * 60 * 1e3);
+      break;
+  }
+  const inputTokens = await db.select().from(metrics).where(
+    and(
+      eq(metrics.name, "token_usage_input"),
+      gte(metrics.timestamp, periodStart)
+    )
+  );
+  const outputTokens = await db.select().from(metrics).where(
+    and(
+      eq(metrics.name, "token_usage_output"),
+      gte(metrics.timestamp, periodStart)
+    )
+  );
+  const totalInput = inputTokens.reduce((sum, m) => sum + m.value, 0);
+  const totalOutput = outputTokens.reduce((sum, m) => sum + m.value, 0);
+  const inputCost = totalInput / 1e6 * TOKEN_COST_PER_MILLION_INPUT;
+  const outputCost = totalOutput / 1e6 * TOKEN_COST_PER_MILLION_OUTPUT;
+  const totalCost = inputCost + outputCost;
+  const breakdown = [];
+  const inputByTag = /* @__PURE__ */ new Map();
+  for (const m of inputTokens) {
+    const category = m.tags?.source || "general";
+    inputByTag.set(category, (inputByTag.get(category) || 0) + m.value);
+  }
+  for (const [category, tokens] of inputByTag) {
+    const cost = tokens / 1e6 * TOKEN_COST_PER_MILLION_INPUT;
+    breakdown.push({
+      category: `input:${category}`,
+      tokens,
+      cost,
+      percentage: totalCost > 0 ? cost / totalCost * 100 : 0
+    });
+  }
+  return {
+    totalTokens: totalInput + totalOutput,
+    inputTokens: totalInput,
+    outputTokens: totalOutput,
+    estimatedCost: totalCost,
+    period,
+    startDate: periodStart,
+    endDate: now,
+    breakdown
+  };
+}
+function registerHealthCheck(name, check) {
+  healthChecks.set(name, check);
+}
+var healthChecks = /* @__PURE__ */ new Map();
+async function runHealthChecks() {
+  const results = [];
+  for (const [name, check] of healthChecks) {
+    const startTime = Date.now();
+    try {
+      const result = await check();
+      const checkResult = {
+        name,
+        status: result.status,
+        message: result.message,
+        latencyMs: Date.now() - startTime,
+        lastCheck: /* @__PURE__ */ new Date(),
+        metadata: result.metadata
+      };
+      results.push(checkResult);
+      healthCheckResults.set(name, checkResult);
+      if (result.status === "unhealthy") {
+        await createAlert(
+          "health_check",
+          "error",
+          `Health Check Failed: ${name}`,
+          result.message,
+          "health_monitor",
+          result.metadata
+        );
+      } else if (result.status === "degraded") {
+        await createAlert(
+          "health_check",
+          "warning",
+          `Health Check Degraded: ${name}`,
+          result.message,
+          "health_monitor",
+          result.metadata
+        );
+      }
+    } catch (error) {
+      const checkResult = {
+        name,
+        status: "unhealthy",
+        message: error instanceof Error ? error.message : "Unknown error",
+        latencyMs: Date.now() - startTime,
+        lastCheck: /* @__PURE__ */ new Date()
+      };
+      results.push(checkResult);
+      healthCheckResults.set(name, checkResult);
+      await createAlert(
+        "health_check",
+        "error",
+        `Health Check Error: ${name}`,
+        checkResult.message,
+        "health_monitor"
+      );
+    }
+  }
+  return results;
+}
+function getHealthCheckResults() {
+  return Array.from(healthCheckResults.values());
+}
+function initializeDefaultHealthChecks() {
+  registerHealthCheck("database", async () => {
+    try {
+      await db.select().from(metrics).limit(1);
+      return { status: "healthy", message: "Database connection is healthy" };
+    } catch (error) {
+      return {
+        status: "unhealthy",
+        message: `Database connection failed: ${error instanceof Error ? error.message : "Unknown error"}`
+      };
+    }
+  });
+  registerHealthCheck("memory", async () => {
+    const memUsage = process.memoryUsage();
+    const heapUsedPercent = memUsage.heapUsed / memUsage.heapTotal * 100;
+    if (heapUsedPercent > 90) {
+      return {
+        status: "unhealthy",
+        message: `Heap usage critical: ${heapUsedPercent.toFixed(1)}%`,
+        metadata: { heapUsed: memUsage.heapUsed, heapTotal: memUsage.heapTotal }
+      };
+    } else if (heapUsedPercent > 75) {
+      return {
+        status: "degraded",
+        message: `Heap usage high: ${heapUsedPercent.toFixed(1)}%`,
+        metadata: { heapUsed: memUsage.heapUsed, heapTotal: memUsage.heapTotal }
+      };
+    }
+    return {
+      status: "healthy",
+      message: `Heap usage: ${heapUsedPercent.toFixed(1)}%`,
+      metadata: { heapUsed: memUsage.heapUsed, heapTotal: memUsage.heapTotal }
+    };
+  });
+  registerHealthCheck("error_rate", async () => {
+    const oneHourAgo = new Date(Date.now() - 60 * 60 * 1e3);
+    const errors = await db.select().from(errorLogs).where(gte(errorLogs.createdAt, oneHourAgo));
+    const errorCount = errors.length;
+    if (errorCount > 100) {
+      return {
+        status: "unhealthy",
+        message: `High error rate: ${errorCount} errors in the last hour`,
+        metadata: { errorCount }
+      };
+    } else if (errorCount > 50) {
+      return {
+        status: "degraded",
+        message: `Elevated error rate: ${errorCount} errors in the last hour`,
+        metadata: { errorCount }
+      };
+    }
+    return {
+      status: "healthy",
+      message: `Error rate normal: ${errorCount} errors in the last hour`,
+      metadata: { errorCount }
+    };
+  });
+}
+function startAlertingSystem(intervalMs = 6e4) {
+  initializeDefaultRules();
+  initializeDefaultHealthChecks();
+  setInterval(async () => {
+    try {
+      await evaluateAlertRules();
+      await runHealthChecks();
+    } catch (error) {
+      console.error("[Alerting] Error in periodic check:", error);
+    }
+  }, intervalMs);
+  console.log(`[Alerting] System started with ${intervalMs}ms check interval`);
+}
+function getAlertingStatus() {
+  const healthResults = getHealthCheckResults();
+  return {
+    activeAlerts: activeAlerts.size,
+    totalRules: alertRules.size,
+    enabledRules: Array.from(alertRules.values()).filter((r) => r.enabled).length,
+    healthChecks: healthChecks.size,
+    healthyChecks: healthResults.filter((r) => r.status === "healthy").length
+  };
+}
+function formatAlert(alert) {
+  const lines = [];
+  lines.push(`=== Alert: ${alert.title} ===`);
+  lines.push(`ID: ${alert.id}`);
+  lines.push(`Type: ${alert.type}`);
+  lines.push(`Severity: ${alert.severity.toUpperCase()}`);
+  lines.push(`Source: ${alert.source}`);
+  lines.push(`Time: ${alert.timestamp.toISOString()}`);
+  lines.push("");
+  lines.push(`Message: ${alert.message}`);
+  if (alert.metadata) {
+    lines.push("");
+    lines.push("Metadata:");
+    for (const [key, value] of Object.entries(alert.metadata)) {
+      lines.push(`  ${key}: ${JSON.stringify(value)}`);
+    }
+  }
+  lines.push("");
+  lines.push(`Status: ${alert.resolved ? "Resolved" : alert.acknowledged ? "Acknowledged" : "Active"}`);
+  return lines.join("\n");
+}
+export {
+  acknowledgeAlert,
+  addAlertRule,
+  alertEmitter,
+  calculateCostMetrics,
+  clearAlertHistory,
+  configureChannels,
+  createAlert,
+  detectAnomaly,
+  evaluateAlertRules,
+  formatAlert,
+  getActiveAlerts,
+  getAlertHistory,
+  getAlertRules,
+  getAlertingStatus,
+  getHealthCheckResults,
+  initializeDefaultHealthChecks,
+  initializeDefaultRules,
+  loadAlertHistoryFromDb,
+  registerHealthCheck,
+  removeAlertRule,
+  resolveAlert,
+  runHealthChecks,
+  setRuleEnabled,
+  startAlertingSystem
+};
+//# sourceMappingURL=alerting-LK7VVYTX.js.map