@thotischner/observability-mcp 1.4.1 → 1.6.0

package/dist/analysis/anomaly.d.ts

@@ -12,6 +12,95 @@ export interface AnomalyPoint {
     severity: AnomalySeverity;
 }
 export declare function detectAnomalyPoints(values: number[], threshold?: number): AnomalyPoint[];
+export declare function median(values: number[]): number;
+/** Median Absolute Deviation, scaled to be a consistent estimator of stdDev. */
+export declare function mad(values: number[], center?: number): number;
+export type MetricKind = "latency" | "error_rate" | "saturation" | "throughput" | "generic";
+export declare function classifyMetric(metric: string): MetricKind;
+export interface RobustAnomalyOptions {
+    /** Minimum samples before any detection (cold-start guard). */
+    minSamples?: number;
+    /** Number of trailing points evaluated as "recent". */
+    recentWindow?: number;
+    /** Robust-z threshold. */
+    threshold?: number;
+    /** Consecutive breaching recent points required to fire (dwell/hysteresis). */
+    dwell?: number;
+    metricKind?: MetricKind;
+}
+export interface RobustAnomalyResult {
+    isAnomaly: boolean;
+    /** Robust z = (median(recent) - median(baseline)) / MAD(baseline). */
+    score: number;
+    method: "robust-z" | "trend" | "none";
+    direction: "above" | "below" | "flat";
+    recentValue: number;
+    baselineValue: number;
+    reason: string;
+}
+/**
+ * Robust anomaly detection.
+ *
+ * Unlike {@link detectRecentAnomaly}, the baseline is the *early stable* portion
+ * of the series (it excludes the recent window AND the trailing ramp), so a slow
+ * monotonic increase — e.g. a memory leak heading toward OOM — no longer poisons
+ * its own baseline. Saturation/latency metrics additionally run a trend detector
+ * that catches gradual ramps even when no single point is a spike.
+ */
+export declare function detectRobustAnomaly(values: number[], opts?: RobustAnomalyOptions): RobustAnomalyResult;
+export interface SeasonalPoint {
+    /** Unix epoch milliseconds, or an ISO-8601 timestamp string. */
+    timestamp: number | string;
+    value: number;
+}
+export interface SeasonalAnomalyOptions {
+    /** Season length in seconds. Default: 86400 (daily / time-of-day). */
+    periodSeconds?: number;
+    /** Phase tolerance in seconds — how close in-phase a historical sample
+     *  must be to count toward the baseline. Default: periodSeconds / 48
+     *  (≈30 min for a daily period). */
+    phaseToleranceSeconds?: number;
+    /** Trailing points treated as "recent". Default: 5. */
+    recentWindow?: number;
+    /** Robust-z threshold against the same-phase distribution. Default: 3.5. */
+    threshold?: number;
+    /** Minimum same-phase historical samples required to trust the baseline. */
+    minPhaseSamples?: number;
+    metricKind?: MetricKind;
+}
+export interface SeasonalAnomalyResult {
+    isAnomaly: boolean;
+    /** false when there is not enough multi-period history — caller should
+     *  fall back to {@link detectRobustAnomaly}. */
+    applicable: boolean;
+    score: number;
+    expected: number;
+    recentValue: number;
+    direction: "above" | "below" | "flat";
+    phaseSamples: number;
+    reason: string;
+}
+/**
+ * Seasonal-naive detection: predict the recent value from the robust
+ * (median/MAD) distribution of historical points at the same phase of the
+ * season, and flag a deviation. Falls back (applicable=false) when the series
+ * does not span enough periods to build a same-phase baseline.
+ */
+export declare function detectSeasonalAnomaly(points: SeasonalPoint[], opts?: SeasonalAnomalyOptions): SeasonalAnomalyResult;
+/**
+ * Orchestrator: prefer the seasonality-aware baseline when the series spans
+ * enough periods to build a same-phase distribution; otherwise fall back to
+ * the robust windowed detector. Returns a normalized verdict.
+ */
+export declare function detectAnomaly(points: SeasonalPoint[], opts?: SeasonalAnomalyOptions & RobustAnomalyOptions): {
+    isAnomaly: boolean;
+    method: "seasonal" | "robust-z" | "trend" | "none";
+    score: number;
+    recentValue: number;
+    baselineValue: number;
+    direction: "above" | "below" | "flat";
+    reason: string;
+};
 /**
  * Check if the most recent values deviate significantly from the baseline.
  * Compares the last `recentWindow` values against the rest.

package/dist/analysis/anomaly.js

@@ -27,6 +27,241 @@ export function detectAnomalyPoints(values, threshold = 2.0) {
     }
     return anomalies;
 }
+// ---------------------------------------------------------------------------
+// Robust detection (median/MAD) — resistant to the trend & outliers that skew
+// mean/stdDev. Adds warmup, dwell/hysteresis, a slow-ramp trend detector, and
+// per-metric-type behaviour.
+// ---------------------------------------------------------------------------
+export function median(values) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = sorted.length >> 1;
+    return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}
+/** Median Absolute Deviation, scaled to be a consistent estimator of stdDev. */
+export function mad(values, center) {
+    if (values.length === 0)
+        return 0;
+    const med = center ?? median(values);
+    const deviations = values.map((v) => Math.abs(v - med));
+    return 1.4826 * median(deviations);
+}
+export function classifyMetric(metric) {
+    const m = metric.toLowerCase();
+    if (/(latency|duration|response_time|p\d{2,3})/.test(m))
+        return "latency";
+    if (/(error|fail|5xx|4xx)/.test(m))
+        return "error_rate";
+    if (/(cpu|mem|memory|heap|disk|saturat|util|queue|pool|fd|gc)/.test(m))
+        return "saturation";
+    if (/(request_rate|rps|qps|throughput|traffic)/.test(m))
+        return "throughput";
+    return "generic";
+}
+const NONE = {
+    isAnomaly: false,
+    score: 0,
+    method: "none",
+    direction: "flat",
+    recentValue: 0,
+    baselineValue: 0,
+    reason: "insufficient data (warmup)",
+};
+/**
+ * Robust anomaly detection.
+ *
+ * Unlike {@link detectRecentAnomaly}, the baseline is the *early stable* portion
+ * of the series (it excludes the recent window AND the trailing ramp), so a slow
+ * monotonic increase — e.g. a memory leak heading toward OOM — no longer poisons
+ * its own baseline. Saturation/latency metrics additionally run a trend detector
+ * that catches gradual ramps even when no single point is a spike.
+ */
+export function detectRobustAnomaly(values, opts = {}) {
+    const minSamples = opts.minSamples ?? 15;
+    const recentWindow = opts.recentWindow ?? 5;
+    const threshold = opts.threshold ?? 3.0;
+    const dwell = opts.dwell ?? 2;
+    const kind = opts.metricKind ?? "generic";
+    // Warmup guard.
+    if (values.length < Math.max(minSamples, recentWindow * 3))
+        return { ...NONE };
+    const recent = values.slice(-recentWindow);
+    // Baseline = leading stable portion only; exclude the recent window and a
+    // trailing margin so a ramp that ends in `recent` cannot inflate it.
+    const baselineEnd = Math.max(Math.floor(values.length * 0.5), values.length - recentWindow * 3);
+    const baseline = values.slice(0, baselineEnd);
+    if (baseline.length < 3)
+        return { ...NONE };
+    const baseMed = median(baseline);
+    const baseMad = mad(baseline, baseMed);
+    const recentMed = median(recent);
+    // One-sided metrics: a drop in error_rate / latency / saturation is good news.
+    const oneSidedUp = kind === "error_rate" || kind === "latency" || kind === "saturation";
+    // Robust z. Guard against MAD == 0 (perfectly flat baseline) with a tiny
+    // relative epsilon so a real shift off a flat baseline still registers.
+    const scale = baseMad > 0 ? baseMad : Math.max(Math.abs(baseMed) * 1e-3, 1e-9);
+    const z = (recentMed - baseMed) / scale;
+    const direction = z > 0 ? "above" : z < 0 ? "below" : "flat";
+    // Dwell: require the last `dwell` points to each individually breach.
+    const tail = values.slice(-dwell);
+    const breaches = tail.filter((v) => {
+        const pz = (v - baseMed) / scale;
+        return oneSidedUp ? pz >= threshold : Math.abs(pz) >= threshold;
+    });
+    const dwellMet = breaches.length >= dwell;
+    const zHit = (oneSidedUp ? z >= threshold : Math.abs(z) >= threshold) && dwellMet;
+    // Trend detector for slow ramps (saturation/latency). Catches a sustained
+    // monotonic climb even when the windowed robust-z is still sub-threshold.
+    let trendHit = false;
+    let trendReason = "";
+    if (!zHit && (kind === "saturation" || kind === "latency") && values.length >= minSamples) {
+        let ups = 0;
+        for (let i = 1; i < values.length; i++)
+            if (values[i] > values[i - 1])
+                ups++;
+        const monotonicFrac = ups / (values.length - 1);
+        const netRise = (recentMed - baseMed) / scale;
+        if (monotonicFrac >= 0.7 && netRise >= 2.0) {
+            trendHit = true;
+            trendReason = `sustained upward trend: ${Math.round(monotonicFrac * 100)}% of steps rising, +${netRise.toFixed(1)} robust-σ net`;
+        }
+    }
+    if (zHit) {
+        return {
+            isAnomaly: true,
+            score: z,
+            method: "robust-z",
+            direction,
+            recentValue: recentMed,
+            baselineValue: baseMed,
+            reason: `recent median ${recentMed.toFixed(2)} is ${z.toFixed(1)} robust-σ ${direction} baseline ${baseMed.toFixed(2)} (dwell ${breaches.length}/${dwell})`,
+        };
+    }
+    if (trendHit) {
+        return {
+            isAnomaly: true,
+            score: (recentMed - baseMed) / scale,
+            method: "trend",
+            direction: "above",
+            recentValue: recentMed,
+            baselineValue: baseMed,
+            reason: trendReason,
+        };
+    }
+    return {
+        isAnomaly: false,
+        score: z,
+        method: "none",
+        direction,
+        recentValue: recentMed,
+        baselineValue: baseMed,
+        reason: "within robust baseline",
+    };
+}
+function toEpochSeconds(t) {
+    if (typeof t === "number")
+        return t > 1e12 ? t / 1000 : t;
+    return new Date(t).getTime() / 1000;
+}
+/**
+ * Seasonal-naive detection: predict the recent value from the robust
+ * (median/MAD) distribution of historical points at the same phase of the
+ * season, and flag a deviation. Falls back (applicable=false) when the series
+ * does not span enough periods to build a same-phase baseline.
+ */
+export function detectSeasonalAnomaly(points, opts = {}) {
+    const period = opts.periodSeconds ?? 86400;
+    const tol = opts.phaseToleranceSeconds ?? period / 48;
+    const recentWindow = opts.recentWindow ?? 5;
+    const threshold = opts.threshold ?? 3.5;
+    const minPhaseSamples = opts.minPhaseSamples ?? 4;
+    const kind = opts.metricKind ?? "generic";
+    const NA = {
+        isAnomaly: false,
+        applicable: false,
+        score: 0,
+        expected: 0,
+        recentValue: 0,
+        direction: "flat",
+        phaseSamples: 0,
+        reason: "insufficient multi-period history",
+    };
+    if (points.length < recentWindow + 2)
+        return NA;
+    const series = points
+        .map((p) => ({ t: toEpochSeconds(p.timestamp), v: p.value }))
+        .filter((p) => Number.isFinite(p.t) && Number.isFinite(p.v))
+        .sort((a, b) => a.t - b.t);
+    if (series.length < recentWindow + 2)
+        return NA;
+    const span = series[series.length - 1].t - series[0].t;
+    // Need at least ~2 full periods of history to have any same-phase samples.
+    if (span < period * 2)
+        return NA;
+    const recent = series.slice(-recentWindow);
+    const history = series.slice(0, -recentWindow);
+    const recentPhase = ((recent[recent.length - 1].t % period) + period) % period;
+    // Same-phase historical samples: phase distance within tolerance (wrapping).
+    const samePhase = history
+        .filter((p) => {
+        const ph = ((p.t % period) + period) % period;
+        const d = Math.abs(ph - recentPhase);
+        return Math.min(d, period - d) <= tol;
+    })
+        .map((p) => p.v);
+    if (samePhase.length < minPhaseSamples)
+        return NA;
+    const expected = median(samePhase);
+    const spread = mad(samePhase, expected);
+    const recentMed = median(recent.map((p) => p.v));
+    const scale = spread > 0 ? spread : Math.max(Math.abs(expected) * 1e-3, 1e-9);
+    const z = (recentMed - expected) / scale;
+    const direction = z > 0 ? "above" : z < 0 ? "below" : "flat";
+    const oneSidedUp = kind === "error_rate" || kind === "latency" || kind === "saturation";
+    const hit = oneSidedUp ? z >= threshold : Math.abs(z) >= threshold;
+    return {
+        isAnomaly: hit,
+        applicable: true,
+        score: z,
+        expected,
+        recentValue: recentMed,
+        direction,
+        phaseSamples: samePhase.length,
+        reason: hit
+            ? `recent ${recentMed.toFixed(2)} is ${z.toFixed(1)} robust-σ ${direction} the seasonal baseline ${expected.toFixed(2)} (n=${samePhase.length} same-phase samples)`
+            : `within seasonal baseline (${expected.toFixed(2)}, n=${samePhase.length})`,
+    };
+}
+/**
+ * Orchestrator: prefer the seasonality-aware baseline when the series spans
+ * enough periods to build a same-phase distribution; otherwise fall back to
+ * the robust windowed detector. Returns a normalized verdict.
+ */
+export function detectAnomaly(points, opts = {}) {
+    const seasonal = detectSeasonalAnomaly(points, opts);
+    if (seasonal.applicable) {
+        return {
+            isAnomaly: seasonal.isAnomaly,
+            method: "seasonal",
+            score: seasonal.score,
+            recentValue: seasonal.recentValue,
+            baselineValue: seasonal.expected,
+            direction: seasonal.direction,
+            reason: seasonal.reason,
+        };
+    }
+    const r = detectRobustAnomaly(points.map((p) => p.value), opts);
+    return {
+        isAnomaly: r.isAnomaly,
+        method: r.method,
+        score: r.score,
+        recentValue: r.recentValue,
+        baselineValue: r.baselineValue,
+        direction: r.direction,
+        reason: r.reason,
+    };
+}
 /**
  * Check if the most recent values deviate significantly from the baseline.
  * Compares the last `recentWindow` values against the rest.

package/dist/analysis/anomaly.test.js

@@ -1,6 +1,6 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
-import { calculateZScore, detectAnomalyPoints, detectRecentAnomaly } from "./anomaly.js";
+import { calculateZScore, detectAnomalyPoints, detectRecentAnomaly, detectRobustAnomaly, detectSeasonalAnomaly, detectAnomaly, classifyMetric, median, mad, } from "./anomaly.js";
 describe("calculateZScore", () => {
     it("returns zeros for empty array", () => {
         const result = calculateZScore([]);
@@ -85,3 +85,151 @@ describe("detectRecentAnomaly", () => {
         assert.ok(lowResult.isAnomaly || !highResult.isAnomaly);
     });
 });
+describe("median / mad", () => {
+    it("median handles odd and even lengths", () => {
+        assert.equal(median([3, 1, 2]), 2);
+        assert.equal(median([1, 2, 3, 4]), 2.5);
+        assert.equal(median([]), 0);
+    });
+    it("mad is robust to outliers", () => {
+        const stable = [8, 10, 12, 9, 11, 10, 13, 7];
+        const withOutlier = [...stable, 100000];
+        const stdDev = (xs) => {
+            const m = xs.reduce((a, b) => a + b, 0) / xs.length;
+            return Math.sqrt(xs.reduce((s, v) => s + (v - m) ** 2, 0) / xs.length);
+        };
+        // MAD barely moves; stdDev explodes by orders of magnitude.
+        assert.ok(mad(stable) > 0);
+        assert.ok(mad(withOutlier) < mad(stable) * 2);
+        assert.ok(stdDev(withOutlier) > stdDev(stable) * 100);
+    });
+});
+describe("classifyMetric", () => {
+    it("classifies by name", () => {
+        assert.equal(classifyMetric("latency_p99"), "latency");
+        assert.equal(classifyMetric("error_rate"), "error_rate");
+        assert.equal(classifyMetric("cpu"), "saturation");
+        assert.equal(classifyMetric("memory_used_bytes"), "saturation");
+        assert.equal(classifyMetric("request_rate"), "throughput");
+        assert.equal(classifyMetric("widgets_total"), "generic");
+    });
+});
+describe("detectRobustAnomaly", () => {
+    it("warmup: no detection below minSamples", () => {
+        const r = detectRobustAnomaly([1, 2, 3, 4, 5]);
+        assert.equal(r.isAnomaly, false);
+        assert.equal(r.method, "none");
+    });
+    it("no anomaly for stable noisy data", () => {
+        const v = Array.from({ length: 40 }, (_, i) => 100 + (i % 3) - 1);
+        assert.equal(detectRobustAnomaly(v).isAnomaly, false);
+    });
+    // The exact production false-negative: a slow memory-leak ramp toward OOM.
+    // detectRecentAnomaly misses it because the rising baseline poisons its own
+    // mean/stdDev; detectRobustAnomaly must catch it.
+    it("REGRESSION: detects slow memory-leak ramp the legacy detector misses", () => {
+        // The query window opened AFTER the leak began, so there is no flat
+        // baseline — the metric climbs monotonically across the whole window.
+        // Legacy windowed z-score stays sub-threshold (the baseline already
+        // contains the ramp); this is the production "all healthy during OOM"
+        // false-negative. The robust trend detector must catch it.
+        const series = Array.from({ length: 40 }, (_, i) => 120 + i * 7);
+        const legacy = detectRecentAnomaly(series, 5, 2.0);
+        assert.equal(legacy.isAnomaly, false, "legacy detector misses the leak spanning the window");
+        const robust = detectRobustAnomaly(series, { metricKind: "saturation" });
+        assert.equal(robust.isAnomaly, true, "robust detector must catch the leak");
+        assert.equal(robust.method, "trend");
+        assert.equal(robust.direction, "above");
+    });
+    it("detects a hard spike via robust-z", () => {
+        const base = Array.from({ length: 25 }, (_, i) => 50 + (i % 3));
+        const spike = Array(5).fill(500);
+        const r = detectRobustAnomaly([...base, ...spike], { metricKind: "latency" });
+        assert.equal(r.isAnomaly, true);
+        assert.equal(r.method, "robust-z");
+    });
+    it("dwell/hysteresis: a single transient spike does not fire", () => {
+        const base = Array.from({ length: 30 }, (_, i) => 50 + (i % 3));
+        const series = [...base, 50, 51, 49, 500]; // one lone spike at the very end
+        const r = detectRobustAnomaly(series, { metricKind: "latency", dwell: 2 });
+        assert.equal(r.isAnomaly, false, "single point should not satisfy dwell");
+    });
+    it("one-sided: a drop in error_rate is not an anomaly", () => {
+        const base = Array.from({ length: 25 }, (_, i) => 20 + (i % 3));
+        const drop = Array(5).fill(0);
+        const r = detectRobustAnomaly([...base, ...drop], { metricKind: "error_rate" });
+        assert.equal(r.isAnomaly, false);
+    });
+    it("two-sided generic metric flags a drop", () => {
+        const base = Array.from({ length: 25 }, (_, i) => 100 + (i % 3));
+        const drop = Array(5).fill(5);
+        const r = detectRobustAnomaly([...base, ...drop], { metricKind: "generic" });
+        assert.equal(r.isAnomaly, true);
+        assert.equal(r.direction, "below");
+    });
+});
+describe("detectSeasonalAnomaly", () => {
+    const HOUR = 3600_000;
+    const NIGHT = (h) => h >= 22 || h <= 5;
+    // `days` of hourly samples with a strong diurnal pattern (night ~10, day
+    // ~100). `lastDayNight` overrides the final day's night value to inject a
+    // regression. Returns points ending at day-`days` hour 3 (a night hour).
+    function diurnal(days, lastDayNight) {
+        const pts = [];
+        const start = Date.UTC(2026, 0, 1, 0, 0, 0);
+        for (let d = 0; d < days; d++) {
+            for (let h = 0; h < 24; h++) {
+                let v = NIGHT(h) ? 10 : 100;
+                v += (d % 3) - 1; // small deterministic spread so MAD > 0
+                if (d === days - 1 && NIGHT(h) && lastDayNight !== undefined)
+                    v = lastDayNight;
+                pts.push({ timestamp: start + (d * 24 + h) * HOUR, value: v });
+            }
+        }
+        // Trim so the series ends mid-night (…23, 0, 1, 2, 3).
+        return pts.slice(0, days * 24 - 20);
+    }
+    it("not applicable with <2 periods of history", () => {
+        const r = detectSeasonalAnomaly(diurnal(1));
+        assert.equal(r.applicable, false);
+        assert.equal(r.isAnomaly, false);
+    });
+    it("KEY: a normal nightly trough is NOT an anomaly (robust would false-positive)", () => {
+        const series = diurnal(6); // ends in a normal low-night window
+        const seasonal = detectSeasonalAnomaly(series);
+        assert.equal(seasonal.applicable, true);
+        assert.equal(seasonal.isAnomaly, false, "night low is expected at this phase");
+        // The naive robust detector, lacking phase awareness, flags the trough.
+        const robust = detectRobustAnomaly(series.map((p) => p.value), { metricKind: "generic" });
+        assert.equal(robust.isAnomaly, true, "robust mistakes the diurnal trough for a drop");
+    });
+    it("flags a real same-phase regression (night value where day-level is wrong)", () => {
+        const series = diurnal(6, 100); // last day's night sits at daytime level
+        const r = detectSeasonalAnomaly(series, { metricKind: "saturation" });
+        assert.equal(r.applicable, true);
+        assert.equal(r.isAnomaly, true);
+        assert.equal(r.direction, "above");
+        assert.ok(r.phaseSamples >= 4);
+    });
+});
+describe("detectAnomaly orchestrator", () => {
+    it("uses seasonal when multi-period history is available", () => {
+        const HOUR = 3600_000;
+        const start = Date.UTC(2026, 0, 1);
+        const pts = [];
+        for (let i = 0; i < 24 * 5; i++) {
+            const h = i % 24;
+            pts.push({ timestamp: start + i * HOUR, value: (h >= 22 || h <= 5 ? 10 : 100) + (i % 3) });
+        }
+        const r = detectAnomaly(pts, { metricKind: "generic" });
+        assert.equal(r.method, "seasonal");
+    });
+    it("falls back to robust when history is too short", () => {
+        const pts = Array.from({ length: 30 }, (_, i) => ({
+            timestamp: 1_700_000_000_000 + i * 60_000,
+            value: 100 + (i % 3),
+        }));
+        const r = detectAnomaly(pts, { metricKind: "generic" });
+        assert.ok(r.method === "none" || r.method === "robust-z" || r.method === "trend");
+    });
+});

package/dist/analysis/backtest.d.ts

@@ -0,0 +1,31 @@
+import { type MetricKind, type SeasonalPoint } from "./anomaly.js";
+export interface BacktestCase {
+    name: string;
+    /** Regime label for the per-category breakdown. */
+    category: string;
+    points: SeasonalPoint[];
+    metricKind: MetricKind;
+    /** Ground truth. */
+    anomalous: boolean;
+}
+/**
+ * Build the labelled suite. Multiple seeded variants per regime so the
+ * precision/recall figures are stable rather than hostage to one sample.
+ */
+export declare function buildSuite(): BacktestCase[];
+export interface BacktestReport {
+    total: number;
+    tp: number;
+    fp: number;
+    tn: number;
+    fn: number;
+    precision: number;
+    recall: number;
+    f1: number;
+    byCategory: Record<string, {
+        total: number;
+        correct: number;
+    }>;
+}
+export declare function runBacktest(cases?: BacktestCase[]): BacktestReport;
+export declare function formatReport(r: BacktestReport): string;