@checkstack/anomaly-backend 0.2.0

package/src/jobs/baseline-analyzer.ts

@@ -0,0 +1,269 @@
+import type { CollectorRegistry, Logger, SafeDatabase } from "@checkstack/backend-api";
+import type { CacheProvider } from "@checkstack/cache-api";
+import type { CatalogApi } from "@checkstack/catalog-common";
+import type { InferClient } from "@checkstack/common";
+import type { HealthCheckApi, HealthCheckRunResult } from "@checkstack/healthcheck-common";
+import { getHealthResultMeta } from "@checkstack/healthcheck-common";
+import {
+  ANOMALY_BASELINE_UPDATED,
+  computeDominance,
+  computeLinearRegressionSlope,
+  computeMean,
+  computeStdDev,
+  type AnomalyDirection,
+  type FieldBaseline,
+} from "@checkstack/anomaly-common";
+import type { QueueManager } from "@checkstack/queue-api";
+import type { SignalService } from "@checkstack/signal-common";
+import type { z } from "zod";
+import * as schema from "../schema";
+import { AnomalyService } from "../service";
+import { evaluateDrift } from "../drift-evaluator";
+
+export const BASELINE_ANALYZER_QUEUE = "anomaly-baseline-analyzer";
+
+/** Minimum data points required before any baseline is persisted (cold start). */
+const MIN_BASELINE_SAMPLES = 24;
+
+export async function setupBaselineAnalyzerJob({
+  db,
+  cache,
+  logger,
+  queueManager,
+  healthCheckClient,
+  signalService,
+  catalogClient,
+  collectorRegistry,
+}: {
+  db: SafeDatabase<typeof schema>;
+  cache: CacheProvider;
+  logger: Logger;
+  queueManager: QueueManager;
+  healthCheckClient: InferClient<typeof HealthCheckApi>;
+  signalService?: SignalService;
+  catalogClient: InferClient<typeof CatalogApi>;
+  collectorRegistry: CollectorRegistry;
+}) {
+  const queue = queueManager.getQueue(BASELINE_ANALYZER_QUEUE);
+  const anomalyService = new AnomalyService(db);
+
+  await queue.consume(
+    async (_job) => {
+      logger.debug("Running anomaly baseline analyzer background job...");
+
+      const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
+
+      const activeAssignments = await healthCheckClient.getRunsForAnalysis({
+        startDate: sevenDaysAgo,
+        limitPerAssignment: 200,
+      });
+
+      for (const assignment of activeAssignments) {
+        // Per-assignment configuration is fetched once and reused per field.
+        let templateConfig;
+        let assignmentConfig;
+        try {
+          const templateRecord = await anomalyService.getAnomalyConfig(
+            assignment.configurationId,
+          );
+          templateConfig = templateRecord.data;
+          const assignmentRecord =
+            await anomalyService.getAnomalyAssignmentConfig(
+              assignment.systemId,
+              assignment.configurationId,
+            );
+          assignmentConfig = assignmentRecord?.data;
+        } catch (error) {
+          logger.warn(
+            `Failed to fetch anomaly config for ${assignment.configurationId}; skipping drift evaluation`,
+            error,
+          );
+        }
+
+        const fieldValues: Record<string, (string | boolean | number)[]> = {};
+        const fieldCollectorIds: Record<string, string> = {};
+        const fieldNames: Record<string, string> = {};
+
+        // `getRunsForAnalysis` returns runs in DESCENDING timestamp order.
+        // Iterate in reverse so per-field arrays end up chronologically ascending —
+        // a property the regression slope relies on.
+        for (let i = assignment.runs.length - 1; i >= 0; i--) {
+          const row = assignment.runs[i];
+          if (!row.result) continue;
+
+          const result = row.result as HealthCheckRunResult;
+
+          if (typeof result.latencyMs === "number") {
+            const fullPath = "latencyMs";
+            if (!fieldValues[fullPath]) fieldValues[fullPath] = [];
+            fieldValues[fullPath].push(result.latencyMs);
+          }
+
+          const collectors = result.metadata?.collectors;
+          if (!collectors) continue;
+
+          for (const collectorData of Object.values(collectors)) {
+            if (typeof collectorData !== "object" || collectorData === null) continue;
+            const data = collectorData as Record<string, unknown>;
+            const realCollectorId = data._collectorId;
+            if (typeof realCollectorId !== "string") continue;
+
+            for (const [fieldName, value] of Object.entries(data)) {
+              if (fieldName === "_collectorId" || fieldName.startsWith("_")) continue;
+              if (
+                typeof value !== "number" &&
+                typeof value !== "string" &&
+                typeof value !== "boolean"
+              ) {
+                continue;
+              }
+              const fullPath = `collectors.${realCollectorId}.${fieldName}`;
+              if (!fieldValues[fullPath]) fieldValues[fullPath] = [];
+              fieldValues[fullPath].push(value);
+              fieldCollectorIds[fullPath] = realCollectorId;
+              fieldNames[fullPath] = fieldName;
+            }
+          }
+        }
+
+        for (const [path, values] of Object.entries(fieldValues)) {
+          if (values.length < MIN_BASELINE_SAMPLES) continue;
+
+          let mean = 0;
+          let stdDev = 0;
+          let trendSlope = 0;
+          let dominantValue: string | undefined;
+          let dominantRatio: number | undefined;
+
+          if (typeof values[0] === "number") {
+            const numValues = values as number[];
+            mean = computeMean(numValues);
+            stdDev = computeStdDev(numValues);
+            trendSlope = computeLinearRegressionSlope(numValues);
+          }
+
+          const dom = computeDominance(values);
+          if (dom.dominantValue !== undefined) {
+            dominantValue = String(dom.dominantValue);
+            dominantRatio = dom.dominantRatio;
+          }
+
+          const baseline = {
+            mean,
+            stdDev,
+            trendSlope,
+            dominantValue,
+            dominantRatio,
+            sampleCount: values.length,
+            computedAt: new Date(),
+          };
+
+          await db
+            .insert(schema.anomalyBaselines)
+            .values({
+              systemId: assignment.systemId,
+              configurationId: assignment.configurationId,
+              fieldPath: path,
+              ...baseline,
+            })
+            .onConflictDoUpdate({
+              target: [
+                schema.anomalyBaselines.systemId,
+                schema.anomalyBaselines.configurationId,
+                schema.anomalyBaselines.fieldPath,
+              ],
+              set: baseline,
+            });
+
+          const cacheKey = `baseline:${assignment.configurationId}:${assignment.systemId}:${path}`;
+          await cache.set(
+            cacheKey,
+            { ...baseline, computedAt: baseline.computedAt.toISOString() },
+            1000 * 60 * 60 * 24,
+          );
+
+          if (signalService && typeof values[0] === "number") {
+            await signalService.broadcast(ANOMALY_BASELINE_UPDATED, {
+              systemId: assignment.systemId,
+              configurationId: assignment.configurationId,
+              fieldPath: path,
+              mean: baseline.mean,
+              stdDev: baseline.stdDev,
+              sampleCount: baseline.sampleCount,
+            });
+          }
+
+          // Drift evaluation runs only for numeric fields scoped to a collector
+          // (the path layout `collectors.${id}.${field}`). Run-level fields like
+          // `latencyMs` are skipped because we have no schema-declared direction
+          // for them.
+          if (typeof values[0] !== "number") continue;
+          const collectorId = fieldCollectorIds[path];
+          const fieldName = fieldNames[path];
+          if (!collectorId || !fieldName) continue;
+
+          const schemaDirection = lookupSchemaDirection({
+            collectorRegistry,
+            collectorId,
+            fieldName,
+          });
+
+          const baselineDto: FieldBaseline = {
+            ...baseline,
+            computedAt: baseline.computedAt.toISOString(),
+          };
+
+          await evaluateDrift({
+            db,
+            logger,
+            catalogClient,
+            signalService,
+            systemId: assignment.systemId,
+            configurationId: assignment.configurationId,
+            fieldPath: path,
+            baseline: baselineDto,
+            schemaDirection,
+            templateConfig,
+            assignmentConfig,
+          });
+        }
+      }
+
+      logger.debug("Anomaly baselines successfully recomputed.");
+    },
+    {
+      consumerGroup: "anomaly-workers",
+    },
+  );
+
+  // Schedule to run every hour
+  await queue.scheduleRecurring(
+    { trigger: "scheduled" },
+    {
+      jobId: "hourly-baseline-analysis",
+      cronPattern: "0 * * * *",
+    },
+  );
+
+  logger.debug("Anomaly baseline analyzer job scheduled.");
+}
+
+function lookupSchemaDirection({
+  collectorRegistry,
+  collectorId,
+  fieldName,
+}: {
+  collectorRegistry: CollectorRegistry;
+  collectorId: string;
+  fieldName: string;
+}): AnomalyDirection | undefined {
+  const collector = collectorRegistry.getCollector(collectorId);
+  if (!collector) return undefined;
+  const collectorSchema = collector.collector.result.schema;
+  if (!("shape" in collectorSchema)) return undefined;
+  const shape = collectorSchema.shape as Record<string, z.ZodTypeAny>;
+  const fieldSchema = shape[fieldName];
+  if (!fieldSchema) return undefined;
+  const meta = getHealthResultMeta(fieldSchema);
+  return meta?.["x-anomaly-direction"];
+}

package/src/notification.ts

@@ -0,0 +1,139 @@
+import type { Logger } from "@checkstack/backend-api";
+import type { CatalogApi } from "@checkstack/catalog-common";
+import { catalogRoutes } from "@checkstack/catalog-common";
+import type { InferClient } from "@checkstack/common";
+import { resolveRoute } from "@checkstack/common";
+
+export type AnomalyNotificationAction =
+  | "confirmed"
+  | "recovered"
+  | "drift_confirmed"
+  | "drift_recovered";
+
+export interface DispatchAnomalyNotificationInput {
+  action: AnomalyNotificationAction;
+  systemId: string;
+  fieldPath: string;
+  observedValue: string | boolean | number;
+  baselineMean: number;
+  catalogClient: InferClient<typeof CatalogApi>;
+  logger: Logger;
+  /** Drift-specific: projected change over the baseline window. */
+  projectedChange?: number;
+}
+
+/**
+ * Dispatches anomaly-related notifications following the Sidecar Notification
+ * Orchestration pattern. Centralizes system lookup, URL resolution, importance
+ * mapping, and error isolation across all anomaly action types (Phase 1 spike
+ * confirmed/recovered + Phase 2 drift confirmed/recovered).
+ */
+export async function dispatchAnomalyNotification({
+  action,
+  systemId,
+  fieldPath,
+  observedValue,
+  baselineMean,
+  catalogClient,
+  logger,
+  projectedChange,
+}: DispatchAnomalyNotificationInput): Promise<void> {
+  try {
+    const system = await catalogClient.getSystem({ systemId });
+    const systemName = system?.name ?? systemId;
+    const actionUrl = resolveRoute(catalogRoutes.routes.systemDetail, {
+      systemId,
+    });
+
+    const obsStr =
+      typeof observedValue === "number"
+        ? observedValue.toFixed(2)
+        : String(observedValue);
+    const baseStr = baselineMean.toFixed(2);
+    const driftStr =
+      projectedChange === undefined
+        ? ""
+        : `${projectedChange >= 0 ? "+" : ""}${projectedChange.toFixed(2)}`;
+
+    const { title, message } = buildNotificationCopy({
+      action,
+      systemName,
+      fieldPath,
+      obsStr,
+      baseStr,
+      driftStr,
+    });
+
+    const importance = getImportance(action);
+
+    await catalogClient.notifySystemSubscribers({
+      systemId,
+      title,
+      body: message,
+      importance,
+      action: { label: "View System", url: actionUrl },
+      includeGroupSubscribers: true,
+    });
+  } catch (error) {
+    logger.warn(
+      `Failed to dispatch anomaly ${action} notification for ${systemId}`,
+      error,
+    );
+  }
+}
+
+function buildNotificationCopy({
+  action,
+  systemName,
+  fieldPath,
+  obsStr,
+  baseStr,
+  driftStr,
+}: {
+  action: AnomalyNotificationAction;
+  systemName: string;
+  fieldPath: string;
+  obsStr: string;
+  baseStr: string;
+  driftStr: string;
+}): { title: string; message: string } {
+  switch (action) {
+    case "confirmed": {
+      return {
+        title: `Anomaly Detected: ${systemName}`,
+        message: `Anomaly confirmed for **${fieldPath}**. Observed: ${obsStr}, Baseline: ${baseStr}.`,
+      };
+    }
+    case "recovered": {
+      return {
+        title: `Anomaly Recovered: ${systemName}`,
+        message: `Anomaly recovered for **${fieldPath}**. Observed: ${obsStr}, Baseline: ${baseStr}.`,
+      };
+    }
+    case "drift_confirmed": {
+      const projectionFragment = driftStr === ""
+        ? ""
+        : ` Projected change over the baseline window: ${driftStr}.`;
+      return {
+        title: `Trend Drift Detected: ${systemName}`,
+        message: `**${fieldPath}** is drifting. Current mean: ${obsStr}, Baseline: ${baseStr}.${projectionFragment}`,
+      };
+    }
+    case "drift_recovered": {
+      return {
+        title: `Trend Drift Recovered: ${systemName}`,
+        message: `**${fieldPath}** has stabilized. Current mean: ${obsStr}, Baseline: ${baseStr}.`,
+      };
+    }
+  }
+}
+
+/**
+ * Action-Based Importance Logic per Sidecar Orchestration standard.
+ * - Terminal "Good News" states (recovered, drift_recovered) are always info.
+ * - "Bad News" states (confirmed, drift_confirmed) are warnings.
+ */
+export function getImportance(action: AnomalyNotificationAction): "info" | "warning" {
+  if (action === "recovered" || action === "drift_recovered") return "info";
+  return "warning";
+}

package/src/plugin.ts

@@ -0,0 +1,85 @@
+import { createBackendPlugin, coreServices, type SafeDatabase } from "@checkstack/backend-api";
+import { healthCheckHooks } from "@checkstack/healthcheck-backend";
+import { setupBaselineAnalyzerJob } from "./jobs/baseline-analyzer";
+import { processCheckCompleted } from "./detector";
+import * as schema from "./schema";
+import { CatalogApi } from "@checkstack/catalog-common";
+import { AnomalyService } from "./service";
+import { createRouter } from "./router";
+import { createAnomalyRouterCache, type AnomalyRouterCache } from "./router-cache";
+import { anomalyContract, anomalyAccessRules } from "@checkstack/anomaly-common";
+import { HealthCheckApi } from "@checkstack/healthcheck-common";
+
+import { definePluginMetadata } from "@checkstack/common";
+
+export const plugin = createBackendPlugin({
+  metadata: definePluginMetadata({
+    pluginId: "anomaly",
+  }),
+  register(env) {
+    env.registerAccessRules(anomalyAccessRules);
+
+    // Shared between init (router) and afterPluginsReady (detector hook),
+    // so the detector can drop the router cache before broadcasting state
+    // change signals.
+    let routerCache: AnomalyRouterCache | undefined;
+
+    env.registerInit({
+      schema,
+      deps: {
+        db: coreServices.database,
+        logger: coreServices.logger,
+        queueManager: coreServices.queueManager,
+        cacheManager: coreServices.cacheManager, // Pre-req
+        rpcClient: coreServices.rpcClient,
+        rpc: coreServices.rpc,
+        signalService: coreServices.signalService,
+        collectorRegistry: coreServices.collectorRegistry,
+      },
+      init: async ({ db, logger, queueManager, cacheManager, rpc, rpcClient, signalService, collectorRegistry }) => {
+        logger.debug("Initializing Anomaly Detection Backend...");
+
+        const cache = cacheManager.getProvider();
+        const typedDb = db as SafeDatabase<typeof schema>;
+        const healthCheckClient = rpcClient.forPlugin(HealthCheckApi);
+        const catalogClient = rpcClient.forPlugin(CatalogApi);
+
+        await setupBaselineAnalyzerJob({
+          db: typedDb,
+          cache,
+          logger,
+          queueManager,
+          healthCheckClient,
+          signalService,
+          catalogClient,
+          collectorRegistry,
+        });
+
+        const service = new AnomalyService(typedDb);
+        routerCache = createAnomalyRouterCache({ cacheManager, logger });
+        const router = createRouter(service, logger, routerCache);
+        rpc.registerRouter(router, anomalyContract);
+
+        logger.debug("Anomaly Detection Backend initialized.");
+      },
+      afterPluginsReady: async ({ onHook, db, logger, cacheManager, rpcClient, collectorRegistry, signalService }) => {
+        const cache = cacheManager.getProvider();
+        const typedDb = db as SafeDatabase<typeof schema>;
+        const catalogClient = rpcClient.forPlugin(CatalogApi);
+
+        onHook(healthCheckHooks.checkCompleted, async (payload) => {
+          await processCheckCompleted({
+            ...payload,
+            db: typedDb,
+            cache,
+            routerCache,
+            logger,
+            catalogClient,
+            signalService,
+            collectorRegistry,
+          });
+        });
+      },
+    });
+  },
+});

package/src/router-cache.ts

@@ -0,0 +1,89 @@
+import type { CacheManager } from "@checkstack/cache-api";
+import {
+  createCachedScope,
+  type CachedScope,
+} from "@checkstack/cache-utils";
+import type { Logger } from "@checkstack/backend-api";
+
+/**
+ * Router-level cache for the anomaly plugin's read APIs.
+ *
+ * NOTE: This is a *separate* cache from the baseline cache the detector
+ * uses (which lives in `detector.ts` / `jobs/baseline-analyzer.ts` and
+ * keys by `baseline:...`). Keeping them in distinct prefixes lets us
+ * invalidate one without touching the other.
+ *
+ * 15s TTL because the detector creates/updates anomaly rows on every
+ * relevant check completion; we drop the cache on every detector write
+ * but TTL acts as a freshness ceiling for any path that forgets.
+ */
+const ANOMALY_TTL_MS = 15_000;
+
+const ANOMALIES_PREFIX = "anomalies:";
+const BASELINES_PREFIX = "baselines:";
+
+function stableStringify(value: unknown): string {
+  if (value === null || typeof value !== "object") {
+    return JSON.stringify(value);
+  }
+  if (Array.isArray(value)) {
+    return `[${value.map((v) => stableStringify(v)).join(",")}]`;
+  }
+  const entries = Object.entries(value as Record<string, unknown>)
+    .filter(([, v]) => v !== undefined)
+    .toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  return `{${entries
+    .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`)
+    .join(",")}}`;
+}
+
+const anomaliesKey = (input: unknown): string =>
+  `${ANOMALIES_PREFIX}${stableStringify(input ?? {})}`;
+const baselinesKey = (input: unknown): string =>
+  `${BASELINES_PREFIX}${stableStringify(input ?? {})}`;
+
+export interface AnomalyRouterCache {
+  wrapAnomalies: <T>(input: unknown, loader: () => Promise<T>) => Promise<T>;
+  wrapBaselines: <T>(input: unknown, loader: () => Promise<T>) => Promise<T>;
+
+  /**
+   * Drop every cached anomaly list. Called by the detector after any
+   * insert/update/delete on the anomalies table, before the
+   * `ANOMALY_STATE_CHANGED` signal goes out, so frontend refetches see
+   * fresh data.
+   */
+  invalidateAnomalies: () => Promise<number>;
+
+  /**
+   * Drop cached baseline list shapes — used after detector writes that
+   * affect baselines.
+   */
+  invalidateBaselines: () => Promise<number>;
+
+  scope: CachedScope;
+}
+
+export function createAnomalyRouterCache({
+  cacheManager,
+  logger,
+}: {
+  cacheManager: CacheManager;
+  logger: Logger;
+}): AnomalyRouterCache {
+  const scope = createCachedScope({
+    cacheManager,
+    pluginId: "anomaly-router",
+    defaultTtlMs: ANOMALY_TTL_MS,
+    onError: (op: string, error: unknown) => {
+      logger.warn(`anomaly router cache ${op} failed: ${String(error)}`);
+    },
+  });
+
+  return {
+    wrapAnomalies: (input, loader) => scope.wrap(anomaliesKey(input), loader),
+    wrapBaselines: (input, loader) => scope.wrap(baselinesKey(input), loader),
+    invalidateAnomalies: () => scope.invalidatePrefix(ANOMALIES_PREFIX),
+    invalidateBaselines: () => scope.invalidatePrefix(BASELINES_PREFIX),
+    scope,
+  };
+}

package/src/router.ts

@@ -0,0 +1,74 @@
+import { implement } from "@orpc/server";
+import { anomalyContract } from "@checkstack/anomaly-common";
+import type { AnomalyService } from "./service";
+import type { Logger } from "@checkstack/backend-api";
+import type { VersionedRecord } from "@checkstack/backend-api";
+import type { AnomalySettings } from "@checkstack/anomaly-common";
+import type { AnomalyRouterCache } from "./router-cache";
+
+export function createRouter(
+  service: AnomalyService,
+  logger: Logger,
+  cache: AnomalyRouterCache,
+) {
+  const os = implement(anomalyContract);
+
+  return os.router({
+    getAnomalies: os.getAnomalies.handler(
+      async ({ input }) => {
+        logger.debug("Fetching anomalies", { input });
+        return cache.wrapAnomalies(input ?? {}, () =>
+          service.getAnomalies(input ?? {}),
+        );
+      }
+    ),
+
+    getAnomalyBaselines: os.getAnomalyBaselines.handler(
+      async ({ input }) => {
+        logger.debug("Fetching anomaly baselines", { input });
+        return cache.wrapBaselines(input, () =>
+          service.getAnomalyBaselines(input),
+        );
+      }
+    ),
+
+    getAnomalyConfig: os.getAnomalyConfig.handler(
+      async ({ input }) => {
+        return await service.getAnomalyConfig(input.configurationId);
+      }
+    ),
+
+    updateAnomalyConfig: os.updateAnomalyConfig.handler(
+      async ({ input }) => {
+        const result = await service.updateAnomalyConfig(input.configurationId, input.config);
+        // Config updates can change which fields are flagged; drop both
+        // anomaly-list and baseline-list caches before returning so any
+        // immediate refetch by the admin UI sees fresh state.
+        await Promise.all([
+          cache.invalidateAnomalies(),
+          cache.invalidateBaselines(),
+        ]);
+        return result as VersionedRecord<AnomalySettings>;
+      }
+    ),
+
+    getAnomalyAssignmentConfig: os.getAnomalyAssignmentConfig.handler(
+      async ({ input }) => {
+        const result = await service.getAnomalyAssignmentConfig(input.systemId, input.configurationId);
+        // eslint-disable-next-line unicorn/no-null
+        return (result as VersionedRecord<Partial<AnomalySettings>>) ?? null;
+      }
+    ),
+
+    updateAnomalyAssignmentConfig: os.updateAnomalyAssignmentConfig.handler(
+      async ({ input }) => {
+        const result = await service.updateAnomalyAssignmentConfig(input.systemId, input.configurationId, input.config);
+        await Promise.all([
+          cache.invalidateAnomalies(),
+          cache.invalidateBaselines(),
+        ]);
+        return result as VersionedRecord<Partial<AnomalySettings>>;
+      }
+    ),
+  });
+}