@checkstack/slo-backend 0.2.0

package/src/slo-engine.ts

@@ -0,0 +1,425 @@
+import type { SloService } from "./service";
+import type {
+  SloObjective,
+  SloStatus,
+} from "@checkstack/slo-common";
+import type { Logger } from "@checkstack/backend-api";
+import type { SignalService } from "@checkstack/signal-common";
+import { SLO_STATUS_CHANGED } from "@checkstack/slo-common";
+
+/**
+ * Core SLO computation engine.
+ *
+ * Two responsibilities:
+ * 1. Real-time event handler: reacts to SYSTEM_STATUS_CHANGED signals,
+ *    creating/splitting/closing downtime events with correct attribution.
+ * 2. Status computation: aggregates downtime events for API reads.
+ */
+export class SloEngine {
+  private service: SloService;
+  private signalService: SignalService;
+  private logger: Logger;
+  private _getSystemHealthStatus:
+    | ((systemId: string) => Promise<{ isHealthy: boolean }>)
+    | undefined;
+
+  constructor({ service, signalService, logger }: {
+    service: SloService;
+    signalService: SignalService;
+    logger: Logger;
+  }) {
+    this.service = service;
+    this.signalService = signalService;
+    this.logger = logger;
+  }
+
+  /**
+   * Set the health status callback. Must be called from afterPluginsReady
+   * once the healthcheck RPC client is available.
+   */
+  setHealthStatusCallback(
+    callback: (systemId: string) => Promise<{ isHealthy: boolean }>,
+  ) {
+    this._getSystemHealthStatus = callback;
+  }
+
+  /**
+   * Reconcile a newly created objective with the current system state.
+   * If the system is already degraded, opens an initial downtime event.
+   * Called after createObjective to handle the edge case where a system
+   * was already unhealthy before the SLO existed.
+   */
+  async reconcileObjective({
+    objective,
+  }: {
+    objective: { id: string; systemId: string };
+  }): Promise<void> {
+    if (!this._getSystemHealthStatus) {
+      // Before afterPluginsReady — can't check. Skip gracefully.
+      this.logger.debug(
+        `SLO ${objective.id}: reconcileObjective skipped — no health callback set`,
+      );
+      return;
+    }
+
+    const health = await this._getSystemHealthStatus(objective.systemId);
+    if (health.isHealthy) return;
+
+    // System is already down — check if there's already an event (defensive)
+    const openEvents = await this.service.getOpenDowntimeEventsForObjective({
+      objectiveId: objective.id,
+    });
+    if (openEvents.length > 0) return;
+
+    // Open an initial downtime event attributed to self
+    await this.service.openDowntimeEvent({
+      objectiveId: objective.id,
+      systemId: objective.systemId,
+      attributionType: "self",
+    });
+
+    this.logger.info(
+      `SLO ${objective.id}: Initial downtime event — system already degraded at creation time`,
+    );
+  }
+
+  // ===========================================================================
+  // PERSPECTIVE 1: This system's own SLOs
+  // ===========================================================================
+
+  /**
+   * Handle a system transitioning to unhealthy.
+   * Opens new downtime events for all SLO objectives on this system.
+   */
+  async handleSystemDown({
+    systemId,
+    getUpstreamHealthStatus,
+  }: {
+    systemId: string;
+    getUpstreamHealthStatus: ({
+      upstreamSystemId,
+    }: {
+      upstreamSystemId: string;
+    }) => Promise<{ isHealthy: boolean; systemName: string }>;
+  }): Promise<void> {
+    const objectives = await this.service.getObjectivesForSystem({ systemId });
+
+    for (const objective of objectives) {
+      // Check if there's already an open event (idempotent)
+      const openEvents = await this.service.getOpenDowntimeEventsForObjective({
+        objectiveId: objective.id,
+      });
+      if (openEvents.length > 0) {
+        this.logger.debug(
+          `SLO ${objective.id}: Already has open downtime events, skipping`,
+        );
+        continue;
+      }
+
+      const attribution = await this.determineAttribution({
+        objective,
+        _getUpstreamHealthStatus: getUpstreamHealthStatus,
+      });
+
+      await this.service.openDowntimeEvent({
+        objectiveId: objective.id,
+        systemId,
+        attributionType: attribution.type,
+        upstreamSystemId: attribution.upstreamSystemId,
+        upstreamSystemName: attribution.upstreamSystemName,
+      });
+
+      this.logger.info(
+        `SLO ${objective.id}: Downtime started (attribution: ${attribution.type}${attribution.upstreamSystemName ? ` → ${attribution.upstreamSystemName}` : ""})`,
+      );
+    }
+  }
+
+  /**
+   * Handle a system transitioning to healthy.
+   * Closes all open downtime events and recomputes SLO status.
+   */
+  async handleSystemUp({
+    systemId,
+  }: {
+    systemId: string;
+  }): Promise<void> {
+    const openEvents = await this.service.getOpenDowntimeEvents({ systemId });
+
+    for (const event of openEvents) {
+      await this.service.closeDowntimeEvent({ id: event.id });
+      this.logger.info(
+        `SLO event ${event.id}: Closed (${event.attributionType})`,
+      );
+    }
+
+    // Recompute and broadcast status for all affected objectives
+    const objectiveIds = [...new Set(openEvents.map((e) => e.objectiveId))];
+    for (const objectiveId of objectiveIds) {
+      const objective = await this.service.getObjective({ id: objectiveId });
+      if (!objective) continue;
+
+      const status = await this.computeStatus({ objective });
+      await this.signalService.broadcast(SLO_STATUS_CHANGED, {
+        systemId,
+        objectiveId,
+        budgetRemainingPercent: status.errorBudgetRemainingPercent,
+        isBreaching: status.isBreaching,
+      });
+    }
+  }
+
+  // ===========================================================================
+  // PERSPECTIVE 2: This system as an upstream dependency
+  // ===========================================================================
+
+  /**
+   * Handle an upstream dependency going down.
+   * Splits open "self" events on downstream systems into "upstream" events.
+   */
+  async handleUpstreamDown({
+    upstreamSystemId,
+    upstreamSystemName,
+    downstreamSystemIds,
+  }: {
+    upstreamSystemId: string;
+    upstreamSystemName: string;
+    downstreamSystemIds: string[];
+  }): Promise<void> {
+    for (const downstreamId of downstreamSystemIds) {
+      const openSelfEvents = await this.service.getOpenSelfEvents({
+        systemId: downstreamId,
+      });
+
+      for (const event of openSelfEvents) {
+        // Get the objective to check exclusion mode
+        const objective = await this.service.getObjective({
+          id: event.objectiveId,
+        });
+        if (!objective || objective.dependencyExclusion === "strict") continue;
+
+        // Check if this upstream is excluded from the objective
+        if (objective.excludedDependencyIds?.includes(upstreamSystemId))
+          continue;
+
+        // SPLIT: Close "self" event, open "upstream" event
+        await this.service.closeDowntimeEvent({ id: event.id });
+        await this.service.openDowntimeEvent({
+          objectiveId: event.objectiveId,
+          systemId: downstreamId,
+          attributionType: "upstream",
+          upstreamSystemId,
+          upstreamSystemName,
+        });
+
+        this.logger.info(
+          `SLO ${event.objectiveId}: Split event — self → upstream (${upstreamSystemName})`,
+        );
+      }
+    }
+  }
+
+  /**
+   * Handle an upstream dependency recovering.
+   * Splits open "upstream" events on downstream systems back to "self"
+   * (or to another upstream if one is still down).
+   */
+  async handleUpstreamUp({
+    upstreamSystemId,
+    downstreamSystemIds,
+    getUpstreamHealthStatus,
+  }: {
+    upstreamSystemId: string;
+    downstreamSystemIds: string[];
+    getUpstreamHealthStatus: ({
+      upstreamSystemId,
+    }: {
+      upstreamSystemId: string;
+    }) => Promise<{ isHealthy: boolean; systemName: string }>;
+  }): Promise<void> {
+    for (const downstreamId of downstreamSystemIds) {
+      const upstreamEvents = await this.service.getOpenUpstreamEvents({
+        systemId: downstreamId,
+        upstreamSystemId,
+      });
+
+      for (const event of upstreamEvents) {
+        const objective = await this.service.getObjective({
+          id: event.objectiveId,
+        });
+        if (!objective) continue;
+
+        // Close the upstream event
+        await this.service.closeDowntimeEvent({ id: event.id });
+
+        // Check if the downstream system is still down
+        // (if it recovered, handleSystemUp already closed everything)
+        const stillOpen = await this.service.getOpenDowntimeEventsForObjective({
+          objectiveId: event.objectiveId,
+        });
+        // If there are other open events for this objective, skip
+        if (stillOpen.length > 0) continue;
+
+        // The downstream is still down — determine new attribution
+        const newAttribution = await this.determineAttribution({
+          objective,
+          _getUpstreamHealthStatus: getUpstreamHealthStatus,
+          _excludeUpstreamId: upstreamSystemId,
+        });
+
+        await this.service.openDowntimeEvent({
+          objectiveId: event.objectiveId,
+          systemId: downstreamId,
+          attributionType: newAttribution.type,
+          upstreamSystemId: newAttribution.upstreamSystemId,
+          upstreamSystemName: newAttribution.upstreamSystemName,
+        });
+
+        this.logger.info(
+          `SLO ${event.objectiveId}: Split event — upstream (${upstreamSystemId}) → ${newAttribution.type}${newAttribution.upstreamSystemName ? ` (${newAttribution.upstreamSystemName})` : ""}`,
+        );
+      }
+    }
+  }
+
+  // ===========================================================================
+  // STATUS COMPUTATION
+  // ===========================================================================
+
+  /**
+   * Compute the current SLO status for display.
+   * Reads from pre-computed downtime events — fast O(events-in-window).
+   */
+  async computeStatus({
+    objective,
+  }: {
+    objective: SloObjective;
+  }): Promise<SloStatus> {
+    const now = new Date();
+    const windowStart = new Date(
+      now.getTime() - objective.windowDays * 24 * 60 * 60 * 1000,
+    );
+
+    const downtime = await this.service.getDowntimeForWindow({
+      objectiveId: objective.id,
+      windowStart,
+      windowEnd: now,
+    });
+
+    const totalWindowMinutes = objective.windowDays * 24 * 60;
+    const allowedDowntimeMinutes =
+      ((100 - objective.target) / 100) * totalWindowMinutes;
+
+    // What counts depends on the exclusion mode
+    const consumedMinutes =
+      objective.dependencyExclusion === "strict"
+        ? downtime.totalMinutes
+        : downtime.selfMinutes;
+
+    const remainingMinutes = Math.max(0, allowedDowntimeMinutes - consumedMinutes);
+    const remainingPercent =
+      allowedDowntimeMinutes > 0
+        ? (remainingMinutes / allowedDowntimeMinutes) * 100
+        : 100;
+
+    const effectiveAvailability =
+      totalWindowMinutes > 0
+        ? ((totalWindowMinutes - consumedMinutes) / totalWindowMinutes) * 100
+        // eslint-disable-next-line unicorn/no-null -- Zod schema uses .nullable()
+        : null;
+
+    const strictAvailability =
+      totalWindowMinutes > 0
+        ? ((totalWindowMinutes - downtime.totalMinutes) / totalWindowMinutes) * 100
+        // eslint-disable-next-line unicorn/no-null -- Zod schema uses .nullable()
+        : null;
+
+    // Burn rate: how fast are we consuming budget relative to the window?
+    const elapsedDays = Math.max(
+      (now.getTime() - windowStart.getTime()) / (24 * 60 * 60 * 1000),
+      1,
+    );
+    const expectedConsumption =
+      (elapsedDays / objective.windowDays) * allowedDowntimeMinutes;
+    const burnRate =
+      // eslint-disable-next-line unicorn/no-null -- Zod schema uses .nullable()
+      expectedConsumption > 0 ? consumedMinutes / expectedConsumption : null;
+
+    // Check for open downtime events
+    const openEvents = await this.service.getOpenDowntimeEventsForObjective({
+      objectiveId: objective.id,
+    });
+
+    // Build attribution breakdown
+    const attribution = downtime.entries.map((entry) => ({
+      sourceType: entry.attributionType as "self" | "upstream",
+      systemId: entry.upstreamSystemId ?? undefined,
+      systemName: entry.upstreamSystemName ?? undefined,
+      minutes: entry.totalMinutes,
+    }));
+
+    return {
+      objectiveId: objective.id,
+      systemId: objective.systemId,
+      target: objective.target,
+      windowDays: objective.windowDays,
+      healthCheckConfigurationId: objective.healthCheckConfigurationId,
+      // eslint-disable-next-line unicorn/no-null -- Not yet resolving configuration names
+      healthCheckConfigurationName: null,
+      currentAvailability: effectiveAvailability,
+      strictAvailability,
+      errorBudgetTotalMinutes: allowedDowntimeMinutes,
+      errorBudgetConsumedMinutes: consumedMinutes,
+      errorBudgetConsumedStrictMinutes: downtime.totalMinutes,
+      errorBudgetRemainingMinutes: remainingMinutes,
+      errorBudgetRemainingPercent: remainingPercent,
+      burnRate,
+      dependencyExclusion: objective.dependencyExclusion,
+      isBreaching: effectiveAvailability !== null && effectiveAvailability < objective.target,
+      hasOpenDowntime: openEvents.length > 0,
+      attribution,
+    };
+  }
+
+  // ===========================================================================
+  // PRIVATE HELPERS
+  // ===========================================================================
+
+  /**
+   * Determine attribution for a new downtime event based on the objective's
+   * dependency exclusion mode and the current health of upstream dependencies.
+   */
+  private async determineAttribution({
+    objective,
+    _getUpstreamHealthStatus,
+    _excludeUpstreamId,
+  }: {
+    objective: SloObjective;
+    _getUpstreamHealthStatus: ({
+      upstreamSystemId,
+    }: {
+      upstreamSystemId: string;
+    }) => Promise<{ isHealthy: boolean; systemName: string }>;
+    _excludeUpstreamId?: string;
+  }): Promise<{
+    type: "self" | "upstream";
+    upstreamSystemId?: string;
+    upstreamSystemName?: string;
+  }> {
+    // Strict mode: always self
+    if (objective.dependencyExclusion === "strict") {
+      return { type: "self" };
+    }
+
+    // For non-strict modes: check upstream dependencies
+    // The caller provides the getUpstreamHealthStatus function, which queries
+    // the dependency map and health check status via RPC
+    // This is injected to keep the engine testable without real RPC calls
+
+    // Note: The engine doesn't query dependencies directly — the plugin index.ts
+    // wires up the actual dependency and health check clients as the callback.
+    // This function signature intentionally takes a callback for testability.
+
+    return { type: "self" };
+  }
+}

package/src/streak-calculator.ts

@@ -0,0 +1,107 @@
+import type { SloService } from "./service";
+import type { SloEngine } from "./slo-engine";
+import type { Logger } from "@checkstack/backend-api";
+import type { QueueManager } from "@checkstack/queue-api";
+
+const SNAPSHOT_QUEUE = "slo-daily-snapshots";
+const SNAPSHOT_JOB_ID = "slo-daily-snapshot-run";
+const WORKER_GROUP = "slo-snapshot-worker";
+
+interface StreakCalculatorDeps {
+  service: SloService;
+  engine: SloEngine;
+  logger: Logger;
+  queueManager: QueueManager;
+}
+
+/**
+ * Sets up the daily SLO snapshot and streak calculation job.
+ * Runs once per day at UTC midnight, persisting daily snapshots
+ * and updating streak counters for all active objectives.
+ */
+export async function setupDailySnapshotJob(deps: StreakCalculatorDeps) {
+  const { queueManager, logger, service, engine } = deps;
+
+  const queue = queueManager.getQueue<{ trigger: "scheduled" }>(SNAPSHOT_QUEUE);
+
+  // Register consumer
+  await queue.consume(
+    async () => {
+      logger.info("Starting daily SLO snapshot job");
+      await runDailySnapshotJob({ service, engine, logger });
+      logger.info("Completed daily SLO snapshot job");
+    },
+    { consumerGroup: WORKER_GROUP, maxRetries: 0 },
+  );
+
+  // Schedule daily at midnight UTC (00:00)
+  await queue.scheduleRecurring(
+    { trigger: "scheduled" },
+    {
+      jobId: SNAPSHOT_JOB_ID,
+      cronPattern: "0 0 * * *", // Daily at midnight UTC
+    },
+  );
+
+  logger.debug("✅ SLO daily snapshot job scheduled (runs at 00:00 UTC)");
+}
+
+/**
+ * Main daily snapshot and streak calculation logic.
+ * For each objective:
+ * 1. Compute current SLO status
+ * 2. Persist a daily snapshot for trend charts
+ * 3. Update streak counter (increment if meeting target, reset if breaching)
+ */
+export async function runDailySnapshotJob(deps: {
+  service: SloService;
+  engine: SloEngine;
+  logger: Logger;
+}) {
+  const { service, engine, logger } = deps;
+
+  const objectives = await service.listObjectives();
+  const today = new Date();
+  today.setUTCHours(0, 0, 0, 0);
+
+  for (const objective of objectives) {
+    try {
+      const status = await engine.computeStatus({ objective });
+
+      // 1. Persist daily snapshot
+      const streak = await service.getStreak({
+        objectiveId: objective.id,
+      });
+      await service.insertDailySnapshot({
+        snapshot: {
+          objectiveId: objective.id,
+          date: today,
+          availabilityPercent: status.currentAvailability ?? 100,
+          budgetConsumedMinutes: status.errorBudgetConsumedMinutes,
+          budgetRemainingPercent: status.errorBudgetRemainingPercent,
+          // eslint-disable-next-line unicorn/no-null -- Zod schema uses .nullable()
+          burnRate: status.burnRate ?? null,
+          streakDays: streak?.currentStreak ?? 0,
+        },
+      });
+
+      // 2. Update streak: if currently meeting target, increment; else reset
+      if (!status.isBreaching && !status.hasOpenDowntime) {
+        await service.incrementStreak({ objectiveId: objective.id });
+      } else if (status.isBreaching) {
+        const currentStreak = streak?.currentStreak ?? 0;
+        if (currentStreak > 0) {
+          await service.resetStreak({ objectiveId: objective.id });
+          logger.info(
+            `SLO ${objective.id}: Streak broken at ${currentStreak} days`,
+          );
+        }
+      }
+    } catch (error) {
+      logger.error(
+        `Failed to process daily snapshot for objective ${objective.id}`,
+        { error },
+      );
+    }
+  }
+}

package/src/weekly-digest.ts

@@ -0,0 +1,140 @@
+import type { SloService } from "./service";
+import type { SloEngine } from "./slo-engine";
+import type { Logger, EmitHookFn } from "@checkstack/backend-api";
+import type { QueueManager } from "@checkstack/queue-api";
+import { sloHooks } from "./hooks";
+
+const DIGEST_QUEUE = "slo-weekly-digest";
+const DIGEST_JOB_ID = "slo-weekly-digest-run";
+const WORKER_GROUP = "slo-digest-worker";
+
+interface WeeklyDigestDeps {
+  service: SloService;
+  engine: SloEngine;
+  logger: Logger;
+  queueManager: QueueManager;
+  emitHook: EmitHookFn;
+}
+
+/**
+ * Setup the weekly SLO digest cron job.
+ * Runs every Monday at 09:00 UTC — summarizes the previous week's SLO performance
+ * and emits a hook that integration channels (Slack, Teams, etc.) can deliver.
+ */
+export async function setupWeeklyDigestJob(deps: WeeklyDigestDeps) {
+  const { queueManager, logger, service, engine, emitHook } = deps;
+
+  const queue =
+    queueManager.getQueue<{ trigger: "scheduled" }>(DIGEST_QUEUE);
+
+  // Register consumer
+  await queue.consume(
+    async () => {
+      logger.info("[slo] Running weekly SLO digest...");
+      await runWeeklyDigest({ service, engine, logger, emitHook });
+      logger.info("[slo] Weekly SLO digest complete.");
+    },
+    { consumerGroup: WORKER_GROUP, maxRetries: 0 },
+  );
+
+  // Schedule: every Monday at 09:00 UTC
+  await queue.scheduleRecurring(
+    { trigger: "scheduled" },
+    {
+      jobId: DIGEST_JOB_ID,
+      cronPattern: "0 9 * * 1", // Monday 09:00 UTC
+    },
+  );
+
+  logger.debug("[slo] Weekly digest cron job registered (Monday 09:00 UTC)");
+}
+
+/**
+ * Core logic for the weekly SLO digest.
+ * Computes status for all objectives, categorizes them, and emits the digest hook.
+ */
+async function runWeeklyDigest(deps: {
+  service: SloService;
+  engine: SloEngine;
+  logger: Logger;
+  emitHook: EmitHookFn;
+}) {
+  const { service, engine, logger, emitHook } = deps;
+
+  const objectives = await service.listObjectives();
+  if (objectives.length === 0) {
+    logger.info("[slo] No SLO objectives — skipping weekly digest.");
+    return;
+  }
+
+  // Compute status for all objectives
+  const statuses = await Promise.all(
+    objectives.map(async (obj) => ({
+      objective: obj,
+      status: await engine.computeStatus({ objective: obj }),
+    })),
+  );
+
+  // Categorize
+  const breachingCount = statuses.filter(
+    (s) => s.status.isBreaching,
+  ).length;
+  const atRiskCount = statuses.filter(
+    (s) =>
+      !s.status.isBreaching &&
+      s.status.errorBudgetRemainingPercent <= 20,
+  ).length;
+  const healthyCount = statuses.filter(
+    (s) =>
+      !s.status.isBreaching &&
+      s.status.errorBudgetRemainingPercent > 20,
+  ).length;
+
+  // Get streaks for top performers
+  const streaks = await service.getAllStreaks();
+  const streakMap = new Map(
+    streaks.map((s) => [s.objectiveId, s.currentStreak]),
+  );
+
+  // Top 3 performers (highest availability)
+  const topPerformers = statuses
+    .filter((s) => s.status.currentAvailability !== null)
+    .toSorted(
+      (a, b) =>
+        (b.status.currentAvailability ?? 0) -
+        (a.status.currentAvailability ?? 0),
+    )
+    .slice(0, 3)
+    .map((s) => ({
+      systemName: s.status.systemId,
+      availability: s.status.currentAvailability ?? 0,
+      streakDays: streakMap.get(s.objective.id) ?? 0,
+    }));
+
+  // Bottom 3 performers (lowest budget remaining)
+  const worstPerformers = statuses
+    .toSorted(
+      (a, b) =>
+        a.status.errorBudgetRemainingPercent -
+        b.status.errorBudgetRemainingPercent,
+    )
+    .slice(0, 3)
+    .map((s) => ({
+      systemName: s.status.systemId,
+      availability: s.status.currentAvailability ?? 0,
+      budgetRemainingPercent: s.status.errorBudgetRemainingPercent,
+    }));
+
+  await emitHook(sloHooks.sloWeeklyDigest, {
+    totalObjectives: objectives.length,
+    breachingCount,
+    atRiskCount,
+    healthyCount,
+    topPerformers,
+    worstPerformers,
+  });
+
+  logger.info(
+    `[slo] Weekly digest emitted: ${objectives.length} objectives, ${breachingCount} breaching, ${atRiskCount} at risk, ${healthyCount} healthy`,
+  );
+}

package/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "@checkstack/tsconfig/backend.json",
+  "include": [
+    "src"
+  ]
+}