@checkstack/slo-backend 0.2.0

package/src/achievement-evaluator.ts

@@ -0,0 +1,201 @@
+import type { SloService } from "./service";
+import type { SloEngine } from "./slo-engine";
+import type { Logger } from "@checkstack/backend-api";
+import type { AchievementType } from "@checkstack/slo-common";
+
+/**
+ * Achievement definitions with their evaluation criteria.
+ * Each achievement is tied to a system (not a user).
+ */
+const ACHIEVEMENT_DEFINITIONS: Array<{
+  type: AchievementType;
+  description: string;
+  evaluate: (ctx: AchievementContext) => boolean;
+}> = [
+  {
+    type: "first_steps",
+    description: "System has at least one SLO objective configured",
+    evaluate: ({ objectiveCount }) => objectiveCount >= 1,
+  },
+  {
+    type: "full_coverage",
+    description: "System has SLOs covering all health checks",
+    evaluate: ({ objectiveCount }) => objectiveCount >= 3,
+  },
+  {
+    type: "iron_uptime",
+    description: "7-day reliability streak",
+    evaluate: ({ streakDays }) => streakDays >= 7,
+  },
+  {
+    type: "diamond_uptime",
+    description: "30-day reliability streak",
+    evaluate: ({ streakDays }) => streakDays >= 30,
+  },
+  {
+    type: "nines_club",
+    description: "Achieved 99.9% or higher availability in a rolling window",
+    evaluate: ({ bestAvailability }) =>
+      bestAvailability !== undefined && bestAvailability >= 99.9,
+  },
+  {
+    type: "budget_miser",
+    description: "Used less than 10% of error budget over a full window",
+    evaluate: ({ budgetRemainingPercent }) =>
+      budgetRemainingPercent !== undefined && budgetRemainingPercent >= 90,
+  },
+  {
+    type: "clean_sheet",
+    description: "Zero downtime events in the current rolling window",
+    evaluate: ({ hasZeroDowntime }) => hasZeroDowntime,
+  },
+  {
+    type: "cascade_breaker",
+    description:
+      "System was down but all downtime was attributed to upstream dependencies",
+    evaluate: ({ hasUpstreamOnlyDowntime }) => hasUpstreamOnlyDowntime,
+  },
+  {
+    type: "rapid_recovery",
+    description: "Recovered from downtime in under 5 minutes",
+    evaluate: ({ fastestRecoverySeconds }) =>
+      fastestRecoverySeconds !== undefined && fastestRecoverySeconds <= 300,
+  },
+];
+
+interface AchievementContext {
+  objectiveCount: number;
+  streakDays: number;
+  bestAvailability: number | undefined;
+  budgetRemainingPercent: number | undefined;
+  hasZeroDowntime: boolean;
+  hasUpstreamOnlyDowntime: boolean;
+  fastestRecoverySeconds: number | undefined;
+}
+
+/**
+ * Evaluates and unlocks achievements for a specific system.
+ * Called after daily snapshot processing or on significant events.
+ */
+export async function evaluateAchievements({
+  systemId,
+  service,
+  engine,
+  logger,
+}: {
+  systemId: string;
+  service: SloService;
+  engine: SloEngine;
+  logger: Logger;
+}): Promise<AchievementType[]> {
+  const objectives = await service.getObjectivesForSystem({ systemId });
+  if (objectives.length === 0) return [];
+
+  // Build context from all objectives on this system
+  let bestStreakDays = 0;
+  let bestAvailability: number | undefined;
+  let bestBudgetRemaining: number | undefined;
+  let hasZeroDowntime = true;
+  let hasUpstreamOnlyDowntime = false;
+  let fastestRecoverySeconds: number | undefined;
+
+  for (const objective of objectives) {
+    const status = await engine.computeStatus({ objective });
+    const streak = await service.getStreak({ objectiveId: objective.id });
+
+    // Best streak across all objectives
+    if (streak && streak.currentStreak > bestStreakDays) {
+      bestStreakDays = streak.currentStreak;
+    }
+
+    // Best availability across all objectives
+    if (
+      status.currentAvailability !== undefined &&
+      status.currentAvailability !== null &&
+      (bestAvailability === undefined ||
+        status.currentAvailability > bestAvailability)
+    ) {
+      bestAvailability = status.currentAvailability;
+    }
+
+    // Best budget remaining
+    if (
+      bestBudgetRemaining === undefined ||
+      status.errorBudgetRemainingPercent > bestBudgetRemaining
+    ) {
+      bestBudgetRemaining = status.errorBudgetRemainingPercent;
+    }
+
+    // Zero downtime check
+    if (
+      status.errorBudgetConsumedMinutes > 0 ||
+      status.errorBudgetConsumedStrictMinutes > 0
+    ) {
+      hasZeroDowntime = false;
+    }
+
+    // Check for cascade_breaker: consumed strict > 0 but consumed self = 0
+    if (
+      status.errorBudgetConsumedStrictMinutes > 0 &&
+      status.errorBudgetConsumedMinutes === 0
+    ) {
+      hasUpstreamOnlyDowntime = true;
+    }
+
+    // Check recent events for rapid recovery
+    const recentEvents = await service.getRecentDowntimeEvents({
+      objectiveId: objective.id,
+      limit: 10,
+    });
+    for (const event of recentEvents) {
+      if (
+        event.durationSeconds !== undefined &&
+        event.durationSeconds !== null &&
+        (fastestRecoverySeconds === undefined ||
+          event.durationSeconds < fastestRecoverySeconds)
+      ) {
+        fastestRecoverySeconds = event.durationSeconds;
+      }
+    }
+  }
+
+  const context: AchievementContext = {
+    objectiveCount: objectives.length,
+    streakDays: bestStreakDays,
+    bestAvailability,
+    budgetRemainingPercent: bestBudgetRemaining,
+    hasZeroDowntime,
+    hasUpstreamOnlyDowntime,
+    fastestRecoverySeconds,
+  };
+
+  // Evaluate all achievements
+  const newlyUnlocked: AchievementType[] = [];
+
+  for (const definition of ACHIEVEMENT_DEFINITIONS) {
+    if (definition.evaluate(context)) {
+      const achievement = await service.unlockAchievement({
+        systemId,
+        achievement: definition.type,
+      });
+      if (achievement) {
+        newlyUnlocked.push(definition.type);
+        logger.info(
+          `🏆 Achievement unlocked for system ${systemId}: ${definition.type}`,
+        );
+      }
+    }
+  }
+
+  return newlyUnlocked;
+}
+
+/**
+ * Get metadata for all achievement types (for display in the frontend).
+ */
+export function getAchievementDefinitions() {
+  return ACHIEVEMENT_DEFINITIONS.map((d) => ({
+    type: d.type,
+    description: d.description,
+  }));
+}

package/src/hooks.ts

@@ -0,0 +1,76 @@
+import { createHook } from "@checkstack/backend-api";
+
+/**
+ * SLO hooks for cross-plugin communication.
+ * Other plugins can subscribe to these hooks to react to SLO lifecycle events.
+ * Registered as integration events so they flow through configured notification channels.
+ */
+export const sloHooks = {
+  /**
+   * Emitted when an SLO's error budget consumption exceeds the warning threshold.
+   */
+  sloBudgetWarning: createHook<{
+    systemId: string;
+    objectiveId: string;
+    target: number;
+    budgetRemainingPercent: number;
+  }>("slo.budget.warning"),
+
+  /**
+   * Emitted when an SLO's error budget consumption exceeds the critical threshold.
+   */
+  sloBudgetCritical: createHook<{
+    systemId: string;
+    objectiveId: string;
+    target: number;
+    budgetRemainingPercent: number;
+  }>("slo.budget.critical"),
+
+  /**
+   * Emitted when an SLO's error budget is fully exhausted.
+   */
+  sloBudgetExhausted: createHook<{
+    systemId: string;
+    objectiveId: string;
+    target: number;
+  }>("slo.budget.exhausted"),
+
+  /**
+   * Emitted when a reliability streak is broken.
+   */
+  sloStreakBroken: createHook<{
+    systemId: string;
+    objectiveId: string;
+    streak: number;
+    bestStreak: number;
+  }>("slo.streak.broken"),
+
+  /**
+   * Emitted when a system unlocks a new reliability achievement.
+   */
+  sloAchievementUnlocked: createHook<{
+    systemId: string;
+    achievement: string;
+  }>("slo.achievement.unlocked"),
+
+  /**
+   * Emitted weekly with a summary of SLO performance across all systems.
+   * Delivered through configured notification channels (Slack, Teams, etc.).
+   */
+  sloWeeklyDigest: createHook<{
+    totalObjectives: number;
+    breachingCount: number;
+    atRiskCount: number;
+    healthyCount: number;
+    topPerformers: Array<{
+      systemName: string;
+      availability: number;
+      streakDays: number;
+    }>;
+    worstPerformers: Array<{
+      systemName: string;
+      availability: number;
+      budgetRemainingPercent: number;
+    }>;
+  }>("slo.weekly.digest"),
+} as const;

package/src/index.ts

@@ -0,0 +1,425 @@
+import * as schema from "./schema";
+import type { SafeDatabase } from "@checkstack/backend-api";
+import { z } from "zod";
+import {
+  sloAccessRules,
+  sloAccess,
+  pluginMetadata,
+  sloContract,
+  sloRoutes,
+} from "@checkstack/slo-common";
+import { createBackendPlugin, coreServices } from "@checkstack/backend-api";
+import { integrationEventExtensionPoint } from "@checkstack/integration-backend";
+import { SloService } from "./service";
+import { SloEngine } from "./slo-engine";
+import { createRouter } from "./router";
+import { DependencyApi } from "@checkstack/dependency-common";
+import { HealthCheckApi } from "@checkstack/healthcheck-common";
+import { catalogHooks } from "@checkstack/catalog-backend";
+import { healthCheckHooks } from "@checkstack/healthcheck-backend";
+import { registerSearchProvider } from "@checkstack/command-backend";
+import { resolveRoute } from "@checkstack/common";
+import { sloHooks } from "./hooks";
+import { setupDailySnapshotJob } from "./streak-calculator";
+import { setupWeeklyDigestJob } from "./weekly-digest";
+import { evaluateAchievements } from "./achievement-evaluator";
+
+// =============================================================================
+// Integration Event Payload Schemas
+// =============================================================================
+
+const sloBudgetWarningPayloadSchema = z.object({
+  systemId: z.string(),
+  objectiveId: z.string(),
+  target: z.number(),
+  budgetRemainingPercent: z.number(),
+});
+
+const sloBudgetCriticalPayloadSchema = z.object({
+  systemId: z.string(),
+  objectiveId: z.string(),
+  target: z.number(),
+  budgetRemainingPercent: z.number(),
+});
+
+const sloBudgetExhaustedPayloadSchema = z.object({
+  systemId: z.string(),
+  objectiveId: z.string(),
+  target: z.number(),
+});
+
+const sloStreakBrokenPayloadSchema = z.object({
+  systemId: z.string(),
+  objectiveId: z.string(),
+  streak: z.number(),
+  bestStreak: z.number(),
+});
+
+const sloAchievementUnlockedPayloadSchema = z.object({
+  systemId: z.string(),
+  achievement: z.string(),
+});
+
+const sloWeeklyDigestPayloadSchema = z.object({
+  totalObjectives: z.number(),
+  breachingCount: z.number(),
+  atRiskCount: z.number(),
+  healthyCount: z.number(),
+  topPerformers: z.array(
+    z.object({
+      systemName: z.string(),
+      availability: z.number(),
+      streakDays: z.number(),
+    }),
+  ),
+  worstPerformers: z.array(
+    z.object({
+      systemName: z.string(),
+      availability: z.number(),
+      budgetRemainingPercent: z.number(),
+    }),
+  ),
+});
+
+// =============================================================================
+// Plugin Definition
+// =============================================================================
+
+export default createBackendPlugin({
+  metadata: pluginMetadata,
+  register(env) {
+    env.registerAccessRules(sloAccessRules);
+
+    // Register hooks as integration events
+    const integrationEvents = env.getExtensionPoint(
+      integrationEventExtensionPoint,
+    );
+
+    integrationEvents.registerEvent(
+      {
+        hook: sloHooks.sloBudgetWarning,
+        displayName: "SLO Budget Warning",
+        description:
+          "Fired when an SLO error budget consumption exceeds the warning threshold",
+        category: "SLO",
+        payloadSchema: sloBudgetWarningPayloadSchema,
+      },
+      pluginMetadata,
+    );
+
+    integrationEvents.registerEvent(
+      {
+        hook: sloHooks.sloBudgetCritical,
+        displayName: "SLO Budget Critical",
+        description:
+          "Fired when an SLO error budget consumption exceeds the critical threshold",
+        category: "SLO",
+        payloadSchema: sloBudgetCriticalPayloadSchema,
+      },
+      pluginMetadata,
+    );
+
+    integrationEvents.registerEvent(
+      {
+        hook: sloHooks.sloBudgetExhausted,
+        displayName: "SLO Budget Exhausted",
+        description: "Fired when an SLO error budget is fully consumed",
+        category: "SLO",
+        payloadSchema: sloBudgetExhaustedPayloadSchema,
+      },
+      pluginMetadata,
+    );
+
+    integrationEvents.registerEvent(
+      {
+        hook: sloHooks.sloStreakBroken,
+        displayName: "SLO Streak Broken",
+        description: "Fired when a reliability streak is broken",
+        category: "SLO",
+        payloadSchema: sloStreakBrokenPayloadSchema,
+      },
+      pluginMetadata,
+    );
+
+    integrationEvents.registerEvent(
+      {
+        hook: sloHooks.sloAchievementUnlocked,
+        displayName: "SLO Achievement Unlocked",
+        description:
+          "Fired when a system unlocks a new reliability achievement",
+        category: "SLO",
+        payloadSchema: sloAchievementUnlockedPayloadSchema,
+      },
+      pluginMetadata,
+    );
+
+    integrationEvents.registerEvent(
+      {
+        hook: sloHooks.sloWeeklyDigest,
+        displayName: "SLO Weekly Digest",
+        description:
+          "Weekly summary of SLO performance across all systems (Monday 09:00 UTC)",
+        category: "SLO",
+        payloadSchema: sloWeeklyDigestPayloadSchema,
+      },
+      pluginMetadata,
+    );
+
+    // Shared references across init/afterPluginsReady (maintenance-backend pattern)
+    let sharedEngine: SloEngine;
+
+    env.registerInit({
+      schema,
+      deps: {
+        logger: coreServices.logger,
+        rpc: coreServices.rpc,
+        signalService: coreServices.signalService,
+        rpcClient: coreServices.rpcClient,
+        queueManager: coreServices.queueManager,
+      },
+      init: async ({ logger, database, rpc, signalService }) => {
+        logger.debug("🔧 Initializing SLO Backend...");
+
+        const service = new SloService(database as SafeDatabase<typeof schema>);
+        const engine = new SloEngine({
+          service,
+          signalService,
+          logger,
+        });
+
+        // Store for afterPluginsReady
+        sharedEngine = engine;
+
+        const router = createRouter({ service, engine, signalService });
+        rpc.registerRouter(router, sloContract);
+
+        // Register command palette entries
+        registerSearchProvider({
+          pluginMetadata,
+          commands: [
+            {
+              id: "create-slo",
+              title: "Create SLO",
+              subtitle: "Define a new Service Level Objective",
+              iconName: "Target",
+              route: resolveRoute(sloRoutes.routes.config) + "?action=create",
+              requiredAccessRules: [sloAccess.slo.manage],
+            },
+            {
+              id: "manage-slos",
+              title: "Manage SLOs",
+              subtitle: "View and configure Service Level Objectives",
+              iconName: "Target",
+              shortcuts: ["meta+shift+l", "ctrl+shift+l"],
+              route: resolveRoute(sloRoutes.routes.overview),
+            },
+          ],
+        });
+
+        logger.debug("✅ SLO Backend initialized.");
+      },
+
+      afterPluginsReady: async ({
+        database,
+        logger,
+        onHook,
+        emitHook,
+        rpcClient,
+        signalService,
+        queueManager,
+      }) => {
+        const typedDb = database as SafeDatabase<typeof schema>;
+        const service = new SloService(typedDb);
+        const engine = new SloEngine({
+          service,
+          signalService,
+          logger,
+        });
+
+        const dependencyClient = rpcClient.forPlugin(DependencyApi);
+        const healthCheckClient = rpcClient.forPlugin(HealthCheckApi);
+
+        /**
+         * Set health status callback on the shared engine instance
+         * (the one used by the router). This enables reconcileObjective
+         * to check current system health when SLOs are created.
+         */
+        sharedEngine.setHealthStatusCallback(async (systemId) => {
+          try {
+            const status = await healthCheckClient.getSystemHealthStatus({
+              systemId,
+            });
+            logger.debug(
+              `SLO reconcile: System ${systemId} health status = ${status.status}`,
+            );
+            return { isHealthy: status.status === "healthy" };
+          } catch (error) {
+            logger.warn(
+              `SLO reconcile: Failed to get health status for system ${systemId}`,
+              { error },
+            );
+            // Default to healthy if we can't determine status
+            return { isHealthy: true };
+          }
+        });
+
+        /**
+         * Helper: check upstream health status via RPC loopback.
+         * Injected as a callback into the engine for testability.
+         */
+        const getUpstreamHealthStatus = async ({
+          upstreamSystemId,
+        }: {
+          upstreamSystemId: string;
+        }) => {
+          try {
+            const healthStatus = await healthCheckClient.getSystemHealthStatus({
+              systemId: upstreamSystemId,
+            });
+            return {
+              isHealthy: healthStatus.status === "healthy",
+              systemName: upstreamSystemId,
+            };
+          } catch {
+            // Fail-open: if we can't check upstream, assume healthy
+            return { isHealthy: true, systemName: upstreamSystemId };
+          }
+        };
+
+        /**
+         * Helper: get downstream dependents of a system.
+         */
+        const getDownstreamSystemIds = async (
+          systemId: string,
+        ): Promise<string[]> => {
+          try {
+            const result = await dependencyClient.getDependencies({
+              systemId,
+              direction: "downstream",
+            });
+            return result.dependencies.map((d) => d.sourceSystemId);
+          } catch {
+            return [];
+          }
+        };
+
+        // =====================================================================
+        // Perspective 1: System goes DOWN — open downtime events
+        // =====================================================================
+        onHook(
+          healthCheckHooks.systemDegraded,
+          async (payload) => {
+            logger.debug(
+              `SLO: System ${payload.systemId} degraded (${payload.previousStatus} → ${payload.newStatus})`,
+            );
+            await engine.handleSystemDown({
+              systemId: payload.systemId,
+              getUpstreamHealthStatus,
+            });
+          },
+          { mode: "work-queue", workerGroup: "slo-system-down" },
+        );
+
+        // =====================================================================
+        // Perspective 1: System goes UP — close downtime events
+        // =====================================================================
+        onHook(
+          healthCheckHooks.systemHealthy,
+          async (payload) => {
+            logger.debug(`SLO: System ${payload.systemId} recovered`);
+            await engine.handleSystemUp({
+              systemId: payload.systemId,
+            });
+
+            // Also handle Perspective 2 (as upstream)
+            const downstreamIds = await getDownstreamSystemIds(
+              payload.systemId,
+            );
+            if (downstreamIds.length > 0) {
+              await engine.handleUpstreamUp({
+                upstreamSystemId: payload.systemId,
+                downstreamSystemIds: downstreamIds,
+                getUpstreamHealthStatus,
+              });
+            }
+
+            // Evaluate achievements on recovery (rapid_recovery, clean_sheet, etc.)
+            await evaluateAchievements({
+              systemId: payload.systemId,
+              service,
+              engine,
+              logger,
+            });
+          },
+          { mode: "work-queue", workerGroup: "slo-system-up" },
+        );
+
+        // =====================================================================
+        // Perspective 2: Upstream degraded — split downstream "self" events
+        // We re-use the systemDegraded hook, checking downstream systems
+        // =====================================================================
+        onHook(
+          healthCheckHooks.systemDegraded,
+          async (payload) => {
+            const downstreamIds = await getDownstreamSystemIds(
+              payload.systemId,
+            );
+            if (downstreamIds.length > 0) {
+              await engine.handleUpstreamDown({
+                upstreamSystemId: payload.systemId,
+                upstreamSystemName: payload.systemName ?? payload.systemId,
+                downstreamSystemIds: downstreamIds,
+              });
+            }
+          },
+          { mode: "work-queue", workerGroup: "slo-upstream-down" },
+        );
+
+        // =====================================================================
+        // Subscribe to catalog system deletion for cleanup
+        // =====================================================================
+        onHook(
+          catalogHooks.systemDeleted,
+          async (payload) => {
+            logger.debug(
+              `Cleaning up SLO data for deleted system: ${payload.systemId}`,
+            );
+            await service.deleteObjectivesForSystem({
+              systemId: payload.systemId,
+            });
+            await service.deleteAchievementsForSystem({
+              systemId: payload.systemId,
+            });
+          },
+          { mode: "work-queue", workerGroup: "slo-system-cleanup" },
+        );
+
+        // =====================================================================
+        // Daily snapshot + streak calculation cron job
+        // =====================================================================
+        await setupDailySnapshotJob({
+          service,
+          engine,
+          logger,
+          queueManager,
+        });
+
+        // =====================================================================
+        // Weekly digest cron job
+        // =====================================================================
+        await setupWeeklyDigestJob({
+          service,
+          engine,
+          logger,
+          queueManager,
+          emitHook,
+        });
+
+        logger.debug("✅ SLO Backend afterPluginsReady complete.");
+      },
+    });
+  },
+});
+
+// Re-export hooks for other plugins to use
+export { sloHooks } from "./hooks";