@checkstack/healthcheck-backend 1.19.0 → 1.20.0

package/drizzle/meta/_journal.json

@@ -134,6 +134,20 @@
       "when": 1780357035712,
       "tag": "0018_abnormal_preak",
       "breakpoints": true
+    },
+    {
+      "idx": 19,
+      "version": "7",
+      "when": 1783585832198,
+      "tag": "0019_chemical_frightful_four",
+      "breakpoints": true
+    },
+    {
+      "idx": 20,
+      "version": "7",
+      "when": 1783607735562,
+      "tag": "0020_certain_mordo",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/healthcheck-backend",
-  "version": "1.19.0",
+  "version": "1.20.0",
   "license": "Elastic-2.0",
   "type": "module",
   "main": "src/index.ts",
@@ -14,32 +14,32 @@
     "lint:code": "eslint . --max-warnings 0"
   },
   "dependencies": {
-    "@checkstack/ai-backend": "0.10.10",
+    "@checkstack/ai-backend": "0.10.11",
     "@checkstack/ai-common": "0.6.6",
-    "@checkstack/automation-backend": "0.11.1",
-    "@checkstack/backend-api": "0.31.1",
+    "@checkstack/automation-backend": "0.11.2",
+    "@checkstack/backend-api": "0.32.0",
     "@checkstack/cache-api": "0.3.19",
-    "@checkstack/cache-utils": "0.2.24",
-    "@checkstack/catalog-backend": "1.7.0",
-    "@checkstack/catalog-common": "2.7.0",
-    "@checkstack/command-backend": "0.2.22",
+    "@checkstack/cache-utils": "0.3.0",
+    "@checkstack/catalog-backend": "1.8.0",
+    "@checkstack/catalog-common": "2.7.1",
+    "@checkstack/command-backend": "0.2.23",
     "@checkstack/common": "0.22.0",
-    "@checkstack/gitops-backend": "0.5.22",
+    "@checkstack/gitops-backend": "0.5.23",
     "@checkstack/gitops-common": "0.7.3",
-    "@checkstack/healthcheck-common": "1.16.0",
-    "@checkstack/incident-backend": "1.12.0",
-    "@checkstack/incident-common": "1.10.0",
-    "@checkstack/maintenance-common": "1.10.0",
-    "@checkstack/notification-common": "1.6.0",
+    "@checkstack/healthcheck-common": "1.16.1",
+    "@checkstack/incident-backend": "1.13.0",
+    "@checkstack/incident-common": "1.10.1",
+    "@checkstack/maintenance-common": "1.10.1",
+    "@checkstack/notification-common": "1.7.0",
     "@checkstack/queue-api": "0.3.19",
-    "@checkstack/satellite-backend": "0.8.4",
-    "@checkstack/script-packages-backend": "0.4.1",
-    "@checkstack/sdk": "0.127.1",
-    "@checkstack/secrets-backend": "0.3.4",
+    "@checkstack/satellite-backend": "0.8.5",
+    "@checkstack/script-packages-backend": "0.4.2",
+    "@checkstack/sdk": "0.128.1",
+    "@checkstack/secrets-backend": "0.3.5",
     "@checkstack/secrets-common": "0.3.2",
     "@checkstack/signal-common": "0.2.17",
-    "@checkstack/status-page-backend": "0.5.0",
-    "@checkstack/status-page-common": "0.6.0",
+    "@checkstack/status-page-backend": "0.6.0",
+    "@checkstack/status-page-common": "0.6.1",
     "@hono/zod-validator": "^0.7.6",
     "@orpc/contract": "^1.14.4",
     "@orpc/server": "^1.14.4",
@@ -53,7 +53,7 @@
   "devDependencies": {
     "@checkstack/drizzle-helper": "0.0.6",
     "@checkstack/scripts": "0.7.3",
-    "@checkstack/test-utils-backend": "0.1.56",
+    "@checkstack/test-utils-backend": "0.1.57",
     "@checkstack/tsconfig": "0.0.7",
     "@types/bun": "^1.0.0",
     "@types/pg": "^8.20.0",

package/src/ai/system-signals-contributor.test.ts

@@ -7,7 +7,8 @@ import type { AuthUser } from "@checkstack/backend-api";
 import type { SystemAccessResolver } from "@checkstack/ai-backend";
 import {
   createHealthcheckSignalsContributor,
-  type HealthcheckSignalsSource,
+  type HealthcheckCandidateSource,
+  type HealthcheckStatusCacheReader,
 } from "./system-signals-contributor";
 
 const unhealthyStatuses: HealthcheckSignalStatuses = {
@@ -27,10 +28,30 @@ const unhealthyStatuses: HealthcheckSignalStatuses = {
   },
 };
 
+/**
+ * Build a candidate source + cache reader pair from a fixed status map: the
+ * candidate ids are the map's keys, and `readBulk` returns the requested subset.
+ * This mirrors the real wiring (service scans candidates, cache serves statuses).
+ */
 function sourceReturning(statuses: HealthcheckSignalStatuses): {
-  source: HealthcheckSignalsSource;
+  candidateSource: HealthcheckCandidateSource;
+  cache: HealthcheckStatusCacheReader;
 } {
-  return { source: { getAllUnhealthySystemStatuses: async () => statuses } };
+  return {
+    candidateSource: {
+      getUnhealthyCandidateSystemIds: async () => Object.keys(statuses),
+    },
+    cache: {
+      readBulk: async (systemIds) => {
+        const out: HealthcheckSignalStatuses = {};
+        for (const id of systemIds) {
+          const s = statuses[id];
+          if (s) out[id] = s;
+        }
+        return out;
+      },
+    },
+  };
 }
 
 // The per-source gate is owned/tested by createGatedSystemSignalsContributor;
@@ -47,18 +68,20 @@ const userWith = (accessRules: string[]): AuthUser => ({
 
 describe("createHealthcheckSignalsContributor", () => {
   it("exposes the shared source id", () => {
-    const { source } = sourceReturning({});
+    const { candidateSource, cache } = sourceReturning({});
     const contributor = createHealthcheckSignalsContributor({
-      service: source,
+      candidateSource,
+      cache,
       resolver: allowAll,
     });
     expect(contributor.sourceId).toBe(HEALTHCHECK_SIGNAL_SOURCE_ID);
   });
 
   it("wires the service + shared deriver for an authorized principal", async () => {
-    const { source } = sourceReturning(unhealthyStatuses);
+    const { candidateSource, cache } = sourceReturning(unhealthyStatuses);
     const contributor = createHealthcheckSignalsContributor({
-      service: source,
+      candidateSource,
+      cache,
       resolver: allowAll,
     });
 
@@ -75,9 +98,10 @@ describe("createHealthcheckSignalsContributor", () => {
   });
 
   it("routes a non-global user through the team gate (no grants -> nothing)", async () => {
-    const { source } = sourceReturning(unhealthyStatuses);
+    const { candidateSource, cache } = sourceReturning(unhealthyStatuses);
     const contributor = createHealthcheckSignalsContributor({
-      service: source,
+      candidateSource,
+      cache,
       resolver: denyAll,
     });
 

package/src/ai/system-signals-contributor.ts

@@ -11,36 +11,58 @@ import {
 } from "@checkstack/ai-backend";
 
 /**
- * The slice of `HealthCheckService` this contributor needs: a global,
- * pod-independent scan of every currently-degraded/unhealthy system. Narrowed
- * to an interface so the contributor is unit-testable without constructing the
- * full service (or a database).
+ * The slice of `HealthCheckService` this contributor needs: the candidate set
+ * for the global problem scan (systems with at least one enabled check
+ * association). Narrowed to an interface so the contributor is unit-testable
+ * without constructing the full service (or a database).
  */
-export interface HealthcheckSignalsSource {
-  getAllUnhealthySystemStatuses(): Promise<HealthcheckSignalStatuses>;
+export interface HealthcheckCandidateSource {
+  getUnhealthyCandidateSystemIds(): Promise<string[]>;
 }
 
 /**
- * Build the healthcheck contributor for the AI `system.issues` aggregator. Reads
- * the global durable scan of unhealthy systems and runs the SAME deriver the
- * dashboard filler uses. The per-source access gate (global `healthcheck.status`
- * plus per-system team grants) is applied by
+ * The slice of `HealthCheckCache` this contributor reads through: the per-entity
+ * bulk status read. Routing the scan through the cache (rather than the service's
+ * uncached N+1 `getAllUnhealthySystemStatuses`) means the AI signals poll reuses
+ * the warm badge/dashboard cache and does not fan out one DB derivation per
+ * system on every poll.
+ */
+export interface HealthcheckStatusCacheReader {
+  readBulk(systemIds: string[]): Promise<HealthcheckSignalStatuses>;
+}
+
+/**
+ * Build the healthcheck contributor for the AI `system.issues` aggregator. Scans
+ * the candidate systems (durable, pod-independent) and resolves each one's status
+ * through the SHARED cache, then runs the SAME deriver the dashboard filler uses
+ * (problems only - healthy systems are dropped). The per-source access gate
+ * (global `healthcheck.status` plus per-system team grants) is applied by
  * {@link createGatedSystemSignalsContributor}.
  */
 export function createHealthcheckSignalsContributor({
-  service,
+  candidateSource,
+  cache,
   resolver,
 }: {
-  service: HealthcheckSignalsSource;
+  candidateSource: HealthcheckCandidateSource;
+  cache: HealthcheckStatusCacheReader;
   resolver: SystemAccessResolver;
 }): SystemSignalsContributor {
   return createGatedSystemSignalsContributor({
     sourceId: HEALTHCHECK_SIGNAL_SOURCE_ID,
     accessRule: healthCheckAccess.status,
     resolver,
-    readSignals: async () =>
-      deriveHealthcheckSignals({
-        statuses: await service.getAllUnhealthySystemStatuses(),
-      }),
+    readSignals: async () => {
+      const systemIds = await candidateSource.getUnhealthyCandidateSystemIds();
+      const all = await cache.readBulk(systemIds);
+      // Keep only degraded/unhealthy systems (the same filter the uncached scan
+      // applied); the deriver drops healthy/unknown anyway, but filtering here
+      // keeps the payload small.
+      const statuses: HealthcheckSignalStatuses = {};
+      for (const [systemId, status] of Object.entries(all)) {
+        if (status.status !== "healthy") statuses[systemId] = status;
+      }
+      return deriveHealthcheckSignals({ statuses });
+    },
   });
 }

package/src/cache-test-stub.ts

@@ -0,0 +1,26 @@
+import { mock } from "bun:test";
+import type { HealthCheckCache } from "./cache";
+
+/**
+ * A no-op {@link HealthCheckCache} for tests that need to satisfy the router /
+ * executor `cache` dependency without exercising real caching. Every method is a
+ * `bun:test` mock so a test can assert calls or override behavior via
+ * `overrides`. `read` throws by default (a test that reaches the read path must
+ * opt in by overriding it), while the invalidators/reconcile no-op so mutation
+ * and run paths run unimpeded.
+ */
+export function createStubHealthCheckCache(
+  overrides: Partial<HealthCheckCache> = {},
+): HealthCheckCache {
+  return {
+    read: mock(async () => {
+      throw new Error("stub HealthCheckCache.read not configured");
+    }),
+    readBulk: mock(async () => ({})),
+    readMatrix: mock(async () => ({})),
+    reconcile: mock(async () => {}),
+    invalidateSystem: mock(async () => {}),
+    invalidateAllSystems: mock(async () => 0),
+    ...overrides,
+  };
+}

package/src/cache.test.ts

@@ -0,0 +1,291 @@
+import { describe, it, expect, mock } from "bun:test";
+import type { CacheManager, CacheProvider } from "@checkstack/cache-api";
+import type { SystemHealthStatusResponse } from "@checkstack/healthcheck-common";
+import { createHealthCheckCache, type HealthStatusReader } from "./cache";
+
+function createMemoryProvider(): CacheProvider {
+  const store = new Map<string, unknown>();
+  return {
+    get: async <T>(key: string) => store.get(key) as T | undefined,
+    set: async (key, value) => {
+      store.set(key, value);
+    },
+    delete: async (key) => {
+      store.delete(key);
+    },
+    deleteByPrefix: async (prefix) => {
+      let removed = 0;
+      for (const key of [...store.keys()]) {
+        if (key.startsWith(prefix)) {
+          store.delete(key);
+          removed++;
+        }
+      }
+      return removed;
+    },
+    has: async (key) => store.has(key),
+  };
+}
+
+function createManager(provider: CacheProvider): CacheManager {
+  return {
+    getProvider: () => provider,
+    getActivePlugin: () => "test",
+    getActiveConfig: () => ({}),
+    setActiveBackend: async () => {},
+    shutdown: async () => {},
+  } as unknown as CacheManager;
+}
+
+const silentLogger = {
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as never;
+
+function response(
+  status: SystemHealthStatusResponse["status"],
+  checks: {
+    configurationId: string;
+    status: SystemHealthStatusResponse["status"];
+    failingSliceCount?: number;
+    sliceCount?: number;
+  }[] = [],
+): SystemHealthStatusResponse {
+  return {
+    status,
+    evaluatedAt: new Date(0),
+    checkStatuses: checks.map((c) => ({
+      configurationId: c.configurationId,
+      configurationName: c.configurationId,
+      status: c.status,
+      runsConsidered: 1,
+      sliceCount: c.sliceCount ?? 1,
+      failingSliceCount: c.failingSliceCount ?? 0,
+    })),
+  };
+}
+
+function setup(
+  opts: {
+    reader?: Partial<HealthStatusReader>;
+  } = {},
+) {
+  const provider = createMemoryProvider();
+  const reader: HealthStatusReader = {
+    getSystemHealthStatus:
+      opts.reader?.getSystemHealthStatus ?? (async () => response("healthy")),
+    getSystemEnvironmentIds:
+      opts.reader?.getSystemEnvironmentIds ?? (async () => []),
+  };
+  const cache = createHealthCheckCache({
+    cacheManager: createManager(provider),
+    logger: silentLogger,
+    service: reader,
+  });
+  return { cache, provider };
+}
+
+describe("HealthCheckCache.read", () => {
+  it("caches the loader value (second read does not re-hit the service)", async () => {
+    const getSystemHealthStatus = mock(async () => response("degraded"));
+    const { cache } = setup({ reader: { getSystemHealthStatus } });
+
+    const first = await cache.read("sys-1");
+    const second = await cache.read("sys-1");
+
+    expect(first.status).toBe("degraded");
+    expect(second.status).toBe("degraded");
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(1);
+  });
+
+  it("keys per (system, environment)", async () => {
+    const getSystemHealthStatus = mock(
+      async (_systemId: string, environmentId?: string | null) =>
+        response(environmentId === "prod" ? "unhealthy" : "healthy"),
+    );
+    const { cache } = setup({ reader: { getSystemHealthStatus } });
+
+    const rollup = await cache.read("sys-1");
+    const prod = await cache.read("sys-1", "prod");
+
+    expect(rollup.status).toBe("healthy");
+    expect(prod.status).toBe("unhealthy");
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(2);
+  });
+});
+
+describe("HealthCheckCache.reconcile", () => {
+  it("evicts only when the per-check vector changed", async () => {
+    let current = response("healthy", [
+      { configurationId: "c1", status: "healthy" },
+    ]);
+    const getSystemHealthStatus = mock(async () => current);
+    const { cache } = setup({ reader: { getSystemHealthStatus } });
+
+    // Warm the cache.
+    await cache.read("sys-1");
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(1);
+
+    // A run whose vector is unchanged: no evict, cache stays warm.
+    await cache.reconcile({
+      systemId: "sys-1",
+      previous: response("healthy", [
+        { configurationId: "c1", status: "healthy" },
+      ]),
+      next: response("healthy", [
+        { configurationId: "c1", status: "healthy" },
+      ]),
+    });
+    await cache.read("sys-1");
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(1); // served from cache
+
+    // A run that flips a check: evict; next read reloads.
+    current = response("degraded", [
+      { configurationId: "c1", status: "degraded" },
+    ]);
+    await cache.reconcile({
+      systemId: "sys-1",
+      previous: response("healthy", [
+        { configurationId: "c1", status: "healthy" },
+      ]),
+      next: current,
+    });
+    const reloaded = await cache.read("sys-1");
+    expect(reloaded.status).toBe("degraded");
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(2);
+  });
+
+  it("a per-env change evicts BOTH its env key and the rollup key", async () => {
+    const getSystemHealthStatus = mock(
+      async (_systemId: string, environmentId?: string | null) =>
+        response(environmentId === "prod" ? "unhealthy" : "healthy"),
+    );
+    const { cache } = setup({ reader: { getSystemHealthStatus } });
+
+    await cache.read("sys-1"); // warm rollup
+    await cache.read("sys-1", "prod"); // warm env
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(2);
+
+    await cache.reconcile({
+      systemId: "sys-1",
+      environmentId: "prod",
+      previous: response("healthy", [
+        { configurationId: "c1", status: "healthy" },
+      ]),
+      next: response("unhealthy", [
+        { configurationId: "c1", status: "unhealthy" },
+      ]),
+    });
+
+    // Both the rollup and the prod key reload from the service (sibling env
+    // keys would stay warm, but there are none here).
+    await cache.read("sys-1");
+    await cache.read("sys-1", "prod");
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(4);
+  });
+
+  it("handles a simultaneous slice swap the rollup fingerprint is blind to", async () => {
+    // A: unhealthy->healthy, B: healthy->unhealthy. The ROLLUP fingerprint does
+    // not move (failingSliceCount stays 1, worst status stays put), so the
+    // rollup is kept fresh by the per-env reconciles, not by a rollup diff.
+    let rollupCall = 0;
+    const getSystemHealthStatus = mock(
+      async (_systemId: string, environmentId?: string | null) => {
+        if (environmentId === undefined || environmentId === null) {
+          rollupCall++;
+          return response("unhealthy", [
+            {
+              configurationId: "c1",
+              status: "unhealthy",
+              sliceCount: 2,
+              failingSliceCount: 1,
+            },
+          ]);
+        }
+        return response("healthy");
+      },
+    );
+    const { cache } = setup({ reader: { getSystemHealthStatus } });
+
+    await cache.read("sys-1"); // warm rollup (rollupCall -> 1)
+    await cache.reconcile({
+      systemId: "sys-1",
+      environmentId: "a",
+      previous: response("unhealthy", [
+        { configurationId: "c1", status: "unhealthy" },
+      ]),
+      next: response("healthy", [
+        { configurationId: "c1", status: "healthy" },
+      ]),
+    });
+    await cache.reconcile({
+      systemId: "sys-1",
+      environmentId: "b",
+      previous: response("healthy", [
+        { configurationId: "c1", status: "healthy" },
+      ]),
+      next: response("unhealthy", [
+        { configurationId: "c1", status: "unhealthy" },
+      ]),
+    });
+
+    await cache.read("sys-1");
+    expect(rollupCall).toBe(2); // rollup was evicted + reloaded despite a stable fingerprint
+  });
+});
+
+describe("HealthCheckCache invalidators", () => {
+  it("invalidateSystem evicts the rollup + every per-env key", async () => {
+    const getSystemHealthStatus = mock(async () => response("healthy"));
+    const { cache } = setup({ reader: { getSystemHealthStatus } });
+
+    await cache.read("sys-1"); // warm rollup
+    await cache.read("sys-1", "prod"); // warm env
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(2);
+
+    await cache.invalidateSystem("sys-1");
+
+    await cache.read("sys-1");
+    await cache.read("sys-1", "prod");
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(4); // both reloaded
+  });
+
+  it("invalidateAllSystems evicts every system's status", async () => {
+    const getSystemHealthStatus = mock(async () => response("healthy"));
+    const { cache } = setup({ reader: { getSystemHealthStatus } });
+
+    await cache.read("sys-1");
+    await cache.read("sys-2");
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(2);
+
+    const removed = await cache.invalidateAllSystems();
+    expect(removed).toBe(2);
+
+    await cache.read("sys-1");
+    await cache.read("sys-2");
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(4); // both reloaded
+  });
+});
+
+describe("HealthCheckCache.readMatrix", () => {
+  it("assembles rollup + per-env slices from the cache", async () => {
+    const getSystemHealthStatus = mock(
+      async (_systemId: string, environmentId?: string | null) =>
+        response(environmentId === "prod" ? "unhealthy" : "healthy", [
+          { configurationId: "c1", status: "healthy" },
+        ]),
+    );
+    const { cache } = setup({
+      reader: {
+        getSystemHealthStatus,
+        getSystemEnvironmentIds: async () => ["prod"],
+      },
+    });
+
+    const matrix = await cache.readMatrix(["sys-1"]);
+    expect(matrix["sys-1"]!.status).toBe("healthy");
+    expect(matrix["sys-1"]!.environments["prod"]!.status).toBe("unhealthy");
+  });
+});