@checkstack/healthcheck-backend 1.6.7 → 1.7.0

package/CHANGELOG.md

@@ -1,5 +1,82 @@
 # @checkstack/healthcheck-backend
 
+## 1.7.0
+
+### Minor Changes
+
+- 0b6f01b: feat(healthcheck): contribute health problems to the backend system.issues aggregator
+
+  The healthcheck plugin now registers a `system.issues` contributor (sourceId
+  `healthcheck`) from its backend `init`, so the AI assistant surfaces degraded
+  and unhealthy systems alongside incidents, SLOs, anomalies, and dependency
+  problems.
+
+  The contributor enforces its own `healthcheck.status` access gate (returning an
+  empty map - never throwing - when the principal lacks access; service users get
+  no signals), then reads the current problem rows for every system from the
+  shared, durable `health_check_runs` / `system_health_checks` tables via a new
+  global `getAllUnhealthySystemStatuses` service method (every system with an
+  enabled check association, evaluated with the same per-system evaluator the
+  dashboard uses, healthy systems omitted). The answer is therefore identical on
+  every pod, and only systems with a current problem appear in the result.
+
+  The row->signal mapping (source/tone/label/detail/href/accessRule/iconName) is
+  extracted into a new pure `deriveHealthcheckSignals` deriver in
+  `@checkstack/healthcheck-common`, shared by both the backend contributor and the
+  frontend `HealthSignalsFiller` so the two surfaces stay in lockstep. The
+  frontend filler now delegates to that deriver with unchanged behavior.
+
+### Patch Changes
+
+- dbb76a2: fix(ai): guide the assistant to find all issues and fix the anomaly tool
+
+  Two assistant problems reported in production:
+
+  1. Asked "are there any issues?", the model answered from a single source (an
+     SLO breach) and missed a system with a failing health check. The chat
+     system prompt now instructs the model to check ALL issue sources before
+     answering - failing health checks (`healthcheck_status`), breaching/at-risk
+     SLOs (`slo_listObjectives`), active anomalies (`anomaly_list`), and open
+     incidents (`incident_list`) - and not to stop after the first source. It
+     also tells the model that `systemId` must be a real system UUID (resolve a
+     name via the catalog tool first) and to never invent ids or filter values.
+
+  2. The anomaly tool was named `anomaly.explain` but actually LISTS anomalies
+     with optional filters. The misleading name led the model to pass a
+     non-existent filter value ("Type validation failed") and a system
+     name/anomaly id as `systemId` ("a value was malformed"). Renamed to
+     `anomaly.list` with a description that spells out the optional filters and
+     their valid enum values (state: suspicious|anomaly|recovered, kind:
+     spike|drift, suppression: active|suppressed|all) and that `systemId` is a
+     system UUID.
+
+  Also sharpened the `healthcheck.status` and `slo.listObjectives` tool
+  descriptions to be use-case oriented ("use when asked what is failing /
+  breaching").
+
+  BREAKING: the anomaly read tool's name changes from `anomaly_explain` to
+  `anomaly_list` over the MCP `tools/list` surface. MCP clients referencing it by
+  the old name must update.
+
+- Updated dependencies [dbb76a2]
+- Updated dependencies [0b6f01b]
+- Updated dependencies [0b6f01b]
+- Updated dependencies [0b6f01b]
+  - @checkstack/ai-backend@0.3.0
+  - @checkstack/healthcheck-common@1.6.0
+  - @checkstack/incident-backend@1.7.0
+  - @checkstack/incident-common@1.5.0
+  - @checkstack/maintenance-common@1.5.0
+  - @checkstack/automation-backend@0.5.8
+  - @checkstack/catalog-backend@1.4.8
+  - @checkstack/satellite-backend@0.6.8
+  - @checkstack/sdk@0.103.1
+  - @checkstack/backend-api@0.21.6
+  - @checkstack/script-packages-backend@0.3.7
+  - @checkstack/command-backend@0.2.6
+  - @checkstack/gitops-backend@0.5.6
+  - @checkstack/secrets-backend@0.2.6
+
 ## 1.6.7
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/healthcheck-backend",
-  "version": "1.6.7",
+  "version": "1.7.0",
   "license": "Elastic-2.0",
   "type": "module",
   "main": "src/index.ts",
@@ -14,29 +14,29 @@
     "lint:code": "eslint . --max-warnings 0"
   },
   "dependencies": {
-    "@checkstack/backend-api": "0.21.5",
-    "@checkstack/ai-backend": "0.2.0",
+    "@checkstack/backend-api": "0.21.6",
+    "@checkstack/ai-backend": "0.3.0",
     "@checkstack/ai-common": "0.1.3",
-    "@checkstack/script-packages-backend": "0.3.6",
+    "@checkstack/script-packages-backend": "0.3.7",
     "@checkstack/cache-api": "0.3.12",
     "@checkstack/cache-utils": "0.2.17",
-    "@checkstack/catalog-backend": "1.4.7",
+    "@checkstack/catalog-backend": "1.4.8",
     "@checkstack/catalog-common": "2.3.4",
-    "@checkstack/command-backend": "0.2.5",
+    "@checkstack/command-backend": "0.2.6",
     "@checkstack/common": "0.15.0",
-    "@checkstack/gitops-backend": "0.5.5",
+    "@checkstack/gitops-backend": "0.5.6",
     "@checkstack/gitops-common": "0.6.3",
-    "@checkstack/healthcheck-common": "1.5.4",
+    "@checkstack/healthcheck-common": "1.6.0",
     "@checkstack/secrets-common": "0.2.3",
-    "@checkstack/secrets-backend": "0.2.5",
-    "@checkstack/incident-backend": "1.6.7",
-    "@checkstack/incident-common": "1.4.4",
-    "@checkstack/automation-backend": "0.5.7",
-    "@checkstack/maintenance-common": "1.4.4",
+    "@checkstack/secrets-backend": "0.2.6",
+    "@checkstack/incident-backend": "1.7.0",
+    "@checkstack/incident-common": "1.5.0",
+    "@checkstack/automation-backend": "0.5.8",
+    "@checkstack/maintenance-common": "1.5.0",
     "@checkstack/notification-common": "1.3.3",
     "@checkstack/queue-api": "0.3.12",
-    "@checkstack/satellite-backend": "0.6.7",
-    "@checkstack/sdk": "0.102.0",
+    "@checkstack/satellite-backend": "0.6.8",
+    "@checkstack/sdk": "0.103.1",
     "@checkstack/signal-common": "0.2.9",
     "@hono/zod-validator": "^0.7.6",
     "drizzle-orm": "^0.45.0",
@@ -50,7 +50,7 @@
   "devDependencies": {
     "@checkstack/drizzle-helper": "0.0.5",
     "@checkstack/scripts": "0.6.1",
-    "@checkstack/test-utils-backend": "0.1.39",
+    "@checkstack/test-utils-backend": "0.1.40",
     "@checkstack/tsconfig": "0.0.7",
     "@types/bun": "^1.0.0",
     "@types/tdigest": "^0.1.5",

package/src/ai/system-signals-contributor.test.ts

@@ -0,0 +1,86 @@
+import { describe, it, expect } from "bun:test";
+import {
+  HEALTHCHECK_SIGNAL_SOURCE_ID,
+  type HealthcheckSignalStatuses,
+} from "@checkstack/healthcheck-common";
+import type { AuthUser } from "@checkstack/backend-api";
+import type { SystemAccessResolver } from "@checkstack/ai-backend";
+import {
+  createHealthcheckSignalsContributor,
+  type HealthcheckSignalsSource,
+} from "./system-signals-contributor";
+
+const unhealthyStatuses: HealthcheckSignalStatuses = {
+  s1: {
+    status: "unhealthy",
+    evaluatedAt: new Date(),
+    checkStatuses: [
+      {
+        configurationId: "c1",
+        configurationName: "Ping",
+        status: "unhealthy",
+        runsConsidered: 5,
+      },
+    ],
+  },
+};
+
+function sourceReturning(statuses: HealthcheckSignalStatuses): {
+  source: HealthcheckSignalsSource;
+} {
+  return { source: { getAllUnhealthySystemStatuses: async () => statuses } };
+}
+
+// The per-source gate is owned/tested by createGatedSystemSignalsContributor;
+// these resolver stubs let us focus on this plugin's wiring.
+const allowAll: SystemAccessResolver = {
+  accessibleSystemIds: async ({ systemIds }) => systemIds,
+};
+const denyAll: SystemAccessResolver = { accessibleSystemIds: async () => [] };
+const userWith = (accessRules: string[]): AuthUser => ({
+  type: "user",
+  id: "u1",
+  accessRules,
+});
+
+describe("createHealthcheckSignalsContributor", () => {
+  it("exposes the shared source id", () => {
+    const { source } = sourceReturning({});
+    const contributor = createHealthcheckSignalsContributor({
+      service: source,
+      resolver: allowAll,
+    });
+    expect(contributor.sourceId).toBe(HEALTHCHECK_SIGNAL_SOURCE_ID);
+  });
+
+  it("wires the service + shared deriver for an authorized principal", async () => {
+    const { source } = sourceReturning(unhealthyStatuses);
+    const contributor = createHealthcheckSignalsContributor({
+      service: source,
+      resolver: allowAll,
+    });
+
+    const result = await contributor.read({
+      principal: userWith(["healthcheck.healthcheck.status.read"]),
+    });
+
+    expect(Object.keys(result.signals)).toEqual(["s1"]);
+    expect(result.signals.s1[0]).toMatchObject({
+      source: HEALTHCHECK_SIGNAL_SOURCE_ID,
+      tone: "error",
+      label: "Unhealthy",
+    });
+  });
+
+  it("routes a non-global user through the team gate (no grants -> nothing)", async () => {
+    const { source } = sourceReturning(unhealthyStatuses);
+    const contributor = createHealthcheckSignalsContributor({
+      service: source,
+      resolver: denyAll,
+    });
+
+    const result = await contributor.read({ principal: userWith([]) });
+
+    expect(result).toEqual({ accessible: false, signals: {} });
+  });
+});

package/src/ai/system-signals-contributor.ts

@@ -0,0 +1,46 @@
+import {
+  deriveHealthcheckSignals,
+  healthCheckAccess,
+  HEALTHCHECK_SIGNAL_SOURCE_ID,
+  type HealthcheckSignalStatuses,
+} from "@checkstack/healthcheck-common";
+import {
+  createGatedSystemSignalsContributor,
+  type SystemAccessResolver,
+  type SystemSignalsContributor,
+} from "@checkstack/ai-backend";
+
+/**
+ * The slice of `HealthCheckService` this contributor needs: a global,
+ * pod-independent scan of every currently-degraded/unhealthy system. Narrowed
+ * to an interface so the contributor is unit-testable without constructing the
+ * full service (or a database).
+ */
+export interface HealthcheckSignalsSource {
+  getAllUnhealthySystemStatuses(): Promise<HealthcheckSignalStatuses>;
+}
+
+/**
+ * Build the healthcheck contributor for the AI `system.issues` aggregator. Reads
+ * the global durable scan of unhealthy systems and runs the SAME deriver the
+ * dashboard filler uses. The per-source access gate (global `healthcheck.status`
+ * plus per-system team grants) is applied by
+ * {@link createGatedSystemSignalsContributor}.
+ */
+export function createHealthcheckSignalsContributor({
+  service,
+  resolver,
+}: {
+  service: HealthcheckSignalsSource;
+  resolver: SystemAccessResolver;
+}): SystemSignalsContributor {
+  return createGatedSystemSignalsContributor({
+    sourceId: HEALTHCHECK_SIGNAL_SOURCE_ID,
+    accessRule: healthCheckAccess.status,
+    resolver,
+    readSignals: async () =>
+      deriveHealthcheckSignals({
+        statuses: await service.getAllUnhealthySystemStatuses(),
+      }),
+  });
+}

package/src/index.ts

@@ -21,8 +21,11 @@ import {
   aiToolExtensionPoint,
   aiToolProjectionExtensionPoint,
   deferredProjectionExecute,
+  systemSignalsExtensionPoint,
+  createSystemAccessResolver,
 } from "@checkstack/ai-backend";
 import { buildHealthcheckAiTools } from "./ai/register-ai-tools";
+import { createHealthcheckSignalsContributor } from "./ai/system-signals-contributor";
 import {
   createBackendPlugin,
   coreServices,
@@ -234,11 +237,12 @@ export default createBackendPlugin({
         // computes each system's aggregate from durable `health_check_runs`,
         // and the queue worker (set up just below — the only mutation site)
         // drives writes through `handle.mutate`.
-        healthEntityService = new HealthCheckService(
+        const service = new HealthCheckService(
           database,
           healthCheckRegistry,
           collectorRegistry,
         );
+        healthEntityService = service;
 
         // Register this plugin's AI tools (propose/update/delete) into the AI
         // registry via the extension point - owned here, not in ai-backend.
@@ -259,11 +263,29 @@ export default createBackendPlugin({
           procedureKey: "getConfigurations",
           name: "healthcheck.status",
           description:
-            "List health-check configurations and their current status. Read-only.",
+            "List systems' health checks and their current status, including " +
+            "which are failing/unhealthy. Use this when asked what is down, " +
+            "failing, or unhealthy. Read-only.",
           effect: "read",
           execute: deferredProjectionExecute,
         });
 
+        // Contribute this plugin's per-system health problems to the AI
+        // `system.issues` aggregator. PER-SOURCE access is OUR job: gate on the
+        // principal's `healthcheck.status` grant and return {} (never throw)
+        // when not satisfied. The read derives from the durable
+        // `health_check_runs` / `system_health_checks` tables (global, identical
+        // on every pod) and reuses the SAME pure deriver as the dashboard
+        // filler, so backend signals match the UI's source/tone/label/detail.
+        env
+          .getExtensionPoint(systemSignalsExtensionPoint)
+          .contribute(
+            createHealthcheckSignalsContributor({
+              service,
+              resolver: createSystemAccessResolver(rpcClient),
+            }),
+          );
+
         // Create catalog client for notification delegation
         const catalogClient = rpcClient.forPlugin(CatalogApi);
 

package/src/service.ts

@@ -10,6 +10,7 @@ import {
   NotificationPolicySchema,
   DEFAULT_NOTIFICATION_POLICY,
   type CollectorConfigEntry,
+  type HealthcheckSignalStatuses,
 } from "@checkstack/healthcheck-common";
 import type { ConfigService } from "@checkstack/backend-api";
 import type { InferClient } from "@checkstack/common";
@@ -624,6 +625,36 @@ export class HealthCheckService {
     };
   }
 
+  /**
+   * Global problem scan across EVERY system that has at least one enabled
+   * health-check association. Returns the evaluated status keyed by systemId,
+   * containing ONLY systems that are currently degraded or unhealthy (healthy
+   * systems are omitted). This is the read source for the AI system-signals
+   * contributor: it must answer the same on every pod, so it derives entirely
+   * from the durable `health_check_runs` / `system_health_checks` tables via
+   * the same per-system evaluator the dashboard uses - no per-caller systemId
+   * list and no process-local state.
+   */
+  async getAllUnhealthySystemStatuses(): Promise<HealthcheckSignalStatuses> {
+    // Distinct systemIds that have at least one ENABLED check association.
+    // `getSystemHealthStatus` already short-circuits to healthy for systems
+    // with no enabled associations, so this is the complete candidate set.
+    const rows = await this.db
+      .selectDistinct({ systemId: systemHealthChecks.systemId })
+      .from(systemHealthChecks)
+      .where(eq(systemHealthChecks.enabled, true));
+
+    const result: HealthcheckSignalStatuses = {};
+    await Promise.all(
+      rows.map(async ({ systemId }) => {
+        const status = await this.getSystemHealthStatus(systemId);
+        if (status.status === "healthy") return; // problems only
+        result[systemId] = status;
+      }),
+    );
+    return result;
+  }
+
   /**
    * Live health-state snapshot for a single system (Wave-2 sensing
    * contract). When `configurationId` is given, status reflects that