@checkstack/healthcheck-backend 1.15.0 → 1.17.0

package/src/collector-assertions.ts

@@ -1,8 +1,12 @@
+import { evaluateAssertion } from "@checkstack/backend-api";
+import type {
+  AssertionOutcome,
+  CollectorAssertion,
+} from "@checkstack/healthcheck-common";
 import {
-  evaluateAssertions,
-  evaluateJsonPathAssertions,
-} from "@checkstack/backend-api";
-import type { CollectorAssertion } from "@checkstack/healthcheck-common";
+  computeAssertionKey,
+  truncateActual,
+} from "@checkstack/healthcheck-common";
 import { extractErrorMessage } from "@checkstack/common";
 import { JSONPath } from "jsonpath-plus";
 
@@ -69,9 +73,23 @@ function formatFailure({
   return detail ? `${base} (${detail})` : base;
 }
 
+/** All outcomes of a collector's assertions plus the legacy failure string. */
+export interface CollectorAssertionEvaluation {
+  /** One structured outcome per configured assertion, in config order. */
+  outcomes: AssertionOutcome[];
+  /**
+   * The FIRST failing assertion's message, formatted exactly like the legacy
+   * `_assertionFailed` string. Undefined when everything passed.
+   */
+  firstFailureMessage?: string;
+}
+
 /**
- * Evaluate a collector's assertions - plain field assertions AND JSONPath
- * assertions - against its result, in the order they were configured.
+ * Evaluate ALL of a collector's assertions - plain field assertions AND
+ * JSONPath assertions - against its result, in the order they were
+ * configured, returning a structured outcome per assertion (pass AND fail;
+ * this is what makes assertions analyzable rather than only visible on
+ * failure).
  *
  * Plain assertions compare `result[field]` directly (unchanged behaviour).
  * JSONPath assertions parse the SOURCE field (e.g. `body` for the field
@@ -79,18 +97,15 @@ function formatFailure({
  * apply the operator to the extracted value. Fail-closed: a missing
  * expression, a non-JSON source value, or an invalid/eval-blocked path fails
  * the assertion (with a diagnostic suffix) - it never fails the collector.
- *
- * Returns the failure message of the FIRST failing assertion, or `undefined`
- * when all pass.
  */
-export function evaluateCollectorAssertions({
+export function evaluateCollectorAssertionOutcomes({
   assertions,
   result,
 }: {
   assertions: CollectorAssertion[] | undefined;
   result: Record<string, unknown>;
-}): string | undefined {
-  if (!assertions?.length) return undefined;
+}): CollectorAssertionEvaluation {
+  if (!assertions?.length) return { outcomes: [] };
 
   // Parse each JSON source field at most once, not once per assertion.
   const parsedSources = new Map<string, { json?: unknown; error?: string }>();
@@ -116,19 +131,72 @@ export function evaluateCollectorAssertions({
     return entry;
   };
 
+  const outcomes: AssertionOutcome[] = [];
+  let firstFailureMessage: string | undefined;
+
+  const baseOutcome = (assertion: CollectorAssertion) => ({
+    key: computeAssertionKey({ assertion }),
+    field: assertion.field,
+    jsonPath: assertion.jsonPath?.trim() || undefined,
+    operator: assertion.operator,
+    value:
+      assertion.value === undefined ? undefined : String(assertion.value),
+  });
+
+  const recordFailure = ({
+    assertion,
+    actual,
+    message,
+    legacyMessage,
+  }: {
+    assertion: CollectorAssertion;
+    actual?: unknown;
+    message: string;
+    legacyMessage: string;
+  }) => {
+    outcomes.push({
+      ...baseOutcome(assertion),
+      passed: false,
+      actual: actual === undefined ? undefined : truncateActual({ value: actual }),
+      message,
+    });
+    if (firstFailureMessage === undefined) firstFailureMessage = legacyMessage;
+  };
+
   for (const assertion of assertions) {
     if (!isJsonPathAssertion(assertion)) {
-      const failed = evaluateAssertions([assertion], result);
-      if (failed) return formatFailure({ assertion: failed });
+      const evaluated = evaluateAssertion(assertion, result);
+      if (evaluated.passed) {
+        outcomes.push({
+          ...baseOutcome(assertion),
+          passed: true,
+          actual:
+            evaluated.actual === undefined
+              ? undefined
+              : truncateActual({ value: evaluated.actual }),
+        });
+      } else {
+        recordFailure({
+          assertion,
+          actual: evaluated.actual,
+          message: evaluated.message ?? formatFailure({ assertion }),
+          legacyMessage: formatFailure({ assertion }),
+        });
+      }
       continue;
     }
 
     const path = assertion.jsonPath?.trim();
     if (!path) {
-      return formatFailure({
+      recordFailure({
         assertion,
-        detail: "missing JSONPath expression",
+        message: "missing JSONPath expression",
+        legacyMessage: formatFailure({
+          assertion,
+          detail: "missing JSONPath expression",
+        }),
       });
+      continue;
     }
 
     const sourceField = assertion.field.endsWith(JSONPATH_FIELD_SUFFIX)
@@ -136,34 +204,73 @@ export function evaluateCollectorAssertions({
       : assertion.field;
     const source = parseSource(sourceField);
     if (source.error) {
-      return formatFailure({ assertion, detail: source.error });
+      recordFailure({
+        assertion,
+        message: source.error,
+        legacyMessage: formatFailure({ assertion, detail: source.error }),
+      });
+      continue;
     }
 
     try {
-      const failed = evaluateJsonPathAssertions(
-        [
-          {
-            path,
-            operator: assertion.operator,
-            value:
-              assertion.value === undefined
-                ? undefined
-                : String(assertion.value),
-          },
-        ],
-        source.json,
-        extractJsonPath,
+      // Extract once, then reuse the shared operator engine on a synthetic
+      // one-field record so the outcome carries the observed value.
+      const extracted = extractJsonPath(path, source.json);
+      const evaluated = evaluateAssertion(
+        {
+          field: "__jsonpath__",
+          operator: assertion.operator,
+          value:
+            assertion.value === undefined
+              ? undefined
+              : String(assertion.value),
+        },
+        { __jsonpath__: extracted },
       );
-      if (failed) return formatFailure({ assertion });
+      if (evaluated.passed) {
+        outcomes.push({
+          ...baseOutcome(assertion),
+          passed: true,
+          actual:
+            extracted === undefined
+              ? undefined
+              : truncateActual({ value: extracted }),
+        });
+      } else {
+        recordFailure({
+          assertion,
+          actual: extracted,
+          message: evaluated.message ?? formatFailure({ assertion }),
+          legacyMessage: formatFailure({ assertion }),
+        });
+      }
     } catch (error) {
       // jsonpath-plus rejects malformed paths and (with eval disabled)
       // filter/script expressions by throwing.
-      return formatFailure({
+      const detail = `invalid JSONPath: ${extractErrorMessage(error)}`;
+      recordFailure({
         assertion,
-        detail: `invalid JSONPath: ${extractErrorMessage(error)}`,
+        message: detail,
+        legacyMessage: formatFailure({ assertion, detail }),
       });
     }
   }
 
-  return undefined;
+  return { outcomes, firstFailureMessage };
+}
+
+/**
+ * Legacy single-string view of {@link evaluateCollectorAssertionOutcomes}:
+ * the failure message of the FIRST failing assertion, or `undefined` when all
+ * pass. Kept for callers that only need the `_assertionFailed` string.
+ */
+export function evaluateCollectorAssertions({
+  assertions,
+  result,
+}: {
+  assertions: CollectorAssertion[] | undefined;
+  result: Record<string, unknown>;
+}): string | undefined {
+  return evaluateCollectorAssertionOutcomes({ assertions, result })
+    .firstFailureMessage;
 }

package/src/index.ts

@@ -524,6 +524,35 @@ export default createBackendPlugin({
           logger,
           signalService,
           configSecrets,
+          // Fold active incident health overrides into the user-facing system
+          // health reads (worst-wins). Reuses the incident client already built
+          // above; maps the incident rows to the source-agnostic override shape
+          // the fold expects. Kept out of the shared deriver so SLO/AI/entity
+          // paths stay checks-only (see router.ts).
+          incidentHealthOverrideReader: {
+            getActiveOverrides: async (systemIds) => {
+              const { overrides } =
+                await incidentClient.getActiveHealthOverrides({ systemIds });
+              const mapped: Record<
+                string,
+                {
+                  status: "degraded" | "unhealthy";
+                  source: string;
+                  reason: string;
+                  sourceId?: string;
+                }[]
+              > = {};
+              for (const [systemId, list] of Object.entries(overrides)) {
+                mapped[systemId] = list.map((o) => ({
+                  status: o.status,
+                  source: "incident",
+                  reason: o.incidentTitle,
+                  sourceId: o.incidentId,
+                }));
+              }
+              return mapped;
+            },
+          },
           recomputeSystemRollupHealth: (systemId) =>
             recomputeSystemRollupHealth({
               systemId,

package/src/queue-executor.test.ts

@@ -1459,3 +1459,185 @@ describe("recomputeSystemRollupHealth", () => {
     expect(logger.error).toHaveBeenCalledTimes(1);
   });
 });
+
+describe("executeHealthCheckJob - structured assertion outcomes", () => {
+  it("stores _assertions per collector entry and downgrades on failure", async () => {
+    const mockDb = createMockDb();
+    const mockRegistry = createMockRegistry();
+    const mockLogger = createMockLogger();
+    const mockQueueManager = createMockQueueManager();
+    const mockCatalogClient = createMockCatalogClient();
+    const mockMaintenanceClient = createMockMaintenanceClient();
+    const mockIncidentClient = createMockIncidentClient();
+    const mockSignalService = createMockSignalService();
+
+    (mockCatalogClient.getSystem as any) = mock(async () => ({
+      id: "system-1",
+      name: "web-01",
+    }));
+
+    // One collector entry with two assertions: isTrue passes, equals fails.
+    let selectCallCount = 0;
+    (mockDb.select as any) = mock(() => {
+      selectCallCount++;
+      if (selectCallCount === 2) {
+        return {
+          from: mock(() => ({
+            innerJoin: mock(() => ({
+              where: mock(() =>
+                Promise.resolve([
+                  {
+                    configId: "config-1",
+                    configName: "checkout",
+                    strategyId: "test-strategy",
+                    config: { timeout: 5000 },
+                    collectors: [
+                      {
+                        id: "col-1",
+                        collectorId: "test-collector",
+                        config: {},
+                        assertions: [
+                          { field: "ok", operator: "isTrue" },
+                          { field: "code", operator: "equals", value: 200 },
+                        ],
+                      },
+                    ],
+                    interval: 45,
+                    enabled: true,
+                    paused: false,
+                    includeLocal: true,
+                    satelliteIds: [],
+                  },
+                ]),
+              ),
+            })),
+          })),
+        };
+      }
+      return {
+        from: mock(() => ({
+          innerJoin: mock(() => ({
+            where: mock(() => Promise.resolve([])),
+          })),
+        })),
+      };
+    });
+
+    // Capture every insert's values so we can find the stored run.
+    const insertedValues: Record<string, unknown>[] = [];
+    (mockDb.insert as any) = mock(() => ({
+      values: mock((vals: Record<string, unknown>) => {
+        insertedValues.push(vals);
+        return Object.assign(Promise.resolve(), {
+          onConflictDoUpdate: mock(() => Promise.resolve()),
+          onConflictDoNothing: mock(() => Promise.resolve()),
+          returning: mock(() => Promise.resolve([])),
+        });
+      }),
+    }));
+
+    const mockCollectorRegistry = {
+      register: mock(() => {}),
+      getCollector: mock(() => ({
+        collector: {
+          id: "test-collector",
+          execute: mock(async () => ({ result: { ok: true, code: 404 } })),
+          config: new Versioned({ version: 1, schema: z.object({}) }),
+          result: new Versioned({
+            version: 1,
+            schema: z.object({ ok: z.boolean(), code: z.number() }),
+          }),
+          mergeResult: mock(() => ({})),
+        },
+      })),
+      getCollectors: mock(() => []),
+    };
+
+    const queue =
+      mockQueueManager.getQueue<HealthCheckJobPayload>("health-checks");
+    let capturedHandler:
+      | ((job: { data: HealthCheckJobPayload }) => Promise<void>)
+      | undefined;
+    (queue.consume as any) = mock(
+      async (
+        handler: (job: { data: HealthCheckJobPayload }) => Promise<void>,
+      ) => {
+        capturedHandler = handler;
+      },
+    );
+
+    await setupHealthCheckWorker({
+      db: mockDb as unknown as Parameters<
+        typeof setupHealthCheckWorker
+      >[0]["db"],
+      advisoryLock: mockAdvisoryLock,
+      registry: mockRegistry,
+      collectorRegistry: mockCollectorRegistry as unknown as Parameters<
+        typeof setupHealthCheckWorker
+      >[0]["collectorRegistry"],
+      logger: mockLogger,
+      queueManager: mockQueueManager,
+      signalService: mockSignalService,
+      catalogClient: mockCatalogClient as unknown as Parameters<
+        typeof setupHealthCheckWorker
+      >[0]["catalogClient"],
+      notificationClient: {
+        notifyForSubscription: () => Promise.resolve({ notifiedCount: 0 }),
+      } as unknown as Parameters<
+        typeof setupHealthCheckWorker
+      >[0]["notificationClient"],
+      maintenanceClient: mockMaintenanceClient as unknown as Parameters<
+        typeof setupHealthCheckWorker
+      >[0]["maintenanceClient"],
+      incidentClient: mockIncidentClient as unknown as Parameters<
+        typeof setupHealthCheckWorker
+      >[0]["incidentClient"],
+      getEmitHook: () => undefined,
+      cache: passthroughCache,
+    });
+
+    if (capturedHandler) {
+      // Downstream aggregation touches DB surfaces the lightweight mock
+      // doesn't model; the run insert we assert on happens before that.
+      await capturedHandler({
+        data: { configId: "config-1", systemId: "system-1" },
+      }).catch(() => {});
+    }
+
+    const runInsert = insertedValues.find(
+      (vals) => "status" in vals && "result" in vals,
+    );
+    expect(runInsert).toBeDefined();
+    // The failed `code equals 200` assertion downgrades the run.
+    expect(runInsert?.status).toBe("unhealthy");
+
+    const runResult = runInsert?.result as {
+      metadata: {
+        collectors: Record<
+          string,
+          {
+            _assertionFailed?: string;
+            _assertions?: {
+              field: string;
+              passed: boolean;
+              actual?: string;
+            }[];
+          }
+        >;
+      };
+    };
+    const entry = runResult.metadata.collectors["col-1"];
+    expect(entry._assertionFailed).toBe("code equals 200");
+    expect(entry._assertions?.length).toBe(2);
+    expect(entry._assertions?.[0]).toMatchObject({
+      field: "ok",
+      passed: true,
+      actual: "true",
+    });
+    expect(entry._assertions?.[1]).toMatchObject({
+      field: "code",
+      passed: false,
+      actual: "404",
+    });
+  });
+});

package/src/queue-executor.ts

@@ -60,7 +60,8 @@ import {
   shouldNotifyTransition,
 } from "./notification-policy";
 import { recordStateTransition } from "./state-transitions";
-import { evaluateCollectorAssertions } from "./collector-assertions";
+import { evaluateCollectorAssertionOutcomes } from "./collector-assertions";
+import type { AssertionOutcome } from "@checkstack/healthcheck-common";
 import {
   writeHealthEntity,
   createHealthEntitySerializer,
@@ -947,13 +948,18 @@ async function executeHealthCheckJob(props: {
                 collectorError = collectorResult.error;
               }
 
-              // Evaluate per-collector assertions (plain fields + JSONPath)
+              // Evaluate per-collector assertions (plain fields + JSONPath).
+              // ALL outcomes are stored (pass AND fail) so assertions are
+              // analyzable over time, not only visible on failure.
               let assertionFailed: string | undefined;
+              let assertionOutcomes: AssertionOutcome[] = [];
               if (collectorResult.result) {
-                assertionFailed = evaluateCollectorAssertions({
+                const evaluation = evaluateCollectorAssertionOutcomes({
                   assertions: collectorEntry.assertions,
                   result: collectorResult.result as Record<string, unknown>,
                 });
+                assertionFailed = evaluation.firstFailureMessage;
+                assertionOutcomes = evaluation.outcomes;
                 if (assertionFailed) {
                   logger.debug(
                     `Collector ${storageKey} assertion failed: ${assertionFailed}`,
@@ -977,6 +983,9 @@ async function executeHealthCheckJob(props: {
                   _collectorId: collectorEntry.collectorId,
                   _assertionFailed: assertionFailed,
                   _collectorError: collectorError,
+                  ...(assertionOutcomes.length > 0
+                    ? { _assertions: assertionOutcomes }
+                    : {}),
                   ...strippedResult,
                 },
               };

package/src/realtime-aggregation.test.ts

@@ -5,6 +5,7 @@ import {
   deserializeTDigest,
   incrementHourlyAggregate,
 } from "./realtime-aggregation";
+import { computeAssertionKey } from "@checkstack/healthcheck-common";
 import { TDigest } from "tdigest";
 
 describe("getHourBucketStart", () => {
@@ -494,3 +495,138 @@ describe("incrementHourlyAggregate", () => {
     expect(inserted.maxLatencyMs).toBe(150);
   });
 });
+
+describe("incrementHourlyAggregate - assertion stats folding", () => {
+  const KEY = computeAssertionKey({
+    assertion: { field: "statusCode", operator: "equals", value: 200 },
+  });
+
+  const outcome = (passed: boolean) => ({
+    key: KEY,
+    field: "statusCode",
+    operator: "equals",
+    value: "200",
+    passed,
+  });
+
+  const collectorRegistry = {
+    register: mock(() => {}),
+    getCollector: mock(() => ({
+      collector: {
+        id: "test-collector",
+        mergeResult: mock(() => ({ count: 1 })),
+      },
+    })),
+    getCollectors: mock(() => []),
+  } as unknown as Parameters<
+    typeof incrementHourlyAggregate
+  >[0]["collectorRegistry"];
+
+  function runResult(outcomes: unknown[]) {
+    return {
+      metadata: {
+        collectors: {
+          "uuid-1": {
+            _collectorId: "test-collector",
+            _assertions: outcomes,
+            statusCode: 200,
+          },
+        },
+      },
+    };
+  }
+
+  it("folds a run's outcomes into the bucket's per-assertion counts", async () => {
+    let inserted: Record<string, unknown> | undefined;
+    const db = {
+      select: mock(() => ({
+        from: mock(() => ({
+          where: mock(() => ({
+            limit: mock(() => Promise.resolve([])),
+          })),
+        })),
+      })),
+      insert: mock(() => ({
+        values: mock((values: Record<string, unknown>) => {
+          inserted = values;
+          return { onConflictDoUpdate: mock(() => Promise.resolve()) };
+        }),
+      })),
+    };
+
+    await incrementHourlyAggregate({
+      db: db as never,
+      systemId: "sys-1",
+      configurationId: "config-1",
+      status: "unhealthy",
+      latencyMs: 100,
+      runTimestamp: new Date("2024-01-15T10:35:00Z"),
+      result: runResult([outcome(true), outcome(false)]) as never,
+      collectorRegistry,
+    });
+
+    const aggregated = inserted?.aggregatedResult as Record<string, unknown>;
+    expect(aggregated.assertions).toEqual({
+      "uuid-1": { [KEY]: { passCount: 1, failCount: 1 } },
+    });
+    // The internal _assertions never leak into collector merge output.
+    const collectors = aggregated.collectors as Record<
+      string,
+      Record<string, unknown>
+    >;
+    expect(collectors["uuid-1"]._assertions).toBeUndefined();
+  });
+
+  it("increments existing counts and keeps distinct keys separate (mid-bucket config edit)", async () => {
+    const OTHER_KEY = computeAssertionKey({
+      assertion: { field: "statusCode", operator: "equals", value: 201 },
+    });
+    const existing = {
+      tdigestState: null,
+      minLatencyMs: null,
+      maxLatencyMs: null,
+      aggregatedResult: {
+        collectors: {},
+        assertions: { "uuid-1": { [KEY]: { passCount: 4, failCount: 0 } } },
+      },
+    };
+    let inserted: Record<string, unknown> | undefined;
+    const db = {
+      select: mock(() => ({
+        from: mock(() => ({
+          where: mock(() => ({
+            limit: mock(() => Promise.resolve([existing])),
+          })),
+        })),
+      })),
+      insert: mock(() => ({
+        values: mock((values: Record<string, unknown>) => {
+          inserted = values;
+          return { onConflictDoUpdate: mock(() => Promise.resolve()) };
+        }),
+      })),
+    };
+
+    // The edited assertion (value 201) starts a NEW series in the same bucket.
+    await incrementHourlyAggregate({
+      db: db as never,
+      systemId: "sys-1",
+      configurationId: "config-1",
+      status: "healthy",
+      latencyMs: 100,
+      runTimestamp: new Date("2024-01-15T10:40:00Z"),
+      result: runResult([
+        { ...outcome(true), key: OTHER_KEY, value: "201" },
+      ]) as never,
+      collectorRegistry,
+    });
+
+    const aggregated = inserted?.aggregatedResult as Record<string, unknown>;
+    expect(aggregated.assertions).toEqual({
+      "uuid-1": {
+        [KEY]: { passCount: 4, failCount: 0 },
+        [OTHER_KEY]: { passCount: 1, failCount: 0 },
+      },
+    });
+  });
+});