@checkstack/healthcheck-backend 1.13.0 → 1.14.0

package/CHANGELOG.md

@@ -1,5 +1,50 @@
 # @checkstack/healthcheck-backend
 
+## 1.14.0
+
+### Minor Changes
+
+- e819276: Fix JSONPath collector assertions: the executor previously evaluated every
+  assertion with a flat field lookup, so a `Body (JSONPath)` assertion compared
+  against `undefined` and the configured path was silently ignored (`Exists`
+  always failed, `Not Exists` always passed). The executor now parses the source
+  field as JSON and extracts the configured path via `jsonpath-plus` (with
+  expression evaluation disabled - filter/script expressions are rejected).
+  Fail-closed: a non-JSON body, missing expression, or invalid path fails the
+  assertion with a diagnostic, never the collection.
+
+  Also adds `isEmpty` / `isNotEmpty` to the JSONPath operator set (and the
+  AssertionBuilder), treating `[]`, `{}`, `""`, and missing values as empty - so
+  "no errors reported" is a single `$.errors Is Empty` assertion, and "key exists
+  but is empty" is `Exists` + `Is Empty` on the same path.
+
+### Patch Changes
+
+- Updated dependencies [e819276]
+- Updated dependencies [e819276]
+  - @checkstack/ai-backend@0.10.5
+  - @checkstack/backend-api@0.28.0
+  - @checkstack/automation-backend@0.10.7
+  - @checkstack/catalog-backend@1.6.5
+  - @checkstack/incident-backend@1.9.3
+  - @checkstack/satellite-backend@0.7.8
+  - @checkstack/command-backend@0.2.17
+  - @checkstack/gitops-backend@0.5.17
+  - @checkstack/script-packages-backend@0.3.21
+  - @checkstack/secrets-backend@0.2.17
+  - @checkstack/status-page-backend@0.4.4
+
+## 1.13.1
+
+### Patch Changes
+
+- Updated dependencies [b4e0832]
+  - @checkstack/ai-backend@0.10.4
+  - @checkstack/automation-backend@0.10.6
+  - @checkstack/catalog-backend@1.6.4
+  - @checkstack/incident-backend@1.9.2
+  - @checkstack/satellite-backend@0.7.7
+
 ## 1.13.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/healthcheck-backend",
-  "version": "1.13.0",
+  "version": "1.14.0",
   "license": "Elastic-2.0",
   "type": "module",
   "main": "src/index.ts",
@@ -14,45 +14,46 @@
     "lint:code": "eslint . --max-warnings 0"
   },
   "dependencies": {
-    "@checkstack/backend-api": "0.27.1",
-    "@checkstack/ai-backend": "0.10.3",
+    "@checkstack/ai-backend": "0.10.5",
     "@checkstack/ai-common": "0.6.3",
-    "@checkstack/script-packages-backend": "0.3.20",
+    "@checkstack/automation-backend": "0.10.7",
+    "@checkstack/backend-api": "0.28.0",
     "@checkstack/cache-api": "0.3.16",
     "@checkstack/cache-utils": "0.2.21",
-    "@checkstack/catalog-backend": "1.6.3",
+    "@checkstack/catalog-backend": "1.6.5",
     "@checkstack/catalog-common": "2.6.0",
-    "@checkstack/command-backend": "0.2.16",
+    "@checkstack/command-backend": "0.2.17",
     "@checkstack/common": "0.19.0",
-    "@checkstack/gitops-backend": "0.5.16",
+    "@checkstack/gitops-backend": "0.5.17",
     "@checkstack/gitops-common": "0.7.0",
     "@checkstack/healthcheck-common": "1.11.0",
-    "@checkstack/secrets-common": "0.2.8",
-    "@checkstack/secrets-backend": "0.2.16",
-    "@checkstack/incident-backend": "1.9.1",
+    "@checkstack/incident-backend": "1.9.3",
     "@checkstack/incident-common": "1.7.0",
-    "@checkstack/automation-backend": "0.10.5",
     "@checkstack/maintenance-common": "1.8.0",
     "@checkstack/notification-common": "1.5.0",
     "@checkstack/queue-api": "0.3.16",
-    "@checkstack/satellite-backend": "0.7.6",
-    "@checkstack/sdk": "0.119.1",
+    "@checkstack/satellite-backend": "0.7.8",
+    "@checkstack/script-packages-backend": "0.3.21",
+    "@checkstack/sdk": "0.121.0",
+    "@checkstack/secrets-backend": "0.2.17",
+    "@checkstack/secrets-common": "0.2.8",
     "@checkstack/signal-common": "0.2.14",
-    "@checkstack/status-page-backend": "0.4.3",
+    "@checkstack/status-page-backend": "0.4.4",
     "@checkstack/status-page-common": "0.5.0",
     "@hono/zod-validator": "^0.7.6",
+    "@orpc/contract": "^1.14.4",
+    "@orpc/server": "^1.14.4",
     "drizzle-orm": "^0.45.0",
     "hono": "^4.12.25",
+    "jsonpath-plus": "^10.4.0",
     "tdigest": "^0.1.2",
     "yaml": "^2.6.1",
-    "zod": "^4.2.1",
-    "@orpc/contract": "^1.14.4",
-    "@orpc/server": "^1.14.4"
+    "zod": "^4.2.1"
   },
   "devDependencies": {
     "@checkstack/drizzle-helper": "0.0.6",
     "@checkstack/scripts": "0.7.0",
-    "@checkstack/test-utils-backend": "0.1.50",
+    "@checkstack/test-utils-backend": "0.1.51",
     "@checkstack/tsconfig": "0.0.7",
     "@types/bun": "^1.0.0",
     "@types/tdigest": "^0.1.5",

package/src/collector-assertions.test.ts

@@ -0,0 +1,276 @@
+import { describe, expect, it } from "bun:test";
+import type { CollectorAssertion } from "@checkstack/healthcheck-common";
+import { evaluateCollectorAssertions } from "./collector-assertions";
+
+/** The HTTP Request collector's result shape, as the executor sees it. */
+const httpResult = (body: string): Record<string, unknown> => ({
+  statusCode: 200,
+  statusText: "OK",
+  responseTimeMs: 12,
+  body,
+  bodyLength: body.length,
+  success: true,
+});
+
+const jsonPathAssertion = (
+  jsonPath: string,
+  operator: string,
+  value?: unknown,
+): CollectorAssertion => ({
+  field: "body.$",
+  jsonPath,
+  operator,
+  ...(value === undefined ? {} : { value }),
+});
+
+describe("evaluateCollectorAssertions", () => {
+  describe("plain field assertions (regression)", () => {
+    it("passes and fails on a direct result field", () => {
+      const result = httpResult("{}");
+
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [{ field: "statusCode", operator: "equals", value: 200 }],
+          result,
+        }),
+      ).toBeUndefined();
+
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [{ field: "statusCode", operator: "equals", value: 404 }],
+          result,
+        }),
+      ).toBe("statusCode equals 404");
+    });
+
+    it("returns undefined for no assertions", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: undefined,
+          result: httpResult(""),
+        }),
+      ).toBeUndefined();
+      expect(
+        evaluateCollectorAssertions({ assertions: [], result: httpResult("") }),
+      ).toBeUndefined();
+    });
+
+    it("returns the FIRST failing assertion in configured order", () => {
+      const failure = evaluateCollectorAssertions({
+        assertions: [
+          { field: "success", operator: "isFalse" },
+          { field: "statusCode", operator: "equals", value: 404 },
+        ],
+        result: httpResult("{}"),
+      });
+      expect(failure).toBe("success isFalse");
+    });
+  });
+
+  describe("JSONPath assertions: extraction actually happens", () => {
+    it("exists fails when the key is missing and passes when present", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.error", "exists")],
+          result: httpResult(`{"status":"ok"}`),
+        }),
+      ).toBe("body.$ $.error exists");
+
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.error", "exists")],
+          result: httpResult(`{"error":""}`),
+        }),
+      ).toBeUndefined();
+    });
+
+    it("notExists passes for a missing key and fails for a present one", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.error", "notExists")],
+          result: httpResult(`{"status":"ok"}`),
+        }),
+      ).toBeUndefined();
+
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.error", "notExists")],
+          result: httpResult(`{"error":"boom"}`),
+        }),
+      ).toBe("body.$ $.error notExists");
+    });
+
+    it("equals compares the EXTRACTED value, not the whole body", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.data.id", "equals", "42")],
+          result: httpResult(`{"data":{"id":42},"noise":"x"}`),
+        }),
+      ).toBeUndefined();
+
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.data.id", "equals", "43")],
+          result: httpResult(`{"data":{"id":42}}`),
+        }),
+      ).toBe("body.$ $.data.id equals 43");
+    });
+
+    it('asserts "error exists but is empty" via exists + isEmpty', () => {
+      const assertions = [
+        jsonPathAssertion("$.error", "exists"),
+        jsonPathAssertion("$.error", "isEmpty"),
+      ];
+
+      // Present and empty string: both pass.
+      expect(
+        evaluateCollectorAssertions({
+          assertions,
+          result: httpResult(`{"error":""}`),
+        }),
+      ).toBeUndefined();
+
+      // Missing entirely: exists fails (isEmpty alone would have passed).
+      expect(
+        evaluateCollectorAssertions({
+          assertions,
+          result: httpResult(`{"status":"ok"}`),
+        }),
+      ).toBe("body.$ $.error exists");
+
+      // Present and non-empty: isEmpty fails.
+      expect(
+        evaluateCollectorAssertions({
+          assertions,
+          result: httpResult(`{"error":"boom"}`),
+        }),
+      ).toBe("body.$ $.error isEmpty");
+    });
+  });
+
+  describe("empty arrays in JSON responses", () => {
+    it("isEmpty passes for [] and fails for a populated array", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.errors", "isEmpty")],
+          result: httpResult(`{"errors":[]}`),
+        }),
+      ).toBeUndefined();
+
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.errors", "isEmpty")],
+          result: httpResult(`{"errors":["boom"]}`),
+        }),
+      ).toBe("body.$ $.errors isEmpty");
+    });
+
+    it("isNotEmpty passes for a populated array and fails for []", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.items", "isNotEmpty")],
+          result: httpResult(`{"items":[1]}`),
+        }),
+      ).toBeUndefined();
+
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.items", "isNotEmpty")],
+          result: httpResult(`{"items":[]}`),
+        }),
+      ).toBe("body.$ $.items isNotEmpty");
+    });
+
+    it("isEmpty handles empty objects and nested paths", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.meta.warnings", "isEmpty")],
+          result: httpResult(`{"meta":{"warnings":{}}}`),
+        }),
+      ).toBeUndefined();
+
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.meta.warnings", "isEmpty")],
+          result: httpResult(`{"meta":{"warnings":{"w1":"x"}}}`),
+        }),
+      ).toBe("body.$ $.meta.warnings isEmpty");
+    });
+
+    it("array length is assertable via the .length property", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [
+            jsonPathAssertion("$.items.length", "greaterThan", "2"),
+          ],
+          result: httpResult(`{"items":[1,2,3]}`),
+        }),
+      ).toBeUndefined();
+
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [
+            jsonPathAssertion("$.items.length", "greaterThan", "5"),
+          ],
+          result: httpResult(`{"items":[1,2,3]}`),
+        }),
+      ).toBe("body.$ $.items.length greaterThan 5");
+    });
+  });
+
+  describe("fail-closed diagnostics", () => {
+    it("fails with a diagnostic when the body is not valid JSON", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.error", "notExists")],
+          result: httpResult("<html>not json</html>"),
+        }),
+      ).toBe('body.$ $.error notExists (field "body" is not valid JSON)');
+    });
+
+    it("fails with a diagnostic when the source field is absent", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [jsonPathAssertion("$.error", "exists")],
+          result: { statusCode: 200 },
+        }),
+      ).toBe('body.$ $.error exists (field "body" has no value)');
+    });
+
+    it("fails with a diagnostic when the JSONPath expression is missing", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [{ field: "body.$", jsonPath: "  ", operator: "exists" }],
+          result: httpResult("{}"),
+        }),
+      ).toBe("body.$ exists (missing JSONPath expression)");
+    });
+
+    it("rejects filter/script expressions instead of evaluating them", () => {
+      const failure = evaluateCollectorAssertions({
+        assertions: [
+          jsonPathAssertion(`$.items[?(@.x==1)]`, "exists"),
+        ],
+        result: httpResult(`{"items":[{"x":1}]}`),
+      });
+      expect(failure).toContain("invalid JSONPath");
+    });
+  });
+
+  describe("structured (non-string) source fields", () => {
+    it("evaluates JSONPath against an already-parsed object field", () => {
+      expect(
+        evaluateCollectorAssertions({
+          assertions: [
+            {
+              field: "details.$",
+              jsonPath: "$.nodes",
+              operator: "isEmpty",
+            },
+          ],
+          result: { details: { nodes: [] } },
+        }),
+      ).toBeUndefined();
+    });
+  });
+});

package/src/collector-assertions.ts

@@ -0,0 +1,169 @@
+import {
+  evaluateAssertions,
+  evaluateJsonPathAssertions,
+} from "@checkstack/backend-api";
+import type { CollectorAssertion } from "@checkstack/healthcheck-common";
+import { extractErrorMessage } from "@checkstack/common";
+import { JSONPath } from "jsonpath-plus";
+
+/**
+ * Suffix the AssertionBuilder appends to a JSONPath field's path: an
+ * `x-jsonpath` result field `body` is offered as the assertable field
+ * `body.$`, with the actual expression stored in `assertion.jsonPath`.
+ */
+const JSONPATH_FIELD_SUFFIX = ".$";
+
+/** Whether an assertion targets a JSONPath into a field, not the field itself. */
+function isJsonPathAssertion(assertion: CollectorAssertion): boolean {
+  return (
+    assertion.field.endsWith(JSONPATH_FIELD_SUFFIX) ||
+    (typeof assertion.jsonPath === "string" && assertion.jsonPath.trim() !== "")
+  );
+}
+
+/**
+ * Extract a JSONPath from a parsed JSON value. `wrap: false` returns the
+ * single matched value directly (or `undefined` for no match), so `exists` /
+ * `isEmpty` operate on the value itself, not on a match array. `eval: false`
+ * rejects script/filter expressions outright - assertion paths are authored
+ * by users and must never evaluate code on the core.
+ */
+/**
+ * Narrow an `unknown` to the input type `JSONPath` accepts. Parsed JSON is
+ * always one of these by construction; the guard exists because the shared
+ * `evaluateJsonPathAssertions` signature hands the json through as `unknown`.
+ */
+function isJsonPathInput(
+  value: unknown,
+): value is string | number | boolean | object | null {
+  return (
+    value === null ||
+    typeof value === "string" ||
+    typeof value === "number" ||
+    typeof value === "boolean" ||
+    typeof value === "object"
+  );
+}
+
+function extractJsonPath(path: string, json: unknown): unknown {
+  if (!isJsonPathInput(json)) return undefined;
+  return JSONPath({ path, json, wrap: false, eval: false });
+}
+
+/** Human-readable failure string, stored on the run as `_assertionFailed`. */
+function formatFailure({
+  assertion,
+  detail,
+}: {
+  assertion: CollectorAssertion;
+  detail?: string;
+}): string {
+  const path = assertion.jsonPath?.trim();
+  const parts = [
+    assertion.field,
+    ...(isJsonPathAssertion(assertion) && path ? [path] : []),
+    assertion.operator,
+    ...(assertion.value === undefined ? [] : [String(assertion.value)]),
+  ];
+  const base = parts.join(" ");
+  return detail ? `${base} (${detail})` : base;
+}
+
+/**
+ * Evaluate a collector's assertions - plain field assertions AND JSONPath
+ * assertions - against its result, in the order they were configured.
+ *
+ * Plain assertions compare `result[field]` directly (unchanged behaviour).
+ * JSONPath assertions parse the SOURCE field (e.g. `body` for the field
+ * `body.$`) as JSON when it is a string, extract `assertion.jsonPath`, and
+ * apply the operator to the extracted value. Fail-closed: a missing
+ * expression, a non-JSON source value, or an invalid/eval-blocked path fails
+ * the assertion (with a diagnostic suffix) - it never fails the collector.
+ *
+ * Returns the failure message of the FIRST failing assertion, or `undefined`
+ * when all pass.
+ */
+export function evaluateCollectorAssertions({
+  assertions,
+  result,
+}: {
+  assertions: CollectorAssertion[] | undefined;
+  result: Record<string, unknown>;
+}): string | undefined {
+  if (!assertions?.length) return undefined;
+
+  // Parse each JSON source field at most once, not once per assertion.
+  const parsedSources = new Map<string, { json?: unknown; error?: string }>();
+  const parseSource = (sourceField: string) => {
+    const cached = parsedSources.get(sourceField);
+    if (cached) return cached;
+
+    const raw = result[sourceField];
+    let entry: { json?: unknown; error?: string };
+    if (raw === undefined || raw === null) {
+      entry = { error: `field "${sourceField}" has no value` };
+    } else if (typeof raw === "string") {
+      try {
+        entry = { json: JSON.parse(raw) };
+      } catch {
+        entry = { error: `field "${sourceField}" is not valid JSON` };
+      }
+    } else {
+      // Already-structured value (object/array/number) - use as-is.
+      entry = { json: raw };
+    }
+    parsedSources.set(sourceField, entry);
+    return entry;
+  };
+
+  for (const assertion of assertions) {
+    if (!isJsonPathAssertion(assertion)) {
+      const failed = evaluateAssertions([assertion], result);
+      if (failed) return formatFailure({ assertion: failed });
+      continue;
+    }
+
+    const path = assertion.jsonPath?.trim();
+    if (!path) {
+      return formatFailure({
+        assertion,
+        detail: "missing JSONPath expression",
+      });
+    }
+
+    const sourceField = assertion.field.endsWith(JSONPATH_FIELD_SUFFIX)
+      ? assertion.field.slice(0, -JSONPATH_FIELD_SUFFIX.length)
+      : assertion.field;
+    const source = parseSource(sourceField);
+    if (source.error) {
+      return formatFailure({ assertion, detail: source.error });
+    }
+
+    try {
+      const failed = evaluateJsonPathAssertions(
+        [
+          {
+            path,
+            operator: assertion.operator,
+            value:
+              assertion.value === undefined
+                ? undefined
+                : String(assertion.value),
+          },
+        ],
+        source.json,
+        extractJsonPath,
+      );
+      if (failed) return formatFailure({ assertion });
+    } catch (error) {
+      // jsonpath-plus rejects malformed paths and (with eval disabled)
+      // filter/script expressions by throwing.
+      return formatFailure({
+        assertion,
+        detail: `invalid JSONPath: ${extractErrorMessage(error)}`,
+      });
+    }
+  }
+
+  return undefined;
+}

package/src/queue-executor.ts

@@ -3,7 +3,6 @@ import {
   Logger,
   type EmitHookFn,
   type CollectorRegistry,
-  evaluateAssertions,
   type SafeDatabase,
   type BaseStrategyConfig,
   type ConnectedClient,
@@ -57,6 +56,7 @@ import {
   shouldNotifyTransition,
 } from "./notification-policy";
 import { recordStateTransition } from "./state-transitions";
+import { evaluateCollectorAssertions } from "./collector-assertions";
 import {
   writeHealthEntity,
   createHealthEntitySerializer,
@@ -901,21 +901,14 @@ async function executeHealthCheckJob(props: {
                 collectorError = collectorResult.error;
               }
 
-              // Evaluate per-collector assertions
+              // Evaluate per-collector assertions (plain fields + JSONPath)
               let assertionFailed: string | undefined;
-              if (
-                collectorEntry.assertions &&
-                collectorEntry.assertions.length > 0 &&
-                collectorResult.result
-              ) {
-                const failedAssertion = evaluateAssertions(
-                  collectorEntry.assertions,
-                  collectorResult.result as Record<string, unknown>,
-                );
-                if (failedAssertion) {
-                  assertionFailed = `${failedAssertion.field} ${
-                    failedAssertion.operator
-                  } ${failedAssertion.value ?? ""}`;
+              if (collectorResult.result) {
+                assertionFailed = evaluateCollectorAssertions({
+                  assertions: collectorEntry.assertions,
+                  result: collectorResult.result as Record<string, unknown>,
+                });
+                if (assertionFailed) {
                   logger.debug(
                     `Collector ${storageKey} assertion failed: ${assertionFailed}`,
                   );