@checkstack/healthcheck-backend 1.14.0 → 1.16.0

package/src/collector-assertions.test.ts

@@ -1,6 +1,13 @@
 import { describe, expect, it } from "bun:test";
 import type { CollectorAssertion } from "@checkstack/healthcheck-common";
-import { evaluateCollectorAssertions } from "./collector-assertions";
+import {
+  ASSERTION_ACTUAL_MAX_LENGTH,
+  computeAssertionKey,
+} from "@checkstack/healthcheck-common";
+import {
+  evaluateCollectorAssertionOutcomes,
+  evaluateCollectorAssertions,
+} from "./collector-assertions";
 
 /** The HTTP Request collector's result shape, as the executor sees it. */
 const httpResult = (body: string): Record<string, unknown> => ({
@@ -274,3 +281,92 @@ describe("evaluateCollectorAssertions", () => {
     });
   });
 });
+
+describe("evaluateCollectorAssertionOutcomes", () => {
+  it("evaluates ALL assertions instead of short-circuiting on failure", () => {
+    const result = httpResult('{"status":"degraded"}');
+    const { outcomes, firstFailureMessage } =
+      evaluateCollectorAssertionOutcomes({
+        assertions: [
+          { field: "statusCode", operator: "equals", value: 500 }, // fails
+          { field: "success", operator: "isTrue" }, // passes
+          jsonPathAssertion("$.status", "equals", "ok"), // fails
+        ],
+        result,
+      });
+    expect(outcomes.length).toBe(3);
+    expect(outcomes.map((o) => o.passed)).toEqual([false, true, false]);
+    // The legacy string still reports the FIRST failure only.
+    expect(firstFailureMessage).toBe("statusCode equals 500");
+  });
+
+  it("captures observed values for plain and JSONPath assertions", () => {
+    const result = httpResult('{"status":"ok","items":[1,2]}');
+    const { outcomes } = evaluateCollectorAssertionOutcomes({
+      assertions: [
+        { field: "statusCode", operator: "equals", value: 200 },
+        jsonPathAssertion("$.status", "equals", "ok"),
+        jsonPathAssertion("$.items", "lengthEquals", 2),
+      ],
+      result,
+    });
+    expect(outcomes[0].actual).toBe("200");
+    expect(outcomes[1].actual).toBe("ok");
+    expect(outcomes[2].actual).toBe("[1,2]");
+    expect(outcomes.every((o) => o.passed)).toBe(true);
+  });
+
+  it("truncates long observed values", () => {
+    const longBody = JSON.stringify({ blob: "x".repeat(5000) });
+    const { outcomes } = evaluateCollectorAssertionOutcomes({
+      assertions: [
+        { field: "body", operator: "contains", value: "x" },
+      ],
+      result: httpResult(longBody),
+    });
+    expect(outcomes[0].passed).toBe(true);
+    expect(outcomes[0].actual?.length).toBe(ASSERTION_ACTUAL_MAX_LENGTH);
+  });
+
+  it("fail-closed diagnostics become failed outcomes with messages", () => {
+    const { outcomes, firstFailureMessage } =
+      evaluateCollectorAssertionOutcomes({
+        assertions: [
+          { field: "body.$", jsonPath: "  ", operator: "exists" },
+          jsonPathAssertion("$.status", "equals", "ok"),
+        ],
+        result: httpResult("not json"),
+      });
+    expect(outcomes[0].passed).toBe(false);
+    expect(outcomes[0].message).toBe("missing JSONPath expression");
+    expect(outcomes[1].passed).toBe(false);
+    expect(outcomes[1].message).toBe('field "body" is not valid JSON');
+    expect(firstFailureMessage).toBe(
+      "body.$ exists (missing JSONPath expression)",
+    );
+  });
+
+  it("outcomes carry the canonical assertion identity key", () => {
+    const assertion: CollectorAssertion = {
+      field: "statusCode",
+      operator: "equals",
+      value: 200,
+    };
+    const { outcomes } = evaluateCollectorAssertionOutcomes({
+      assertions: [assertion],
+      result: httpResult("{}"),
+    });
+    expect(outcomes[0].key).toBe(computeAssertionKey({ assertion }));
+    expect(outcomes[0].value).toBe("200");
+  });
+
+  it("no assertions yields an empty evaluation", () => {
+    const { outcomes, firstFailureMessage } =
+      evaluateCollectorAssertionOutcomes({
+        assertions: undefined,
+        result: httpResult("{}"),
+      });
+    expect(outcomes).toEqual([]);
+    expect(firstFailureMessage).toBeUndefined();
+  });
+});

package/src/collector-assertions.ts

@@ -1,8 +1,12 @@
+import { evaluateAssertion } from "@checkstack/backend-api";
+import type {
+  AssertionOutcome,
+  CollectorAssertion,
+} from "@checkstack/healthcheck-common";
 import {
-  evaluateAssertions,
-  evaluateJsonPathAssertions,
-} from "@checkstack/backend-api";
-import type { CollectorAssertion } from "@checkstack/healthcheck-common";
+  computeAssertionKey,
+  truncateActual,
+} from "@checkstack/healthcheck-common";
 import { extractErrorMessage } from "@checkstack/common";
 import { JSONPath } from "jsonpath-plus";
 
@@ -69,9 +73,23 @@ function formatFailure({
   return detail ? `${base} (${detail})` : base;
 }
 
+/** All outcomes of a collector's assertions plus the legacy failure string. */
+export interface CollectorAssertionEvaluation {
+  /** One structured outcome per configured assertion, in config order. */
+  outcomes: AssertionOutcome[];
+  /**
+   * The FIRST failing assertion's message, formatted exactly like the legacy
+   * `_assertionFailed` string. Undefined when everything passed.
+   */
+  firstFailureMessage?: string;
+}
+
 /**
- * Evaluate a collector's assertions - plain field assertions AND JSONPath
- * assertions - against its result, in the order they were configured.
+ * Evaluate ALL of a collector's assertions - plain field assertions AND
+ * JSONPath assertions - against its result, in the order they were
+ * configured, returning a structured outcome per assertion (pass AND fail;
+ * this is what makes assertions analyzable rather than only visible on
+ * failure).
  *
  * Plain assertions compare `result[field]` directly (unchanged behaviour).
  * JSONPath assertions parse the SOURCE field (e.g. `body` for the field
@@ -79,18 +97,15 @@ function formatFailure({
  * apply the operator to the extracted value. Fail-closed: a missing
  * expression, a non-JSON source value, or an invalid/eval-blocked path fails
  * the assertion (with a diagnostic suffix) - it never fails the collector.
- *
- * Returns the failure message of the FIRST failing assertion, or `undefined`
- * when all pass.
  */
-export function evaluateCollectorAssertions({
+export function evaluateCollectorAssertionOutcomes({
   assertions,
   result,
 }: {
   assertions: CollectorAssertion[] | undefined;
   result: Record<string, unknown>;
-}): string | undefined {
-  if (!assertions?.length) return undefined;
+}): CollectorAssertionEvaluation {
+  if (!assertions?.length) return { outcomes: [] };
 
   // Parse each JSON source field at most once, not once per assertion.
   const parsedSources = new Map<string, { json?: unknown; error?: string }>();
@@ -116,19 +131,72 @@ export function evaluateCollectorAssertions({
     return entry;
   };
 
+  const outcomes: AssertionOutcome[] = [];
+  let firstFailureMessage: string | undefined;
+
+  const baseOutcome = (assertion: CollectorAssertion) => ({
+    key: computeAssertionKey({ assertion }),
+    field: assertion.field,
+    jsonPath: assertion.jsonPath?.trim() || undefined,
+    operator: assertion.operator,
+    value:
+      assertion.value === undefined ? undefined : String(assertion.value),
+  });
+
+  const recordFailure = ({
+    assertion,
+    actual,
+    message,
+    legacyMessage,
+  }: {
+    assertion: CollectorAssertion;
+    actual?: unknown;
+    message: string;
+    legacyMessage: string;
+  }) => {
+    outcomes.push({
+      ...baseOutcome(assertion),
+      passed: false,
+      actual: actual === undefined ? undefined : truncateActual({ value: actual }),
+      message,
+    });
+    if (firstFailureMessage === undefined) firstFailureMessage = legacyMessage;
+  };
+
   for (const assertion of assertions) {
     if (!isJsonPathAssertion(assertion)) {
-      const failed = evaluateAssertions([assertion], result);
-      if (failed) return formatFailure({ assertion: failed });
+      const evaluated = evaluateAssertion(assertion, result);
+      if (evaluated.passed) {
+        outcomes.push({
+          ...baseOutcome(assertion),
+          passed: true,
+          actual:
+            evaluated.actual === undefined
+              ? undefined
+              : truncateActual({ value: evaluated.actual }),
+        });
+      } else {
+        recordFailure({
+          assertion,
+          actual: evaluated.actual,
+          message: evaluated.message ?? formatFailure({ assertion }),
+          legacyMessage: formatFailure({ assertion }),
+        });
+      }
       continue;
     }
 
     const path = assertion.jsonPath?.trim();
     if (!path) {
-      return formatFailure({
+      recordFailure({
         assertion,
-        detail: "missing JSONPath expression",
+        message: "missing JSONPath expression",
+        legacyMessage: formatFailure({
+          assertion,
+          detail: "missing JSONPath expression",
+        }),
       });
+      continue;
     }
 
     const sourceField = assertion.field.endsWith(JSONPATH_FIELD_SUFFIX)
@@ -136,34 +204,73 @@ export function evaluateCollectorAssertions({
       : assertion.field;
     const source = parseSource(sourceField);
     if (source.error) {
-      return formatFailure({ assertion, detail: source.error });
+      recordFailure({
+        assertion,
+        message: source.error,
+        legacyMessage: formatFailure({ assertion, detail: source.error }),
+      });
+      continue;
     }
 
     try {
-      const failed = evaluateJsonPathAssertions(
-        [
-          {
-            path,
-            operator: assertion.operator,
-            value:
-              assertion.value === undefined
-                ? undefined
-                : String(assertion.value),
-          },
-        ],
-        source.json,
-        extractJsonPath,
+      // Extract once, then reuse the shared operator engine on a synthetic
+      // one-field record so the outcome carries the observed value.
+      const extracted = extractJsonPath(path, source.json);
+      const evaluated = evaluateAssertion(
+        {
+          field: "__jsonpath__",
+          operator: assertion.operator,
+          value:
+            assertion.value === undefined
+              ? undefined
+              : String(assertion.value),
+        },
+        { __jsonpath__: extracted },
       );
-      if (failed) return formatFailure({ assertion });
+      if (evaluated.passed) {
+        outcomes.push({
+          ...baseOutcome(assertion),
+          passed: true,
+          actual:
+            extracted === undefined
+              ? undefined
+              : truncateActual({ value: extracted }),
+        });
+      } else {
+        recordFailure({
+          assertion,
+          actual: extracted,
+          message: evaluated.message ?? formatFailure({ assertion }),
+          legacyMessage: formatFailure({ assertion }),
+        });
+      }
     } catch (error) {
       // jsonpath-plus rejects malformed paths and (with eval disabled)
       // filter/script expressions by throwing.
-      return formatFailure({
+      const detail = `invalid JSONPath: ${extractErrorMessage(error)}`;
+      recordFailure({
         assertion,
-        detail: `invalid JSONPath: ${extractErrorMessage(error)}`,
+        message: detail,
+        legacyMessage: formatFailure({ assertion, detail }),
       });
     }
   }
 
-  return undefined;
+  return { outcomes, firstFailureMessage };
+}
+
+/**
+ * Legacy single-string view of {@link evaluateCollectorAssertionOutcomes}:
+ * the failure message of the FIRST failing assertion, or `undefined` when all
+ * pass. Kept for callers that only need the `_assertionFailed` string.
+ */
+export function evaluateCollectorAssertions({
+  assertions,
+  result,
+}: {
+  assertions: CollectorAssertion[] | undefined;
+  result: Record<string, unknown>;
+}): string | undefined {
+  return evaluateCollectorAssertionOutcomes({ assertions, result })
+    .firstFailureMessage;
 }

package/src/config-secrets-backfill.test.ts

@@ -0,0 +1,142 @@
+import { describe, it, expect, mock } from "bun:test";
+import { z } from "zod";
+import { configSecret, Versioned } from "@checkstack/backend-api";
+import type { InternalSecretsService } from "@checkstack/secrets-backend";
+import { internalSecretName } from "@checkstack/secrets-common";
+import { backfillConfigSecrets } from "./config-secrets-backfill";
+import {
+  healthcheckConfigLockKey,
+  isHealthcheckSecretMarker,
+} from "./config-secrets";
+
+const collectorSchema = z.object({
+  url: z.string(),
+  password: configSecret({ id: "password" }).optional(),
+});
+
+function fakeInternalSecrets(): InternalSecretsService & {
+  store: Map<string, string>;
+} {
+  const store = new Map<string, string>();
+  return {
+    store,
+    async set({ parts, value }) {
+      store.set(internalSecretName(...parts), value);
+    },
+    async get({ parts }) {
+      return store.get(internalSecretName(...parts));
+    },
+    async delete({ parts }) {
+      store.delete(internalSecretName(...parts));
+    },
+  };
+}
+
+const noopLogger = {
+  debug: mock(() => {}),
+  info: mock(() => {}),
+  warn: mock(() => {}),
+  error: mock(() => {}),
+} as never;
+
+describe("backfillConfigSecrets", () => {
+  it("migrates a registered collector's inline secret even under an UNREGISTERED strategy, per config lock", async () => {
+    // One legacy row: strategy plugin is gone, but the collector plugin is
+    // present and its config carries an inline plaintext secret.
+    let row: Record<string, unknown> = {
+      id: "cfg-bf",
+      strategyId: "gone.strategy",
+      config: { url: "https://x", password: "strategy-plain" }, // no schema -> left as-is
+      collectors: [
+        {
+          id: "c1",
+          collectorId: "reg.collector",
+          config: { url: "https://y", password: "collector-plain" },
+        },
+      ],
+    };
+    const updates: Record<string, unknown>[] = [];
+    const db = {
+      // Initial scan: `db.select().from(table)` awaited directly.
+      // Per-row re-read: `db.select().from(table).where(eq(...))`.
+      select: () => ({
+        from: () =>
+          Object.assign(Promise.resolve([row]), {
+            where: () => Promise.resolve([row]),
+          }),
+      }),
+      update: () => ({
+        set: (set: Record<string, unknown>) => {
+          updates.push(set);
+          row = { ...row, ...set };
+          return { where: () => Promise.resolve(undefined) };
+        },
+      }),
+    };
+
+    const registry = { getStrategy: mock(() => undefined) };
+    const collectorRegistry = {
+      getCollector: mock((id: string) =>
+        id === "reg.collector"
+          ? { collector: { config: new Versioned({ version: 1, schema: collectorSchema }) } }
+          : undefined,
+      ),
+    };
+
+    const lockKeys: string[] = [];
+    const advisoryLock = {
+      tryAcquire: mock(async () => ({ release: mock(async () => {}) })),
+      withXactLock: async ({ key, fn }: { key: string; fn: () => Promise<unknown> }) => {
+        lockKeys.push(key);
+        return fn();
+      },
+    };
+
+    const internalSecrets = fakeInternalSecrets();
+
+    await backfillConfigSecrets({
+      db: db as never,
+      registry: registry as never,
+      collectorRegistry: collectorRegistry as never,
+      internalSecrets,
+      advisoryLock: advisoryLock as never,
+      logger: noopLogger,
+    });
+
+    // Per-config lock was taken for the row.
+    expect(lockKeys).toEqual([healthcheckConfigLockKey("cfg-bf")]);
+
+    // The collector's inline secret was migrated to a marker + the store.
+    const persistedCollectors = updates.at(-1)?.collectors as Array<{
+      config: Record<string, unknown>;
+    }>;
+    expect(
+      isHealthcheckSecretMarker(persistedCollectors[0].config.password as string),
+    ).toBe(true);
+    expect([...internalSecrets.store.values()]).toContain("collector-plain");
+
+    // The strategy config (no schema to walk) is left untouched - never wiped,
+    // and its legacy plaintext is NOT migrated (nothing else we can do).
+    const persistedConfig = updates.at(-1)?.config as Record<string, unknown>;
+    expect(persistedConfig.password).toBe("strategy-plain");
+    expect([...internalSecrets.store.values()]).not.toContain("strategy-plain");
+  });
+
+  it("skips the scan entirely when another pod holds the backfill lock", async () => {
+    const advisoryLock = {
+      tryAcquire: mock(async () => undefined), // lock not acquired
+      withXactLock: mock(async () => {
+        throw new Error("must not run when the global lock is unavailable");
+      }),
+    };
+    await backfillConfigSecrets({
+      db: {} as never,
+      registry: {} as never,
+      collectorRegistry: {} as never,
+      internalSecrets: fakeInternalSecrets(),
+      advisoryLock: advisoryLock as never,
+      logger: noopLogger,
+    });
+    expect(advisoryLock.withXactLock).not.toHaveBeenCalled();
+  });
+});

package/src/config-secrets-backfill.ts

@@ -0,0 +1,129 @@
+import { eq } from "drizzle-orm";
+import type {
+  AdvisoryLockService,
+  HealthCheckRegistry,
+  CollectorRegistry,
+  Logger,
+  SafeDatabase,
+} from "@checkstack/backend-api";
+import type { InternalSecretsService } from "@checkstack/secrets-backend";
+import type { CollectorConfigEntry } from "@checkstack/healthcheck-common";
+import { healthCheckConfigurations } from "./schema";
+import * as schema from "./schema";
+import {
+  extractConfigurationSecrets,
+  healthcheckConfigLockKey,
+} from "./config-secrets";
+
+/**
+ * One-time (idempotent) boot backfill: move inline `x-secret` values that
+ * pre-date the config-secrets channel out of stored health-check
+ * configurations and into internal secrets.
+ *
+ * Rows written after the channel shipped hold only markers / `${{ secrets.* }}`
+ * references, which the extraction walk skips - so re-running is a no-op and
+ * the job is safe to run on EVERY boot. A cross-pod advisory lock ensures a
+ * single pod performs the scan; the others skip (the winner's result is in
+ * the shared DB either way).
+ *
+ * Fail-open per row: one row with an unregistered strategy (plugin removed)
+ * or a broken shape must not wedge boot - it is logged and left as-is; the
+ * executor treats its bare literal exactly as before (legacy pass-through).
+ */
+export async function backfillConfigSecrets({
+  db,
+  registry,
+  collectorRegistry,
+  internalSecrets,
+  advisoryLock,
+  logger,
+}: {
+  db: SafeDatabase<typeof schema>;
+  registry: HealthCheckRegistry;
+  collectorRegistry: CollectorRegistry;
+  internalSecrets: InternalSecretsService;
+  advisoryLock: AdvisoryLockService;
+  logger: Logger;
+}): Promise<void> {
+  const lock = await advisoryLock.tryAcquire("healthcheck:config-secrets-backfill");
+  if (!lock) {
+    logger.debug("Config-secrets backfill already running on another pod, skipping");
+    return;
+  }
+
+  try {
+    const rows = await db.select().from(healthCheckConfigurations);
+    let migratedRows = 0;
+    let movedValues = 0;
+
+    for (const { id: rowId } of rows) {
+      // Serialize each row's read-modify-write under the SAME per-config lock
+      // updateConfiguration uses, and re-read the row INSIDE the lock. Without
+      // this the backfill could write back a pre-lock snapshot over a concurrent
+      // edit - resurrecting a just-rotated secret and reverting the edit.
+      await advisoryLock.withXactLock({
+        key: healthcheckConfigLockKey(rowId),
+        fn: async () => {
+          const [row] = await db
+            .select()
+            .from(healthCheckConfigurations)
+            .where(eq(healthCheckConfigurations.id, rowId));
+          if (!row) return; // deleted between the scan and the lock
+
+          // An unregistered strategy has no schema: its own strategy-level
+          // legacy secrets stay as-is (pass-through, as before), but we still
+          // migrate every REGISTERED collector's inline secrets rather than
+          // skipping the whole row - leaving those plaintext would defeat the
+          // backfill. extractConfigurationSecrets handles the undefined schema.
+          const strategySchema =
+            registry.getStrategy(row.strategyId)?.config.schema;
+          if (!strategySchema) {
+            logger.warn(
+              `Config-secrets backfill: strategy ${row.strategyId} not registered; leaving its strategy config as-is and migrating collector secrets for configuration ${row.id}`,
+            );
+          }
+
+          try {
+            const collectors: CollectorConfigEntry[] | undefined =
+              row.collectors ?? undefined;
+            const extracted = await extractConfigurationSecrets({
+              configurationId: row.id,
+              strategySchema,
+              config: row.config as Record<string, unknown>,
+              collectors,
+              getCollectorSchema: (collectorId) =>
+                collectorRegistry.getCollector(collectorId)?.collector.config
+                  .schema,
+              internalSecrets,
+            });
+            if (extracted.extracted === 0) return;
+
+            await db
+              .update(healthCheckConfigurations)
+              .set({
+                config: extracted.config,
+                collectors: extracted.collectors,
+                updatedAt: new Date(),
+              })
+              .where(eq(healthCheckConfigurations.id, row.id));
+            migratedRows++;
+            movedValues += extracted.extracted;
+          } catch (error) {
+            logger.warn(
+              `Config-secrets backfill: failed for configuration ${row.id}, leaving as-is`,
+              error,
+            );
+          }
+        },
+      });
+    }
+
+    if (migratedRows > 0) {
+      logger.info(
+        `Config-secrets backfill: moved ${movedValues} inline secret value(s) from ${migratedRows} configuration(s) into the internal secret store`,
+      );
+    }
+  } finally {
+    await lock.release();
+  }
+}