@checkstack/healthcheck-backend 1.14.0 → 1.16.0

package/CHANGELOG.md

@@ -1,5 +1,176 @@
 # @checkstack/healthcheck-backend
 
+## 1.16.0
+
+### Minor Changes
+
+- c55d7c6: Make collector assertions analyzable: structured per-assertion outcomes on
+  every run, pass/fail counts in every aggregate tier, and dedicated analysis
+  surfaces. Previously a passing assertion left no trace and only the first
+  failure was recorded as a string.
+
+  - `@checkstack/healthcheck-common` adds the assertion-analytics contract:
+    `AssertionOutcomeSchema`, per-bucket `BucketAssertionStats` (stored under
+    the platform-owned top-level `assertions` key of `aggregatedResult`), and
+    the canonical assertion identity key (`computeAssertionKey` /
+    `parseAssertionKey`, a JSON tuple of field/jsonPath/operator/value).
+    Editing an assertion starts a new series; identical duplicates collapse.
+  - The executor evaluates ALL assertions (no first-failure short-circuit) and
+    stores `_assertions` on each collector entry alongside the unchanged
+    `_assertionFailed` compatibility string. Pass/fail counts are folded into
+    the hourly realtime aggregation, the on-read raw tier, cross-tier bucket
+    re-merges, and the daily retention rollup (assertion counts are the only
+    `aggregatedResult` content that survives the rollup - they are purely
+    additive), so assertion analytics do not silently end at the hourly
+    retention horizon.
+  - Satellite ingest now evaluates assertions on the core
+    (`ingestSatelliteResult`), downgrading a satellite-reported healthy run
+    whose assertions fail, and strips ephemeral result fields (e.g. raw HTTP
+    bodies) at ingest for parity with local runs. BEHAVIOR CHANGE:
+    satellite-executed checks previously never enforced assertions at all;
+    they now do, with no satellite upgrade or wire-protocol change. Buffered
+    satellite results are evaluated against the configuration current at
+    ingest time.
+  - The run detail gains an Assertions tab (per-collector groups, pass AND
+    fail rows with expected vs actual, a legacy fallback for pre-feature
+    runs), and the drawer's auto-chart grid leads each collector group with
+    per-assertion pass-rate tiles (sparkline of per-bucket pass rate,
+    expandable to a pass/fail StackedTimeline; currently-configured assertions
+    appear before any data exists, historical-only series are flagged).
+
+  State & scale: all new state lives in the existing `healthCheckRuns.result`
+  and `healthCheckAggregates.aggregated_result` jsonb columns (durable, shared
+  Postgres - no new tables, no pod-local state); reads resolve identically on
+  every pod; the run-vs-bucket duplication is the platform's existing
+  raw-vs-aggregate tiering with the existing single-writer upsert paths.
+
+### Patch Changes
+
+- Updated dependencies [c55d7c6]
+- Updated dependencies [c55d7c6]
+- Updated dependencies [a83bcc2]
+- Updated dependencies [c55d7c6]
+  - @checkstack/ai-backend@0.10.7
+  - @checkstack/healthcheck-common@1.13.0
+  - @checkstack/common@0.21.0
+  - @checkstack/automation-backend@0.10.9
+  - @checkstack/catalog-backend@1.6.7
+  - @checkstack/incident-backend@1.9.5
+  - @checkstack/satellite-backend@0.8.1
+  - @checkstack/backend-api@0.29.1
+  - @checkstack/sdk@0.123.1
+  - @checkstack/ai-common@0.6.5
+  - @checkstack/cache-api@0.3.18
+  - @checkstack/catalog-common@2.6.2
+  - @checkstack/command-backend@0.2.19
+  - @checkstack/gitops-backend@0.5.19
+  - @checkstack/gitops-common@0.7.2
+  - @checkstack/incident-common@1.7.2
+  - @checkstack/maintenance-common@1.8.2
+  - @checkstack/notification-common@1.5.2
+  - @checkstack/queue-api@0.3.18
+  - @checkstack/script-packages-backend@0.3.23
+  - @checkstack/secrets-backend@0.3.1
+  - @checkstack/secrets-common@0.3.1
+  - @checkstack/signal-common@0.2.16
+  - @checkstack/status-page-backend@0.4.6
+  - @checkstack/status-page-common@0.5.2
+  - @checkstack/cache-utils@0.2.23
+
+## 1.15.0
+
+### Minor Changes
+
+- faf98f5: Security: config secrets (health-check strategy/collector credentials such as
+  SSH passwords, DB credentials, HTTP auth, and integration connection
+  credentials) ride ONE shared, domain-agnostic extraction channel instead of
+  being stored as plaintext or re-implemented per plugin.
+
+  New primitive and shared service:
+
+  - `configSecret({ id })` (in `@checkstack/backend-api`) declares an
+    extraction-channel secret keyed by a STABLE `id`, independent of field name or
+    position, so renaming or reordering a field never orphans its value. Use it
+    (not `configString({ "x-secret": true })`) for any credential whose config is
+    relayed to a satellite, projected to AI, or diffed by GitOps. `validateSecretIds`
+    rejects, at plugin registration, an `x-secret` field with no `id`, a duplicate
+    `id`, or a secret nested in an un-keyable container (array / record / tuple /
+    map) - so a mis-keyable schema fails boot rather than at run time.
+  - `ConfigSecretChannel` (in `@checkstack/secrets-backend`) is the single
+    extract / inflate / collect / redact / merge / delete / prune implementation.
+    Health-checks and integration connections both BIND it to their own scope
+    (marker prefix + internal-secret key layout); neither re-implements the walk.
+
+  Lifecycle (both bindings):
+
+  - **Write**: an inline value is extracted into the encrypted internal secret
+    store; the stored config keeps only an opaque marker. `${{ secrets.NAME }}`
+    references are stored verbatim and resolve through the active backend (local
+    or Vault) at run time.
+  - **Read**: configuration and connection reads strip `x-secret` values and
+    internal markers while keeping `${{ secrets.NAME }}` references visible; the
+    AI `getConfigurations` tool and create/update responses are redacted too. A
+    value never reaches a browser or an AI model context.
+  - **Run**: the core executor inflates markers/references in memory just before
+    the client is built. Satellites receive markers only and fetch values
+    just-in-time over the authenticated WS channel, per run, never persisted, then
+    fail CLOSED if any marker/reference survives resolution.
+  - **No orphan**: clearing a secret, removing a field/collector, swapping an
+    inline value for a reference, updating a connection, or deleting a
+    configuration/connection deletes the now-unreferenced internal secret. Cleanup
+    is schema-free (scans markers by prefix) and best-effort on delete, so it works
+    even when the owning plugin is uninstalled and never blocks a delete.
+  - **Forged-marker safe**: extract/inflate key each internal secret by the
+    SCHEMA leaf's stable `id`, never by an id parsed out of a stored marker string,
+    so a crafted marker can never resolve or delete another scope's secret.
+
+  Health-checks additionally get an idempotent, advisory-locked backfill that
+  moves pre-existing plaintext values into the internal store, and per-config-id
+  locking so concurrent writers across pods can never leave a dangling marker.
+  Integration connection credentials keep their released `__connref__:` marker
+  prefix and key layout (id equals the flat field name), so existing stored
+  connections are byte-compatible.
+
+  BREAKING CHANGES:
+
+  - Configuration and connection reads no longer include `x-secret` field values
+    (clients must treat blank-on-save as keep-existing; the bundled editors
+    already do).
+  - Satellites must be upgraded together with the core: an old satellite cannot
+    resolve the markers a new core stores, so its credentialed checks fail until
+    upgraded.
+
+### Patch Changes
+
+- Updated dependencies [faf98f5]
+- Updated dependencies [faf98f5]
+  - @checkstack/ai-backend@0.10.6
+  - @checkstack/backend-api@0.29.0
+  - @checkstack/secrets-backend@0.3.0
+  - @checkstack/secrets-common@0.3.0
+  - @checkstack/common@0.20.0
+  - @checkstack/healthcheck-common@1.12.0
+  - @checkstack/satellite-backend@0.8.0
+  - @checkstack/automation-backend@0.10.8
+  - @checkstack/catalog-backend@1.6.6
+  - @checkstack/incident-backend@1.9.4
+  - @checkstack/command-backend@0.2.18
+  - @checkstack/gitops-backend@0.5.18
+  - @checkstack/script-packages-backend@0.3.22
+  - @checkstack/status-page-backend@0.4.5
+  - @checkstack/gitops-common@0.7.1
+  - @checkstack/sdk@0.122.1
+  - @checkstack/ai-common@0.6.4
+  - @checkstack/cache-api@0.3.17
+  - @checkstack/catalog-common@2.6.1
+  - @checkstack/incident-common@1.7.1
+  - @checkstack/maintenance-common@1.8.1
+  - @checkstack/notification-common@1.5.1
+  - @checkstack/queue-api@0.3.17
+  - @checkstack/signal-common@0.2.15
+  - @checkstack/status-page-common@0.5.1
+  - @checkstack/cache-utils@0.2.22
+
 ## 1.14.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/healthcheck-backend",
-  "version": "1.14.0",
+  "version": "1.16.0",
   "license": "Elastic-2.0",
   "type": "module",
   "main": "src/index.ts",
@@ -14,32 +14,32 @@
     "lint:code": "eslint . --max-warnings 0"
   },
   "dependencies": {
-    "@checkstack/ai-backend": "0.10.5",
-    "@checkstack/ai-common": "0.6.3",
-    "@checkstack/automation-backend": "0.10.7",
-    "@checkstack/backend-api": "0.28.0",
-    "@checkstack/cache-api": "0.3.16",
-    "@checkstack/cache-utils": "0.2.21",
-    "@checkstack/catalog-backend": "1.6.5",
-    "@checkstack/catalog-common": "2.6.0",
-    "@checkstack/command-backend": "0.2.17",
-    "@checkstack/common": "0.19.0",
-    "@checkstack/gitops-backend": "0.5.17",
-    "@checkstack/gitops-common": "0.7.0",
-    "@checkstack/healthcheck-common": "1.11.0",
-    "@checkstack/incident-backend": "1.9.3",
-    "@checkstack/incident-common": "1.7.0",
-    "@checkstack/maintenance-common": "1.8.0",
-    "@checkstack/notification-common": "1.5.0",
-    "@checkstack/queue-api": "0.3.16",
-    "@checkstack/satellite-backend": "0.7.8",
-    "@checkstack/script-packages-backend": "0.3.21",
-    "@checkstack/sdk": "0.121.0",
-    "@checkstack/secrets-backend": "0.2.17",
-    "@checkstack/secrets-common": "0.2.8",
-    "@checkstack/signal-common": "0.2.14",
-    "@checkstack/status-page-backend": "0.4.4",
-    "@checkstack/status-page-common": "0.5.0",
+    "@checkstack/ai-backend": "0.10.7",
+    "@checkstack/ai-common": "0.6.5",
+    "@checkstack/automation-backend": "0.10.9",
+    "@checkstack/backend-api": "0.29.1",
+    "@checkstack/cache-api": "0.3.18",
+    "@checkstack/cache-utils": "0.2.23",
+    "@checkstack/catalog-backend": "1.6.7",
+    "@checkstack/catalog-common": "2.6.2",
+    "@checkstack/command-backend": "0.2.19",
+    "@checkstack/common": "0.21.0",
+    "@checkstack/gitops-backend": "0.5.19",
+    "@checkstack/gitops-common": "0.7.2",
+    "@checkstack/healthcheck-common": "1.13.0",
+    "@checkstack/incident-backend": "1.9.5",
+    "@checkstack/incident-common": "1.7.2",
+    "@checkstack/maintenance-common": "1.8.2",
+    "@checkstack/notification-common": "1.5.2",
+    "@checkstack/queue-api": "0.3.18",
+    "@checkstack/satellite-backend": "0.8.1",
+    "@checkstack/script-packages-backend": "0.3.23",
+    "@checkstack/sdk": "0.123.1",
+    "@checkstack/secrets-backend": "0.3.1",
+    "@checkstack/secrets-common": "0.3.1",
+    "@checkstack/signal-common": "0.2.16",
+    "@checkstack/status-page-backend": "0.4.6",
+    "@checkstack/status-page-common": "0.5.2",
     "@hono/zod-validator": "^0.7.6",
     "@orpc/contract": "^1.14.4",
     "@orpc/server": "^1.14.4",
@@ -52,8 +52,8 @@
   },
   "devDependencies": {
     "@checkstack/drizzle-helper": "0.0.6",
-    "@checkstack/scripts": "0.7.0",
-    "@checkstack/test-utils-backend": "0.1.51",
+    "@checkstack/scripts": "0.7.2",
+    "@checkstack/test-utils-backend": "0.1.53",
     "@checkstack/tsconfig": "0.0.7",
     "@types/bun": "^1.0.0",
     "@types/tdigest": "^0.1.5",

package/src/aggregation-utils.test.ts

@@ -4,12 +4,14 @@ import {
   calculateLatencyStats,
   countStatuses,
   extractLatencies,
+  foldRunAssertionStats,
   mergeTieredBuckets,
   combineBuckets,
   reaggregateBuckets,
   mergeAggregatedBucketResults,
   type NormalizedBucket,
 } from "./aggregation-utils";
+import { computeAssertionKey } from "@checkstack/healthcheck-common";
 import {
   VersionedAggregated,
   aggregatedCounter,
@@ -982,3 +984,133 @@ describe("aggregation-utils", () => {
     });
   });
 });
+
+describe("assertion stats in aggregation", () => {
+  const KEY = computeAssertionKey({
+    assertion: { field: "statusCode", operator: "equals", value: 200 },
+  });
+
+  const outcome = (passed: boolean) => ({
+    key: KEY,
+    field: "statusCode",
+    operator: "equals",
+    value: "200",
+    passed,
+  });
+
+  describe("foldRunAssertionStats", () => {
+    it("folds outcomes across a bucket's runs per collector entry", () => {
+      const runs = [
+        {
+          metadata: {
+            collectors: {
+              "uuid-1": { _collectorId: "c", _assertions: [outcome(true)] },
+            },
+          },
+        },
+        {
+          metadata: {
+            collectors: {
+              "uuid-1": { _collectorId: "c", _assertions: [outcome(false)] },
+            },
+          },
+        },
+        // Pre-feature run without outcomes is tolerated.
+        { metadata: { collectors: { "uuid-1": { _collectorId: "c" } } } },
+      ];
+      expect(foldRunAssertionStats(runs)).toEqual({
+        "uuid-1": { [KEY]: { passCount: 1, failCount: 1 } },
+      });
+    });
+
+    it("returns undefined when no run carries outcomes", () => {
+      expect(foldRunAssertionStats([{ metadata: {} }, {}])).toBeUndefined();
+    });
+
+    it("ignores malformed outcome entries", () => {
+      const runs = [
+        {
+          metadata: {
+            collectors: {
+              "uuid-1": {
+                _collectorId: "c",
+                _assertions: ["garbage", outcome(true)],
+              },
+            },
+          },
+        },
+      ];
+      expect(foldRunAssertionStats(runs)).toEqual({
+        "uuid-1": { [KEY]: { passCount: 1, failCount: 0 } },
+      });
+    });
+  });
+
+  describe("mergeAggregatedBucketResults with assertion stats", () => {
+    it("merges counts additively and keeps them out of strategy merging", () => {
+      const mergeAggregatedStates = mock(
+        (a: Record<string, unknown>, b: Record<string, unknown>) => ({
+          ...a,
+          ...b,
+        }),
+      );
+      const registry = {
+        getStrategy: mock(() => ({
+          aggregatedResult: { mergeAggregatedStates },
+        })),
+        register: mock(() => {}),
+        getStrategies: mock(() => []),
+        getStrategiesWithMeta: mock(() => []),
+      } as unknown as HealthCheckRegistry;
+      const collectorRegistry = {
+        register: mock(() => {}),
+        getCollector: mock(() => undefined),
+        getCollectors: mock(() => []),
+      } as unknown as CollectorRegistry;
+
+      const merged = mergeAggregatedBucketResults({
+        aggregatedResults: [
+          {
+            uptime: 1,
+            assertions: { "uuid-1": { [KEY]: { passCount: 3, failCount: 1 } } },
+          },
+          {
+            uptime: 2,
+            assertions: { "uuid-1": { [KEY]: { passCount: 2, failCount: 0 } } },
+          },
+        ],
+        collectorRegistry,
+        registry,
+        strategyId: "test-strategy",
+      });
+
+      expect(merged?.assertions).toEqual({
+        "uuid-1": { [KEY]: { passCount: 5, failCount: 1 } },
+      });
+      // The strategy merger only ever saw its own fields.
+      for (const call of mergeAggregatedStates.mock.calls) {
+        for (const arg of call) {
+          expect(
+            (arg as Record<string, unknown>).assertions,
+          ).toBeUndefined();
+        }
+      }
+    });
+
+    it("single-bucket results pass through with their stats intact", () => {
+      const { collectorRegistry, registry, strategyId } =
+        createMockRegistries();
+      const only = {
+        assertions: { "uuid-1": { [KEY]: { passCount: 7, failCount: 0 } } },
+      };
+      expect(
+        mergeAggregatedBucketResults({
+          aggregatedResults: [only],
+          collectorRegistry,
+          registry,
+          strategyId,
+        }),
+      ).toBe(only);
+    });
+  });
+});

package/src/aggregation-utils.ts

@@ -2,6 +2,15 @@ import type {
   CollectorRegistry,
   HealthCheckRegistry,
 } from "@checkstack/backend-api";
+import {
+  ASSERTIONS_AGG_KEY,
+  AssertionOutcomeSchema,
+  foldOutcomesIntoStats,
+  mergeAssertionStats,
+  readAssertionStats,
+  type AssertionOutcome,
+  type BucketAssertionStats,
+} from "@checkstack/healthcheck-common";
 
 // ===== Percentile Calculation =====
 
@@ -138,7 +147,8 @@ export function aggregateCollectorData(
       const existing = aggregatedByUuid.get(uuid)?.aggregated;
 
       // Strip internal fields from collector data
-      const { _collectorId, _assertionFailed, ...collectorMetadata } = data;
+      const { _collectorId, _assertionFailed, _assertions, ...collectorMetadata } =
+        data;
 
       // Call mergeResult to incrementally aggregate
       const merged = registered.collector.mergeResult(existing, {
@@ -166,6 +176,39 @@ export function aggregateCollectorData(
   return result;
 }
 
+/**
+ * Fold the structured assertion outcomes of a bucket's raw runs into the
+ * per-assertion pass/fail stats (the on-read raw tier's counterpart of the
+ * realtime hourly fold). Returns undefined when no run carries outcomes.
+ */
+export function foldRunAssertionStats(
+  runs: Array<{ metadata?: Record<string, unknown> }>,
+): BucketAssertionStats | undefined {
+  let stats: BucketAssertionStats | undefined;
+  for (const run of runs) {
+    const collectors = run.metadata?.collectors as
+      | Record<string, Record<string, unknown>>
+      | undefined;
+    if (!collectors) continue;
+    for (const [uuid, data] of Object.entries(collectors)) {
+      const rawOutcomes = data._assertions;
+      if (!Array.isArray(rawOutcomes) || rawOutcomes.length === 0) continue;
+      const outcomes: AssertionOutcome[] = [];
+      for (const raw of rawOutcomes) {
+        const parsed = AssertionOutcomeSchema.safeParse(raw);
+        if (parsed.success) outcomes.push(parsed.data);
+      }
+      if (outcomes.length === 0) continue;
+      stats = foldOutcomesIntoStats({
+        stats,
+        collectorEntryId: uuid,
+        outcomes,
+      });
+    }
+  }
+  return stats;
+}
+
 // ===== Bucket Result Merging =====
 
 /**
@@ -201,15 +244,29 @@ export function mergeAggregatedBucketResults(params: {
     return validResults[0];
   }
 
+  // === Platform-level assertion stats (additive, never strategy-merged) ===
+  let mergedAssertionStats: BucketAssertionStats | undefined;
+  for (const result of validResults) {
+    mergedAssertionStats = mergeAssertionStats({
+      a: mergedAssertionStats,
+      b: readAssertionStats({ aggregatedResult: result }),
+    });
+  }
+
   // === Strategy-level field merging ===
   let mergedStrategyFields: Record<string, unknown> = {};
 
   const registeredStrategy = registry.getStrategy(strategyId);
   if (registeredStrategy?.aggregatedResult) {
-    // Extract strategy-level fields (everything except 'collectors')
+    // Extract strategy-level fields (everything except the platform-owned
+    // 'collectors' and 'assertions' keys)
     const strategyDataSets: Array<Record<string, unknown>> = [];
     for (const result of validResults) {
-      const { collectors: _collectors, ...strategyFields } = result;
+      const {
+        collectors: _collectors,
+        [ASSERTIONS_AGG_KEY]: _assertions,
+        ...strategyFields
+      } = result;
       if (Object.keys(strategyFields).length > 0) {
         strategyDataSets.push(strategyFields);
       }
@@ -233,7 +290,11 @@ export function mergeAggregatedBucketResults(params: {
     }
   } else {
     // Strategy not found - preserve strategy fields from first result
-    const { collectors: _collectors, ...firstStrategyFields } = validResults[0];
+    const {
+      collectors: _collectors,
+      [ASSERTIONS_AGG_KEY]: _assertions,
+      ...firstStrategyFields
+    } = validResults[0];
     mergedStrategyFields = firstStrategyFields;
   }
 
@@ -302,17 +363,20 @@ export function mergeAggregatedBucketResults(params: {
     }
   }
 
-  // Combine strategy fields and collector fields
+  // Combine strategy fields, collector fields, and assertion stats
   const hasCollectors = Object.keys(mergedCollectors).length > 0;
   const hasStrategyFields = Object.keys(mergedStrategyFields).length > 0;
 
-  if (!hasCollectors && !hasStrategyFields) {
+  if (!hasCollectors && !hasStrategyFields && mergedAssertionStats === undefined) {
     return undefined;
   }
 
   return {
     ...mergedStrategyFields,
     ...(hasCollectors ? { collectors: mergedCollectors } : {}),
+    ...(mergedAssertionStats === undefined
+      ? {}
+      : { [ASSERTIONS_AGG_KEY]: mergedAssertionStats }),
   };
 }
 

package/src/ai/healthcheck-propose.ts

@@ -103,9 +103,17 @@ const UNASSIGNED_GUIDANCE =
 export async function validateHealthcheckDraft({
   input,
   rpcClient,
+  existingConfigurationId,
 }: {
   input: HealthcheckProposeInput;
   rpcClient: RpcClient;
+  /**
+   * When validating an UPDATE, the id of the config being edited. The server
+   * restores that config's stored secrets before validating, so a kept secret
+   * (redacted out of the read) does not fail a required-secret check. Omit for
+   * a create draft, where blank secrets stay required.
+   */
+  existingConfigurationId?: string;
 }): Promise<{ strategy: HealthCheckStrategyDto; collectors: CollectorDto[] }> {
   const healthcheckClient = rpcClient.forPlugin(HealthCheckApi);
 
@@ -116,6 +124,7 @@ export async function validateHealthcheckDraft({
     config: input.config,
     intervalSeconds: input.intervalSeconds,
     collectors: input.collectors,
+    ...(existingConfigurationId ? { existingConfigurationId } : {}),
   });
   if (!validation.valid) {
     throw new HealthcheckProposeValidationError(

package/src/ai/healthcheck-update.ts

@@ -82,10 +82,14 @@ export function createHealthcheckUpdateTool(): RegisteredAiTool<
     }
     const merged = mergeConfig({ existing, body: input.body });
     // Deep-validate the merged result (throws with structured detail if invalid,
-    // including bad assertion fields/operators).
+    // including bad assertion fields/operators). Pass the id so the server
+    // restores stored secrets before validating - `existing.config` is
+    // redacted, so a kept `x-secret` field arrives blank and would otherwise
+    // fail a required-secret check even for a name-only edit.
     const { strategy } = await validateHealthcheckDraft({
       input: merged,
       rpcClient,
+      existingConfigurationId: input.id,
     });
     // before -> after diff over the updatable fields only (the existing config
     // also carries id/timestamps/paused that are not part of an update).