@checkstack/healthcheck-backend 1.14.0 → 1.15.0

package/CHANGELOG.md

@@ -1,5 +1,99 @@
 # @checkstack/healthcheck-backend
 
+## 1.15.0
+
+### Minor Changes
+
+- faf98f5: Security: config secrets (health-check strategy/collector credentials such as
+  SSH passwords, DB credentials, HTTP auth, and integration connection
+  credentials) ride ONE shared, domain-agnostic extraction channel instead of
+  being stored as plaintext or re-implemented per plugin.
+
+  New primitive and shared service:
+
+  - `configSecret({ id })` (in `@checkstack/backend-api`) declares an
+    extraction-channel secret keyed by a STABLE `id`, independent of field name or
+    position, so renaming or reordering a field never orphans its value. Use it
+    (not `configString({ "x-secret": true })`) for any credential whose config is
+    relayed to a satellite, projected to AI, or diffed by GitOps. `validateSecretIds`
+    rejects, at plugin registration, an `x-secret` field with no `id`, a duplicate
+    `id`, or a secret nested in an un-keyable container (array / record / tuple /
+    map) - so a mis-keyable schema fails boot rather than at run time.
+  - `ConfigSecretChannel` (in `@checkstack/secrets-backend`) is the single
+    extract / inflate / collect / redact / merge / delete / prune implementation.
+    Health-checks and integration connections both BIND it to their own scope
+    (marker prefix + internal-secret key layout); neither re-implements the walk.
+
+  Lifecycle (both bindings):
+
+  - **Write**: an inline value is extracted into the encrypted internal secret
+    store; the stored config keeps only an opaque marker. `${{ secrets.NAME }}`
+    references are stored verbatim and resolve through the active backend (local
+    or Vault) at run time.
+  - **Read**: configuration and connection reads strip `x-secret` values and
+    internal markers while keeping `${{ secrets.NAME }}` references visible; the
+    AI `getConfigurations` tool and create/update responses are redacted too. A
+    value never reaches a browser or an AI model context.
+  - **Run**: the core executor inflates markers/references in memory just before
+    the client is built. Satellites receive markers only and fetch values
+    just-in-time over the authenticated WS channel, per run, never persisted, then
+    fail CLOSED if any marker/reference survives resolution.
+  - **No orphan**: clearing a secret, removing a field/collector, swapping an
+    inline value for a reference, updating a connection, or deleting a
+    configuration/connection deletes the now-unreferenced internal secret. Cleanup
+    is schema-free (scans markers by prefix) and best-effort on delete, so it works
+    even when the owning plugin is uninstalled and never blocks a delete.
+  - **Forged-marker safe**: extract/inflate key each internal secret by the
+    SCHEMA leaf's stable `id`, never by an id parsed out of a stored marker string,
+    so a crafted marker can never resolve or delete another scope's secret.
+
+  Health-checks additionally get an idempotent, advisory-locked backfill that
+  moves pre-existing plaintext values into the internal store, and per-config-id
+  locking so concurrent writers across pods can never leave a dangling marker.
+  Integration connection credentials keep their released `__connref__:` marker
+  prefix and key layout (id equals the flat field name), so existing stored
+  connections are byte-compatible.
+
+  BREAKING CHANGES:
+
+  - Configuration and connection reads no longer include `x-secret` field values
+    (clients must treat blank-on-save as keep-existing; the bundled editors
+    already do).
+  - Satellites must be upgraded together with the core: an old satellite cannot
+    resolve the markers a new core stores, so its credentialed checks fail until
+    upgraded.
+
+### Patch Changes
+
+- Updated dependencies [faf98f5]
+- Updated dependencies [faf98f5]
+  - @checkstack/ai-backend@0.10.6
+  - @checkstack/backend-api@0.29.0
+  - @checkstack/secrets-backend@0.3.0
+  - @checkstack/secrets-common@0.3.0
+  - @checkstack/common@0.20.0
+  - @checkstack/healthcheck-common@1.12.0
+  - @checkstack/satellite-backend@0.8.0
+  - @checkstack/automation-backend@0.10.8
+  - @checkstack/catalog-backend@1.6.6
+  - @checkstack/incident-backend@1.9.4
+  - @checkstack/command-backend@0.2.18
+  - @checkstack/gitops-backend@0.5.18
+  - @checkstack/script-packages-backend@0.3.22
+  - @checkstack/status-page-backend@0.4.5
+  - @checkstack/gitops-common@0.7.1
+  - @checkstack/sdk@0.122.1
+  - @checkstack/ai-common@0.6.4
+  - @checkstack/cache-api@0.3.17
+  - @checkstack/catalog-common@2.6.1
+  - @checkstack/incident-common@1.7.1
+  - @checkstack/maintenance-common@1.8.1
+  - @checkstack/notification-common@1.5.1
+  - @checkstack/queue-api@0.3.17
+  - @checkstack/signal-common@0.2.15
+  - @checkstack/status-page-common@0.5.1
+  - @checkstack/cache-utils@0.2.22
+
 ## 1.14.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/healthcheck-backend",
-  "version": "1.14.0",
+  "version": "1.15.0",
   "license": "Elastic-2.0",
   "type": "module",
   "main": "src/index.ts",
@@ -14,32 +14,32 @@
     "lint:code": "eslint . --max-warnings 0"
   },
   "dependencies": {
-    "@checkstack/ai-backend": "0.10.5",
-    "@checkstack/ai-common": "0.6.3",
-    "@checkstack/automation-backend": "0.10.7",
-    "@checkstack/backend-api": "0.28.0",
-    "@checkstack/cache-api": "0.3.16",
-    "@checkstack/cache-utils": "0.2.21",
-    "@checkstack/catalog-backend": "1.6.5",
-    "@checkstack/catalog-common": "2.6.0",
-    "@checkstack/command-backend": "0.2.17",
-    "@checkstack/common": "0.19.0",
-    "@checkstack/gitops-backend": "0.5.17",
-    "@checkstack/gitops-common": "0.7.0",
-    "@checkstack/healthcheck-common": "1.11.0",
-    "@checkstack/incident-backend": "1.9.3",
-    "@checkstack/incident-common": "1.7.0",
-    "@checkstack/maintenance-common": "1.8.0",
-    "@checkstack/notification-common": "1.5.0",
-    "@checkstack/queue-api": "0.3.16",
-    "@checkstack/satellite-backend": "0.7.8",
-    "@checkstack/script-packages-backend": "0.3.21",
-    "@checkstack/sdk": "0.121.0",
-    "@checkstack/secrets-backend": "0.2.17",
-    "@checkstack/secrets-common": "0.2.8",
-    "@checkstack/signal-common": "0.2.14",
-    "@checkstack/status-page-backend": "0.4.4",
-    "@checkstack/status-page-common": "0.5.0",
+    "@checkstack/ai-backend": "0.10.6",
+    "@checkstack/ai-common": "0.6.4",
+    "@checkstack/automation-backend": "0.10.8",
+    "@checkstack/backend-api": "0.29.0",
+    "@checkstack/cache-api": "0.3.17",
+    "@checkstack/cache-utils": "0.2.22",
+    "@checkstack/catalog-backend": "1.6.6",
+    "@checkstack/catalog-common": "2.6.1",
+    "@checkstack/command-backend": "0.2.18",
+    "@checkstack/common": "0.20.0",
+    "@checkstack/gitops-backend": "0.5.18",
+    "@checkstack/gitops-common": "0.7.1",
+    "@checkstack/healthcheck-common": "1.12.0",
+    "@checkstack/incident-backend": "1.9.4",
+    "@checkstack/incident-common": "1.7.1",
+    "@checkstack/maintenance-common": "1.8.1",
+    "@checkstack/notification-common": "1.5.1",
+    "@checkstack/queue-api": "0.3.17",
+    "@checkstack/satellite-backend": "0.8.0",
+    "@checkstack/script-packages-backend": "0.3.22",
+    "@checkstack/sdk": "0.122.1",
+    "@checkstack/secrets-backend": "0.3.0",
+    "@checkstack/secrets-common": "0.3.0",
+    "@checkstack/signal-common": "0.2.15",
+    "@checkstack/status-page-backend": "0.4.5",
+    "@checkstack/status-page-common": "0.5.1",
     "@hono/zod-validator": "^0.7.6",
     "@orpc/contract": "^1.14.4",
     "@orpc/server": "^1.14.4",
@@ -52,8 +52,8 @@
   },
   "devDependencies": {
     "@checkstack/drizzle-helper": "0.0.6",
-    "@checkstack/scripts": "0.7.0",
-    "@checkstack/test-utils-backend": "0.1.51",
+    "@checkstack/scripts": "0.7.1",
+    "@checkstack/test-utils-backend": "0.1.52",
     "@checkstack/tsconfig": "0.0.7",
     "@types/bun": "^1.0.0",
     "@types/tdigest": "^0.1.5",

package/src/ai/healthcheck-propose.ts

@@ -103,9 +103,17 @@ const UNASSIGNED_GUIDANCE =
 export async function validateHealthcheckDraft({
   input,
   rpcClient,
+  existingConfigurationId,
 }: {
   input: HealthcheckProposeInput;
   rpcClient: RpcClient;
+  /**
+   * When validating an UPDATE, the id of the config being edited. The server
+   * restores that config's stored secrets before validating, so a kept secret
+   * (redacted out of the read) does not fail a required-secret check. Omit for
+   * a create draft, where blank secrets stay required.
+   */
+  existingConfigurationId?: string;
 }): Promise<{ strategy: HealthCheckStrategyDto; collectors: CollectorDto[] }> {
   const healthcheckClient = rpcClient.forPlugin(HealthCheckApi);
 
@@ -116,6 +124,7 @@ export async function validateHealthcheckDraft({
     config: input.config,
     intervalSeconds: input.intervalSeconds,
     collectors: input.collectors,
+    ...(existingConfigurationId ? { existingConfigurationId } : {}),
   });
   if (!validation.valid) {
     throw new HealthcheckProposeValidationError(

package/src/ai/healthcheck-update.ts

@@ -82,10 +82,14 @@ export function createHealthcheckUpdateTool(): RegisteredAiTool<
     }
     const merged = mergeConfig({ existing, body: input.body });
     // Deep-validate the merged result (throws with structured detail if invalid,
-    // including bad assertion fields/operators).
+    // including bad assertion fields/operators). Pass the id so the server
+    // restores stored secrets before validating - `existing.config` is
+    // redacted, so a kept `x-secret` field arrives blank and would otherwise
+    // fail a required-secret check even for a name-only edit.
     const { strategy } = await validateHealthcheckDraft({
       input: merged,
       rpcClient,
+      existingConfigurationId: input.id,
     });
     // before -> after diff over the updatable fields only (the existing config
     // also carries id/timestamps/paused that are not part of an update).

package/src/config-secrets-backfill.test.ts

@@ -0,0 +1,142 @@
+import { describe, it, expect, mock } from "bun:test";
+import { z } from "zod";
+import { configSecret, Versioned } from "@checkstack/backend-api";
+import type { InternalSecretsService } from "@checkstack/secrets-backend";
+import { internalSecretName } from "@checkstack/secrets-common";
+import { backfillConfigSecrets } from "./config-secrets-backfill";
+import {
+  healthcheckConfigLockKey,
+  isHealthcheckSecretMarker,
+} from "./config-secrets";
+
+const collectorSchema = z.object({
+  url: z.string(),
+  password: configSecret({ id: "password" }).optional(),
+});
+
+function fakeInternalSecrets(): InternalSecretsService & {
+  store: Map<string, string>;
+} {
+  const store = new Map<string, string>();
+  return {
+    store,
+    async set({ parts, value }) {
+      store.set(internalSecretName(...parts), value);
+    },
+    async get({ parts }) {
+      return store.get(internalSecretName(...parts));
+    },
+    async delete({ parts }) {
+      store.delete(internalSecretName(...parts));
+    },
+  };
+}
+
+const noopLogger = {
+  debug: mock(() => {}),
+  info: mock(() => {}),
+  warn: mock(() => {}),
+  error: mock(() => {}),
+} as never;
+
+describe("backfillConfigSecrets", () => {
+  it("migrates a registered collector's inline secret even under an UNREGISTERED strategy, per config lock", async () => {
+    // One legacy row: strategy plugin is gone, but the collector plugin is
+    // present and its config carries an inline plaintext secret.
+    let row: Record<string, unknown> = {
+      id: "cfg-bf",
+      strategyId: "gone.strategy",
+      config: { url: "https://x", password: "strategy-plain" }, // no schema -> left as-is
+      collectors: [
+        {
+          id: "c1",
+          collectorId: "reg.collector",
+          config: { url: "https://y", password: "collector-plain" },
+        },
+      ],
+    };
+    const updates: Record<string, unknown>[] = [];
+    const db = {
+      // Initial scan: `db.select().from(table)` awaited directly.
+      // Per-row re-read: `db.select().from(table).where(eq(...))`.
+      select: () => ({
+        from: () =>
+          Object.assign(Promise.resolve([row]), {
+            where: () => Promise.resolve([row]),
+          }),
+      }),
+      update: () => ({
+        set: (set: Record<string, unknown>) => {
+          updates.push(set);
+          row = { ...row, ...set };
+          return { where: () => Promise.resolve(undefined) };
+        },
+      }),
+    };
+
+    const registry = { getStrategy: mock(() => undefined) };
+    const collectorRegistry = {
+      getCollector: mock((id: string) =>
+        id === "reg.collector"
+          ? { collector: { config: new Versioned({ version: 1, schema: collectorSchema }) } }
+          : undefined,
+      ),
+    };
+
+    const lockKeys: string[] = [];
+    const advisoryLock = {
+      tryAcquire: mock(async () => ({ release: mock(async () => {}) })),
+      withXactLock: async ({ key, fn }: { key: string; fn: () => Promise<unknown> }) => {
+        lockKeys.push(key);
+        return fn();
+      },
+    };
+
+    const internalSecrets = fakeInternalSecrets();
+
+    await backfillConfigSecrets({
+      db: db as never,
+      registry: registry as never,
+      collectorRegistry: collectorRegistry as never,
+      internalSecrets,
+      advisoryLock: advisoryLock as never,
+      logger: noopLogger,
+    });
+
+    // Per-config lock was taken for the row.
+    expect(lockKeys).toEqual([healthcheckConfigLockKey("cfg-bf")]);
+
+    // The collector's inline secret was migrated to a marker + the store.
+    const persistedCollectors = updates.at(-1)?.collectors as Array<{
+      config: Record<string, unknown>;
+    }>;
+    expect(
+      isHealthcheckSecretMarker(persistedCollectors[0].config.password as string),
+    ).toBe(true);
+    expect([...internalSecrets.store.values()]).toContain("collector-plain");
+
+    // The strategy config (no schema to walk) is left untouched - never wiped,
+    // and its legacy plaintext is NOT migrated (nothing else we can do).
+    const persistedConfig = updates.at(-1)?.config as Record<string, unknown>;
+    expect(persistedConfig.password).toBe("strategy-plain");
+    expect([...internalSecrets.store.values()]).not.toContain("strategy-plain");
+  });
+
+  it("skips the scan entirely when another pod holds the backfill lock", async () => {
+    const advisoryLock = {
+      tryAcquire: mock(async () => undefined), // lock not acquired
+      withXactLock: mock(async () => {
+        throw new Error("must not run when the global lock is unavailable");
+      }),
+    };
+    await backfillConfigSecrets({
+      db: {} as never,
+      registry: {} as never,
+      collectorRegistry: {} as never,
+      internalSecrets: fakeInternalSecrets(),
+      advisoryLock: advisoryLock as never,
+      logger: noopLogger,
+    });
+    expect(advisoryLock.withXactLock).not.toHaveBeenCalled();
+  });
+});

package/src/config-secrets-backfill.ts

@@ -0,0 +1,129 @@
+import { eq } from "drizzle-orm";
+import type {
+  AdvisoryLockService,
+  HealthCheckRegistry,
+  CollectorRegistry,
+  Logger,
+  SafeDatabase,
+} from "@checkstack/backend-api";
+import type { InternalSecretsService } from "@checkstack/secrets-backend";
+import type { CollectorConfigEntry } from "@checkstack/healthcheck-common";
+import { healthCheckConfigurations } from "./schema";
+import * as schema from "./schema";
+import {
+  extractConfigurationSecrets,
+  healthcheckConfigLockKey,
+} from "./config-secrets";
+
+/**
+ * One-time (idempotent) boot backfill: move inline `x-secret` values that
+ * pre-date the config-secrets channel out of stored health-check
+ * configurations and into internal secrets.
+ *
+ * Rows written after the channel shipped hold only markers / `${{ secrets.* }}`
+ * references, which the extraction walk skips - so re-running is a no-op and
+ * the job is safe to run on EVERY boot. A cross-pod advisory lock ensures a
+ * single pod performs the scan; the others skip (the winner's result is in
+ * the shared DB either way).
+ *
+ * Fail-open per row: one row with an unregistered strategy (plugin removed)
+ * or a broken shape must not wedge boot - it is logged and left as-is; the
+ * executor treats its bare literal exactly as before (legacy pass-through).
+ */
+export async function backfillConfigSecrets({
+  db,
+  registry,
+  collectorRegistry,
+  internalSecrets,
+  advisoryLock,
+  logger,
+}: {
+  db: SafeDatabase<typeof schema>;
+  registry: HealthCheckRegistry;
+  collectorRegistry: CollectorRegistry;
+  internalSecrets: InternalSecretsService;
+  advisoryLock: AdvisoryLockService;
+  logger: Logger;
+}): Promise<void> {
+  const lock = await advisoryLock.tryAcquire("healthcheck:config-secrets-backfill");
+  if (!lock) {
+    logger.debug("Config-secrets backfill already running on another pod, skipping");
+    return;
+  }
+
+  try {
+    const rows = await db.select().from(healthCheckConfigurations);
+    let migratedRows = 0;
+    let movedValues = 0;
+
+    for (const { id: rowId } of rows) {
+      // Serialize each row's read-modify-write under the SAME per-config lock
+      // updateConfiguration uses, and re-read the row INSIDE the lock. Without
+      // this the backfill could write back a pre-lock snapshot over a concurrent
+      // edit - resurrecting a just-rotated secret and reverting the edit.
+      await advisoryLock.withXactLock({
+        key: healthcheckConfigLockKey(rowId),
+        fn: async () => {
+          const [row] = await db
+            .select()
+            .from(healthCheckConfigurations)
+            .where(eq(healthCheckConfigurations.id, rowId));
+          if (!row) return; // deleted between the scan and the lock
+
+          // An unregistered strategy has no schema: its own strategy-level
+          // legacy secrets stay as-is (pass-through, as before), but we still
+          // migrate every REGISTERED collector's inline secrets rather than
+          // skipping the whole row - leaving those plaintext would defeat the
+          // backfill. extractConfigurationSecrets handles the undefined schema.
+          const strategySchema =
+            registry.getStrategy(row.strategyId)?.config.schema;
+          if (!strategySchema) {
+            logger.warn(
+              `Config-secrets backfill: strategy ${row.strategyId} not registered; leaving its strategy config as-is and migrating collector secrets for configuration ${row.id}`,
+            );
+          }
+
+          try {
+            const collectors: CollectorConfigEntry[] | undefined =
+              row.collectors ?? undefined;
+            const extracted = await extractConfigurationSecrets({
+              configurationId: row.id,
+              strategySchema,
+              config: row.config as Record<string, unknown>,
+              collectors,
+              getCollectorSchema: (collectorId) =>
+                collectorRegistry.getCollector(collectorId)?.collector.config
+                  .schema,
+              internalSecrets,
+            });
+            if (extracted.extracted === 0) return;
+
+            await db
+              .update(healthCheckConfigurations)
+              .set({
+                config: extracted.config,
+                collectors: extracted.collectors,
+                updatedAt: new Date(),
+              })
+              .where(eq(healthCheckConfigurations.id, row.id));
+            migratedRows++;
+            movedValues += extracted.extracted;
+          } catch (error) {
+            logger.warn(
+              `Config-secrets backfill: failed for configuration ${row.id}, leaving as-is`,
+              error,
+            );
+          }
+        },
+      });
+    }
+
+    if (migratedRows > 0) {
+      logger.info(
+        `Config-secrets backfill: moved ${movedValues} inline secret value(s) from ${migratedRows} configuration(s) into the internal secret store`,
+      );
+    }
+  } finally {
+    await lock.release();
+  }
+}