@checkstack/healthcheck-backend 1.2.0 → 1.4.0

package/src/collector-script-test.test.ts

@@ -0,0 +1,236 @@
+import { describe, expect, test } from "bun:test";
+import type {
+  EsmScriptRunner,
+  ShellScriptRunner,
+} from "@checkstack/backend-api";
+import {
+  buildCollectorContext,
+  buildShellRunContextEnv,
+  runCollectorScriptTest,
+} from "./collector-script-test";
+
+function fakeEsm(impl: EsmScriptRunner["run"]): {
+  runner: EsmScriptRunner;
+  calls: Parameters<EsmScriptRunner["run"]>[0][];
+} {
+  const calls: Parameters<EsmScriptRunner["run"]>[0][] = [];
+  return { calls, runner: { run: (o) => (calls.push(o), impl(o)) } };
+}
+
+function fakeShell(impl: ShellScriptRunner["run"]): {
+  runner: ShellScriptRunner;
+  calls: Parameters<ShellScriptRunner["run"]>[0][];
+} {
+  const calls: Parameters<ShellScriptRunner["run"]>[0][] = [];
+  return { calls, runner: { run: (o) => (calls.push(o), impl(o)) } };
+}
+
+describe("buildShellRunContextEnv", () => {
+  test("emits check + system vars when present", () => {
+    const env = buildShellRunContextEnv({
+      check: { id: "c1", name: "CPU", intervalSeconds: 60 },
+      system: { id: "s1", name: "web-1" },
+    });
+    expect(env).toEqual({
+      CHECKSTACK_CHECK_ID: "c1",
+      CHECKSTACK_CHECK_NAME: "CPU",
+      CHECKSTACK_CHECK_INTERVAL_SECONDS: "60",
+      CHECKSTACK_SYSTEM_ID: "s1",
+      CHECKSTACK_SYSTEM_NAME: "web-1",
+    });
+  });
+
+  test("emits nothing for an empty run context", () => {
+    expect(buildShellRunContextEnv(undefined)).toEqual({});
+  });
+
+  test("emits only the provided half", () => {
+    const env = buildShellRunContextEnv({
+      system: { id: "s1", name: "web-1" },
+    });
+    expect(env).toEqual({
+      CHECKSTACK_SYSTEM_ID: "s1",
+      CHECKSTACK_SYSTEM_NAME: "web-1",
+    });
+  });
+});
+
+describe("buildCollectorContext", () => {
+  test("always includes config, includes check/system only when present", () => {
+    expect(buildCollectorContext({ config: { threshold: 1 } })).toEqual({
+      config: { threshold: 1 },
+    });
+    expect(
+      buildCollectorContext({
+        config: {},
+        runContext: { check: { id: "c", name: "n", intervalSeconds: 30 } },
+      }),
+    ).toEqual({
+      config: {},
+      check: { id: "c", name: "n", intervalSeconds: 30 },
+    });
+  });
+
+  test("defaults config to an empty object", () => {
+    expect(buildCollectorContext({})).toEqual({ config: {} });
+  });
+});
+
+describe("runCollectorScriptTest — typescript", () => {
+  test("runs with the healthcheck helper + built context", async () => {
+    const { runner, calls } = fakeEsm(async () => ({
+      result: { success: true, value: 0.4 },
+      stdout: "",
+      stderr: "",
+      timedOut: false,
+    }));
+    const out = await runCollectorScriptTest({
+      input: {
+        kind: "typescript",
+        script: "export default { success: true }",
+        config: { threshold: 0.6 },
+        runContext: { check: { id: "c", name: "Load", intervalSeconds: 60 } },
+        timeoutMs: 5000,
+      },
+      deps: { esmRunner: runner },
+    });
+    expect(calls[0]?.helperModuleName).toBe("@checkstack/healthcheck");
+    expect(calls[0]?.helperFunctionName).toBe("defineHealthCheck");
+    expect(calls[0]?.context).toEqual({
+      config: { threshold: 0.6 },
+      check: { id: "c", name: "Load", intervalSeconds: 60 },
+    });
+    expect(out.result).toEqual({ success: true, value: 0.4 });
+    expect(out.error).toBeUndefined();
+  });
+
+  test("surfaces a thrown error and a timeout", async () => {
+    const thrown = await runCollectorScriptTest({
+      input: { kind: "typescript", script: "throw 1", timeoutMs: 1000 },
+      deps: { esmRunner: fakeEsm(async () => ({ error: "boom", stdout: "", stderr: "", timedOut: false })).runner },
+    });
+    expect(thrown.error).toBe("boom");
+
+    const timedOut = await runCollectorScriptTest({
+      input: { kind: "typescript", script: "while(1){}", timeoutMs: 50 },
+      deps: { esmRunner: fakeEsm(async () => ({ stdout: "", stderr: "", timedOut: true })).runner },
+    });
+    expect(timedOut.timedOut).toBe(true);
+    expect(timedOut.error).toBe("Script execution timed out");
+  });
+
+  test("catches an unexpected runner rejection", async () => {
+    const out = await runCollectorScriptTest({
+      input: { kind: "typescript", script: "x", timeoutMs: 1000 },
+      deps: {
+        esmRunner: fakeEsm(async () => {
+          throw new Error("spawn failed");
+        }).runner,
+      },
+    });
+    expect(out.error).toBe("spawn failed");
+  });
+});
+
+describe("runCollectorScriptTest — shell", () => {
+  test("injects CHECKSTACK_* run-context env + merges explicit env", async () => {
+    const { runner, calls } = fakeShell(async () => ({
+      exitCode: 0,
+      stdout: "ok",
+      stderr: "",
+      timedOut: false,
+    }));
+    const out = await runCollectorScriptTest({
+      input: {
+        kind: "shell",
+        script: "echo $CHECKSTACK_CHECK_NAME",
+        runContext: { check: { id: "c", name: "Disk", intervalSeconds: 30 } },
+        env: { EXTRA: "1" },
+        timeoutMs: 3000,
+      },
+      deps: { shellRunner: runner },
+    });
+    const env = calls[0]?.env ?? {};
+    expect(env.CHECKSTACK_CHECK_NAME).toBe("Disk");
+    expect(env.EXTRA).toBe("1");
+    expect(out.exitCode).toBe(0);
+    expect(out.stdout).toBe("ok");
+  });
+
+  test("reports a non-zero exit as an error", async () => {
+    const out = await runCollectorScriptTest({
+      input: { kind: "shell", script: "exit 5", timeoutMs: 1000 },
+      deps: {
+        shellRunner: fakeShell(async () => ({
+          exitCode: 5,
+          stdout: "",
+          stderr: "bad",
+          timedOut: false,
+        })).runner,
+      },
+    });
+    expect(out.exitCode).toBe(5);
+    expect(out.error).toContain("exited with code 5");
+  });
+});
+
+describe("runCollectorScriptTest secret placeholders + overrides (decision 4)", () => {
+  test("injects __SECRET_<NAME>__ placeholders into the collector env by default", async () => {
+    const { runner, calls } = fakeEsm(async (opts) => ({
+      result: opts.env ?? null,
+      stdout: "",
+      stderr: "",
+      timedOut: false,
+    }));
+    await runCollectorScriptTest({
+      input: {
+        kind: "typescript",
+        script: "export default () => ({ success: true })",
+        timeoutMs: 1000,
+        secretEnv: { TOKEN: "${{ secrets.api }}" },
+      },
+      deps: { esmRunner: runner },
+    });
+    expect(calls[0]?.env).toEqual({ TOKEN: "__SECRET_api__" });
+  });
+
+  test("injects + masks a user override (no real resolution)", async () => {
+    const { runner, calls } = fakeShell(async () => ({
+      exitCode: 0,
+      stdout: "value=override-secret",
+      stderr: "",
+      timedOut: false,
+    }));
+    const out = await runCollectorScriptTest({
+      input: {
+        kind: "shell",
+        script: "echo value=$TOKEN",
+        timeoutMs: 1000,
+        secretEnv: { TOKEN: "${{ secrets.api }}" },
+        secretOverrides: { api: "override-secret" },
+      },
+      deps: { shellRunner: runner },
+    });
+    expect(calls[0]?.env?.TOKEN).toBe("override-secret");
+    expect(out.stdout).toBe("value=****");
+    expect(JSON.stringify(out)).not.toContain("override-secret");
+  });
+
+  test("no secretEnv -> no secret env injected (least-privilege)", async () => {
+    const { runner, calls } = fakeEsm(async (opts) => ({
+      result: opts.env ?? null,
+      stdout: "",
+      stderr: "",
+      timedOut: false,
+    }));
+    await runCollectorScriptTest({
+      input: {
+        kind: "typescript",
+        script: "export default () => ({ success: true })",
+        timeoutMs: 1000,
+      },
+      deps: { esmRunner: runner },
+    });
+    expect(calls[0]?.env).toBeUndefined();
+  });
+});

package/src/collector-script-test.ts

@@ -0,0 +1,221 @@
+/**
+ * In-UI testing for health-check collector scripts (the inline-script TS
+ * collector and the shell `script` collector from
+ * `@checkstack/healthcheck-script-backend`).
+ *
+ * Exercises the same sandboxed runners the real collectors use against an
+ * editable sample context, so an operator can click "Run" in the collector
+ * editor and see the result without scheduling a real check execution.
+ *
+ * Mirrors the automation `runScriptTest` design: pure, runner-injectable,
+ * never throws for ordinary script failures (they're returned in the
+ * result), central-only, time-bounded. Healthcheck *replay* from a past
+ * execution is intentionally NOT supported - `health_check_runs` persists
+ * only the result, never the script/config/check/system that produced it,
+ * so a faithful replay cannot be reconstructed. Auto-seed is the only
+ * context source (see the feature plan, open item g).
+ */
+import {
+  defaultEsmScriptRunner,
+  defaultShellScriptRunner,
+  type EsmScriptRunner,
+  type ShellScriptRunner,
+} from "@checkstack/backend-api";
+import { extractErrorMessage } from "@checkstack/common";
+import {
+  buildTestSecretEnv,
+  maskScriptRunOutput,
+} from "@checkstack/secrets-common";
+
+export type CollectorScriptTestKind = "typescript" | "shell";
+
+/** Curated check/system metadata a collector script can read. */
+export interface CollectorTestRunContext {
+  check?: { id: string; name: string; intervalSeconds: number };
+  system?: { id: string; name: string };
+}
+
+export interface CollectorScriptTestInput {
+  kind: CollectorScriptTestKind;
+  script: string;
+  /** Collector config the script reads via `context.config` / its own fields. */
+  config?: Record<string, unknown>;
+  /** Extra env vars for shell collectors (merged over the run-context vars). */
+  env?: Record<string, string>;
+  /**
+   * The collector's declared secret -> env mapping. The test panel NEVER
+   * resolves real secret values (decision 4): each declared env var gets a
+   * `__SECRET_<NAME>__` placeholder, or the user override below.
+   */
+  secretEnv?: Record<string, string>;
+  /** User-supplied per-secret-NAME override values, masked out of the result. */
+  secretOverrides?: Record<string, string>;
+  /** Working directory for shell collectors. */
+  workingDirectory?: string;
+  runContext?: CollectorTestRunContext;
+  timeoutMs: number;
+}
+
+export interface CollectorScriptTestResult {
+  result?: unknown;
+  stdout: string;
+  stderr: string;
+  exitCode?: number;
+  durationMs: number;
+  timedOut: boolean;
+  error?: string;
+}
+
+export interface CollectorScriptTestDeps {
+  esmRunner?: EsmScriptRunner;
+  shellRunner?: ShellScriptRunner;
+  /**
+   * Managed npm-package resolution root, so a TypeScript collector test
+   * resolves the same allowlisted packages the real collector would. Omit
+   * when no packages are configured. Plan §4.1.
+   */
+  resolutionRoot?: string;
+}
+
+/**
+ * Map curated run-context metadata to the reserved `CHECKSTACK_*` env vars
+ * the shell collector exposes. Mirrors `runContextEnv` in
+ * `@checkstack/healthcheck-script-backend` (kept local - we don't import
+ * across plugins). Only emits vars for the parts of the context provided.
+ */
+export function buildShellRunContextEnv(
+  runContext: CollectorTestRunContext | undefined,
+): Record<string, string> {
+  const env: Record<string, string> = {};
+  if (runContext?.check) {
+    env.CHECKSTACK_CHECK_ID = runContext.check.id;
+    env.CHECKSTACK_CHECK_NAME = runContext.check.name;
+    env.CHECKSTACK_CHECK_INTERVAL_SECONDS = String(
+      runContext.check.intervalSeconds,
+    );
+  }
+  if (runContext?.system) {
+    env.CHECKSTACK_SYSTEM_ID = runContext.system.id;
+    env.CHECKSTACK_SYSTEM_NAME = runContext.system.name;
+  }
+  return env;
+}
+
+/**
+ * Build the `globalThis.context` object the inline-script (TS) collector
+ * sees: `{ config, check?, system? }`. Matches the runtime collector so a
+ * test mirrors production.
+ */
+export function buildCollectorContext(
+  input: Pick<CollectorScriptTestInput, "config" | "runContext">,
+): Record<string, unknown> {
+  return {
+    config: input.config ?? {},
+    ...(input.runContext?.check ? { check: input.runContext.check } : {}),
+    ...(input.runContext?.system ? { system: input.runContext.system } : {}),
+  };
+}
+
+/**
+ * Execute a single collector-script test against a sample context. Never
+ * throws for ordinary script failures - those land in the result.
+ */
+export async function runCollectorScriptTest({
+  input,
+  deps = {},
+}: {
+  input: CollectorScriptTestInput;
+  deps?: CollectorScriptTestDeps;
+}): Promise<CollectorScriptTestResult> {
+  const startedAt = Date.now();
+  // Build the test secret env: placeholders by default, user overrides if
+  // given. NO real secret value is resolved in the test path (decision 4).
+  const secretTest = buildTestSecretEnv({
+    secretEnv: input.secretEnv,
+    secretOverrides: input.secretOverrides,
+  });
+  // Mask any user-override value out of the result so it can't round-trip.
+  const mask = (
+    res: CollectorScriptTestResult,
+  ): CollectorScriptTestResult => {
+    const masked = maskScriptRunOutput({
+      output: {
+        result: res.result,
+        stdout: res.stdout,
+        stderr: res.stderr,
+        error: res.error,
+      },
+      values: secretTest.maskValues,
+    });
+    return { ...res, ...masked };
+  };
+
+  try {
+    if (input.kind === "shell") {
+      const runner = deps.shellRunner ?? defaultShellScriptRunner;
+      const res = await runner.run({
+        script: input.script,
+        // Run-context vars, operator env, then the test secret env on top.
+        env: {
+          ...buildShellRunContextEnv(input.runContext),
+          ...input.env,
+          ...secretTest.env,
+        },
+        cwd: input.workingDirectory,
+        timeoutMs: input.timeoutMs,
+      });
+      const durationMs = Date.now() - startedAt;
+      if (res.timedOut) {
+        return mask({
+          stdout: res.stdout,
+          stderr: res.stderr,
+          exitCode: res.exitCode,
+          durationMs,
+          timedOut: true,
+          error: "Script execution timed out",
+        });
+      }
+      return mask({
+        stdout: res.stdout,
+        stderr: res.stderr,
+        exitCode: res.exitCode,
+        durationMs,
+        timedOut: false,
+        error:
+          res.exitCode === 0
+            ? undefined
+            : `Shell script exited with code ${res.exitCode}`,
+      });
+    }
+
+    const runner = deps.esmRunner ?? defaultEsmScriptRunner;
+    const res = await runner.run({
+      script: input.script,
+      context: buildCollectorContext(input),
+      timeoutMs: input.timeoutMs,
+      helperModuleName: "@checkstack/healthcheck",
+      helperFunctionName: "defineHealthCheck",
+      ...(Object.keys(secretTest.env).length > 0
+        ? { env: secretTest.env }
+        : {}),
+      ...(deps.resolutionRoot ? { resolutionRoot: deps.resolutionRoot } : {}),
+    });
+    const durationMs = Date.now() - startedAt;
+    return mask({
+      result: res.result,
+      stdout: res.stdout,
+      stderr: res.stderr,
+      durationMs,
+      timedOut: res.timedOut,
+      error: res.timedOut ? "Script execution timed out" : res.error,
+    });
+  } catch (error) {
+    return mask({
+      stdout: "",
+      stderr: "",
+      durationMs: Date.now() - startedAt,
+      timedOut: false,
+      error: extractErrorMessage(error),
+    });
+  }
+}