@checkstack/automation-backend 0.2.0 → 0.3.0

package/src/script-test.test.ts

@@ -0,0 +1,386 @@
+import { describe, expect, test } from "bun:test";
+import type {
+  EsmScriptRunner,
+  ShellScriptRunner,
+} from "@checkstack/backend-api";
+import {
+  buildScriptContext,
+  buildTestSecretEnv,
+  runScriptTest,
+  type ScriptTestContext,
+} from "./script-test";
+
+function fakeEsmRunner(
+  impl: EsmScriptRunner["run"],
+): { runner: EsmScriptRunner; calls: Parameters<EsmScriptRunner["run"]>[0][] } {
+  const calls: Parameters<EsmScriptRunner["run"]>[0][] = [];
+  return {
+    calls,
+    runner: {
+      run: (options) => {
+        calls.push(options);
+        return impl(options);
+      },
+    },
+  };
+}
+
+function fakeShellRunner(
+  impl: ShellScriptRunner["run"],
+): {
+  runner: ShellScriptRunner;
+  calls: Parameters<ShellScriptRunner["run"]>[0][];
+} {
+  const calls: Parameters<ShellScriptRunner["run"]>[0][] = [];
+  return {
+    calls,
+    runner: {
+      run: (options) => {
+        calls.push(options);
+        return impl(options);
+      },
+    },
+  };
+}
+
+describe("buildScriptContext", () => {
+  test("fills defaults for an empty sample", () => {
+    const ctx = buildScriptContext(undefined);
+    expect(ctx).toEqual({
+      trigger: { event: "", payload: {} },
+      artifacts: {},
+      var: {},
+      automation: {
+        runId: "test-run",
+        automationId: "test-automation",
+        contextKey: null,
+      },
+    });
+  });
+
+  test("passes through provided trigger / artifacts / var and includes repeat only when present", () => {
+    const sample: ScriptTestContext = {
+      trigger: { event: "incident.created", payload: { id: "INC-1" } },
+      artifacts: { jira_issue: { key: "PROJ-1" } },
+      var: { count: 3 },
+      repeat: { index: 2, item: { name: "a" } },
+    };
+    const ctx = buildScriptContext(sample);
+    expect(ctx.trigger).toEqual({
+      event: "incident.created",
+      payload: { id: "INC-1" },
+    });
+    expect(ctx.artifacts).toEqual({ jira_issue: { key: "PROJ-1" } });
+    expect(ctx.var).toEqual({ count: 3 });
+    expect(ctx.repeat).toEqual({ index: 2, item: { name: "a" } });
+  });
+
+  test("omits repeat when not in the sample", () => {
+    const ctx = buildScriptContext({ trigger: { event: "x" } });
+    expect("repeat" in ctx).toBe(false);
+  });
+});
+
+describe("runScriptTest — typescript", () => {
+  test("invokes the ESM runner with the built context + helper module", async () => {
+    const { runner, calls } = fakeEsmRunner(async () => ({
+      result: { id: "ok" },
+      stdout: "log line",
+      stderr: "",
+      timedOut: false,
+    }));
+
+    const out = await runScriptTest({
+      input: {
+        kind: "typescript",
+        script: "export default { id: 'ok' }",
+        context: { trigger: { event: "e", payload: { a: 1 } } },
+        timeoutMs: 5000,
+      },
+      deps: { esmRunner: runner },
+    });
+
+    expect(calls).toHaveLength(1);
+    expect(calls[0]?.helperModuleName).toBe("@checkstack/integration");
+    expect(calls[0]?.helperFunctionName).toBe("defineIntegration");
+    expect(calls[0]?.timeoutMs).toBe(5000);
+    expect(out.result).toEqual({ id: "ok" });
+    expect(out.stdout).toBe("log line");
+    expect(out.error).toBeUndefined();
+    expect(out.timedOut).toBe(false);
+    expect(out.durationMs).toBeGreaterThanOrEqual(0);
+  });
+
+  test("surfaces a thrown-script error", async () => {
+    const { runner } = fakeEsmRunner(async () => ({
+      error: "boom",
+      stack: "Error: boom",
+      stdout: "",
+      stderr: "",
+      timedOut: false,
+    }));
+
+    const out = await runScriptTest({
+      input: { kind: "typescript", script: "throw new Error('boom')", timeoutMs: 1000 },
+      deps: { esmRunner: runner },
+    });
+    expect(out.error).toBe("boom");
+  });
+
+  test("reports timeout as an error", async () => {
+    const { runner } = fakeEsmRunner(async () => ({
+      stdout: "",
+      stderr: "",
+      timedOut: true,
+      error: "Script execution timed out",
+    }));
+
+    const out = await runScriptTest({
+      input: { kind: "typescript", script: "while(true){}", timeoutMs: 100 },
+      deps: { esmRunner: runner },
+    });
+    expect(out.timedOut).toBe(true);
+    expect(out.error).toBe("Script execution timed out");
+  });
+
+  test("catches an unexpected runner rejection instead of throwing", async () => {
+    const { runner } = fakeEsmRunner(async () => {
+      throw new Error("spawn failed");
+    });
+    const out = await runScriptTest({
+      input: { kind: "typescript", script: "x", timeoutMs: 1000 },
+      deps: { esmRunner: runner },
+    });
+    expect(out.error).toBe("spawn failed");
+  });
+});
+
+describe("runScriptTest — shell", () => {
+  test("flattens the sample context into CHECKSTACK_* env and merges explicit env", async () => {
+    const { runner, calls } = fakeShellRunner(async () => ({
+      exitCode: 0,
+      stdout: "hello",
+      stderr: "",
+      timedOut: false,
+    }));
+
+    const out = await runScriptTest({
+      input: {
+        kind: "shell",
+        script: "echo $CHECKSTACK_TRIGGER_PAYLOAD_ID",
+        context: { trigger: { event: "e", payload: { id: "INC-9" } } },
+        env: { EXTRA: "1" },
+        timeoutMs: 3000,
+      },
+      deps: { shellRunner: runner },
+    });
+
+    expect(calls).toHaveLength(1);
+    const env = calls[0]?.env ?? {};
+    expect(env.CHECKSTACK_TRIGGER_EVENT).toBe("e");
+    expect(env.CHECKSTACK_TRIGGER_PAYLOAD_ID).toBe("INC-9");
+    expect(env.EXTRA).toBe("1");
+    expect(out.exitCode).toBe(0);
+    expect(out.stdout).toBe("hello");
+    expect(out.error).toBeUndefined();
+  });
+
+  test("explicit env wins on key collision", async () => {
+    const { runner, calls } = fakeShellRunner(async () => ({
+      exitCode: 0,
+      stdout: "",
+      stderr: "",
+      timedOut: false,
+    }));
+    await runScriptTest({
+      input: {
+        kind: "shell",
+        script: "true",
+        context: { trigger: { event: "from-context" } },
+        env: { CHECKSTACK_TRIGGER_EVENT: "from-env" },
+        timeoutMs: 1000,
+      },
+      deps: { shellRunner: runner },
+    });
+    expect(calls[0]?.env?.CHECKSTACK_TRIGGER_EVENT).toBe("from-env");
+  });
+
+  test("reports a non-zero exit code as an error", async () => {
+    const { runner } = fakeShellRunner(async () => ({
+      exitCode: 2,
+      stdout: "",
+      stderr: "nope",
+      timedOut: false,
+    }));
+    const out = await runScriptTest({
+      input: { kind: "shell", script: "exit 2", timeoutMs: 1000 },
+      deps: { shellRunner: runner },
+    });
+    expect(out.exitCode).toBe(2);
+    expect(out.error).toContain("exited with code 2");
+  });
+
+  test("reports a shell timeout", async () => {
+    const { runner } = fakeShellRunner(async () => ({
+      exitCode: -1,
+      stdout: "",
+      stderr: "Script execution timed out",
+      timedOut: true,
+    }));
+    const out = await runScriptTest({
+      input: { kind: "shell", script: "sleep 100", timeoutMs: 50 },
+      deps: { shellRunner: runner },
+    });
+    expect(out.timedOut).toBe(true);
+    expect(out.error).toBe("Script execution timed out");
+  });
+});
+
+describe("runScriptTest secret masking (leak guard)", () => {
+  test("redacts run-scoped secret values from a TS test result/stdout/stderr", async () => {
+    const { runner } = fakeEsmRunner(async () => ({
+      result: { token: "gh_injectedSecret999" },
+      stdout: "echoing gh_injectedSecret999 to stdout",
+      stderr: "warn: gh_injectedSecret999 seen",
+      timedOut: false,
+    }));
+    const out = await runScriptTest({
+      input: { kind: "typescript", script: "export default () => {}", timeoutMs: 1000 },
+      deps: { esmRunner: runner, maskValues: ["gh_injectedSecret999"] },
+    });
+    expect(out.stdout).toBe("echoing **** to stdout");
+    expect(out.stderr).toBe("warn: **** seen");
+    expect(out.result).toEqual({ token: "****" });
+    expect(JSON.stringify(out)).not.toContain("gh_injectedSecret999");
+  });
+
+  test("redacts secret values from shell test stdout/stderr", async () => {
+    const { runner } = fakeShellRunner(async () => ({
+      exitCode: 0,
+      stdout: "$ echo db-password-456",
+      stderr: "",
+      timedOut: false,
+    }));
+    const out = await runScriptTest({
+      input: { kind: "shell", script: "echo $X", timeoutMs: 1000 },
+      deps: { shellRunner: runner, maskValues: ["db-password-456"] },
+    });
+    expect(out.stdout).toBe("$ echo ****");
+  });
+
+  test("is a no-op when no mask values are provided (Phase 1 default)", async () => {
+    const { runner } = fakeEsmRunner(async () => ({
+      result: undefined,
+      stdout: "no secrets here",
+      stderr: "",
+      timedOut: false,
+    }));
+    const out = await runScriptTest({
+      input: { kind: "typescript", script: "export default () => {}", timeoutMs: 1000 },
+      deps: { esmRunner: runner },
+    });
+    expect(out.stdout).toBe("no secrets here");
+  });
+});
+
+describe("runScriptTest secret placeholders + overrides (decision 4)", () => {
+  test("injects __SECRET_<NAME>__ placeholders by default (no real resolution)", async () => {
+    const { runner, calls } = fakeEsmRunner(async (opts) => ({
+      result: opts.env ?? null,
+      stdout: "",
+      stderr: "",
+      timedOut: false,
+    }));
+    const out = await runScriptTest({
+      input: {
+        kind: "typescript",
+        script: "export default () => {}",
+        timeoutMs: 1000,
+        secretEnv: { API_TOKEN: "${{ secrets.jira_token }}" },
+      },
+      deps: { esmRunner: runner },
+    });
+    // The injected env is the placeholder, never a real value.
+    expect(calls[0].env).toEqual({ API_TOKEN: "__SECRET_jira_token__" });
+    expect(out.result).toEqual({ API_TOKEN: "__SECRET_jira_token__" });
+  });
+
+  test("injects a user override instead of the placeholder", async () => {
+    const { runner, calls } = fakeShellRunner(async () => ({
+      exitCode: 0,
+      stdout: "ran",
+      stderr: "",
+      timedOut: false,
+    }));
+    await runScriptTest({
+      input: {
+        kind: "shell",
+        script: "echo $DB",
+        timeoutMs: 1000,
+        secretEnv: { DB: "${{ secrets.db_pass }}" },
+        secretOverrides: { db_pass: "my-test-override" },
+      },
+      deps: { shellRunner: runner },
+    });
+    expect(calls[0].env?.DB).toBe("my-test-override");
+  });
+
+  test("masks the user-override value out of the result", async () => {
+    const { runner } = fakeEsmRunner(async () => ({
+      result: { leaked: "my-test-override" },
+      stdout: "echo my-test-override",
+      stderr: "",
+      timedOut: false,
+    }));
+    const out = await runScriptTest({
+      input: {
+        kind: "typescript",
+        script: "export default () => {}",
+        timeoutMs: 1000,
+        secretEnv: { TOKEN: "${{ secrets.tok }}" },
+        secretOverrides: { tok: "my-test-override" },
+      },
+      deps: { esmRunner: runner },
+    });
+    expect(out.stdout).toBe("echo ****");
+    expect(out.result).toEqual({ leaked: "****" });
+    expect(JSON.stringify(out)).not.toContain("my-test-override");
+  });
+
+  test("no secretEnv -> no secret env injected (least-privilege)", async () => {
+    const { runner, calls } = fakeShellRunner(async () => ({
+      exitCode: 0,
+      stdout: "",
+      stderr: "",
+      timedOut: false,
+    }));
+    await runScriptTest({
+      input: { kind: "shell", script: "true", timeoutMs: 1000 },
+      deps: { shellRunner: runner },
+    });
+    // Only the $CHECKSTACK_* context env is present; no secret-derived keys.
+    const envKeys = Object.keys(calls[0].env ?? {});
+    expect(envKeys.some((k) => !k.startsWith("CHECKSTACK_"))).toBe(false);
+  });
+});
+
+describe("buildTestSecretEnv", () => {
+  test("placeholder when no override; override when given; masks overrides", () => {
+    const { env, maskValues } = buildTestSecretEnv({
+      secretEnv: {
+        A: "${{ secrets.alpha }}",
+        B: "${{ secrets.beta }}",
+      },
+      secretOverrides: { beta: "real-override" },
+    });
+    expect(env).toEqual({
+      A: "__SECRET_alpha__",
+      B: "real-override",
+    });
+    expect(maskValues).toEqual(["real-override"]);
+  });
+
+  test("empty when no secretEnv declared", () => {
+    expect(buildTestSecretEnv({})).toEqual({ env: {}, maskValues: [] });
+  });
+});

package/src/script-test.ts

@@ -0,0 +1,258 @@
+/**
+ * In-UI script testing for automation `run_script` / `run_shell` actions.
+ *
+ * This is the backend half of Feature 2 (test scripts in the UI). It
+ * exercises the **same** sandboxed runners the real actions use
+ * (`defaultEsmScriptRunner` for TypeScript/JavaScript, `defaultShellScriptRunner`
+ * for shell) against an editable, user-supplied sample context — so an
+ * operator can click "Run" in the editor and see the result without
+ * triggering a whole automation.
+ *
+ * Security: the test runs on the central backend with the same subprocess
+ * isolation + curated `SAFE_ENV_VARS` env as production. Authoring a
+ * script already lets the user execute code, so no new privilege is
+ * granted; the endpoint is `manage`-gated and time-bounded.
+ *
+ * The module is deliberately pure (no DB, no oRPC) and takes injectable
+ * runners so it can be unit-tested without spawning a real subprocess.
+ */
+import {
+  defaultEsmScriptRunner,
+  defaultShellScriptRunner,
+  type EsmScriptRunner,
+  type ShellScriptRunner,
+} from "@checkstack/backend-api";
+import { extractErrorMessage } from "@checkstack/common";
+import {
+  maskScriptRunOutput,
+  buildTestSecretEnv,
+} from "@checkstack/secrets-common";
+import { flattenScopeToShellEnv } from "./script-test-shell-env";
+
+// =============================================================================
+// PUBLIC TYPES
+// =============================================================================
+
+export type ScriptTestKind = "typescript" | "shell";
+
+/**
+ * Sample run context an operator edits in the test panel. Mirrors the
+ * shape `run_script` exposes as `globalThis.context` and `run_shell`
+ * flattens into `$CHECKSTACK_*` env vars. Every field is optional so a
+ * partial sample still runs.
+ */
+export interface ScriptTestContext {
+  trigger?: {
+    event?: string;
+    payload?: Record<string, unknown>;
+  };
+  artifacts?: Record<string, unknown>;
+  var?: Record<string, unknown>;
+  repeat?: { index: number; item?: unknown };
+}
+
+export interface ScriptTestInput {
+  kind: ScriptTestKind;
+  script: string;
+  /** Editable sample context (auto-seeded in the UI). */
+  context?: ScriptTestContext;
+  /** Extra env vars for shell tests (merged over the flattened context). */
+  env?: Record<string, string>;
+  /**
+   * The script's declared secret -> env mapping
+   * (`{ ENV_NAME: "${{ secrets.NAME }}" }`). The test panel NEVER resolves
+   * real secret values (decision 4): each declared env var is injected with
+   * a named placeholder (`__SECRET_<NAME>__`) by default, or the user's
+   * override (see `secretOverrides`) for a realistic run.
+   */
+  secretEnv?: Record<string, string>;
+  /**
+   * Optional user-supplied per-secret-NAME override values (keyed by the
+   * `${{ secrets.NAME }}` name). Injected instead of the placeholder, and
+   * masked out of the result so an override can't round-trip unmasked.
+   */
+  secretOverrides?: Record<string, string>;
+  /** Working directory for shell tests. */
+  workingDirectory?: string;
+  timeoutMs: number;
+}
+
+export interface ScriptTestResult {
+  /** Return value of a TypeScript test (undefined for shell). */
+  result?: unknown;
+  stdout: string;
+  stderr: string;
+  /** Exit code of a shell test (undefined for TypeScript). */
+  exitCode?: number;
+  durationMs: number;
+  timedOut: boolean;
+  /** Populated when the script threw / failed to load / exited non-zero. */
+  error?: string;
+}
+
+export interface ScriptTestDeps {
+  esmRunner?: EsmScriptRunner;
+  shellRunner?: ShellScriptRunner;
+  /**
+   * Managed npm-package resolution root, so a TypeScript test resolves the
+   * same allowlisted packages the real `run_script` action would. Omit when
+   * no packages are configured (today's behavior). Plan §4.1: "with the
+   * managed resolutionRoot once Feature 1 lands."
+   */
+  resolutionRoot?: string;
+  /**
+   * Extra secret VALUES to redact (Jenkins-style) from the test result, on
+   * top of those derived from the input's `secretOverrides`. Normally
+   * unused — the test path masks the user overrides automatically.
+   */
+  maskValues?: Iterable<string>;
+}
+
+// `buildTestSecretEnv` / `secretTestPlaceholder` are the shared, pure
+// secret-test helpers from `@checkstack/secrets-common` (re-exported so the
+// existing import path keeps working).
+export { buildTestSecretEnv, secretTestPlaceholder } from "@checkstack/secrets-common";
+
+// =============================================================================
+// CONTEXT NORMALISATION
+// =============================================================================
+
+/**
+ * Build the `globalThis.context` object a `run_script` action sees, from
+ * the editable sample. Keeps the same key names (`trigger`, `artifacts`,
+ * `var`, `automation`, optional `repeat`) so testing matches runtime.
+ */
+export function buildScriptContext(
+  context: ScriptTestContext | undefined,
+): Record<string, unknown> {
+  return {
+    trigger: {
+      event: context?.trigger?.event ?? "",
+      payload: context?.trigger?.payload ?? {},
+    },
+    artifacts: context?.artifacts ?? {},
+    var: context?.var ?? {},
+    ...(context?.repeat ? { repeat: context.repeat } : {}),
+    // A test run has no real automation identity; surface stable
+    // placeholder values so `context.automation.runId` is non-null.
+    automation: {
+      runId: "test-run",
+      automationId: "test-automation",
+      contextKey: null,
+    },
+  };
+}
+
+// =============================================================================
+// RUN
+// =============================================================================
+
+/**
+ * Execute a single test run of a script against a sample context.
+ *
+ * Never throws for ordinary script failures (syntax error, thrown error,
+ * non-zero exit, timeout) — those are returned in the {@link ScriptTestResult}
+ * so the UI can render them. Only truly unexpected infrastructure errors
+ * propagate, and even those are caught and surfaced as `error`.
+ */
+export async function runScriptTest({
+  input,
+  deps = {},
+}: {
+  input: ScriptTestInput;
+  deps?: ScriptTestDeps;
+}): Promise<ScriptTestResult> {
+  const startedAt = Date.now();
+  // Build the test secret env: placeholders by default, user overrides if
+  // given. NO real secret value is ever resolved in the test path.
+  const secretTest = buildTestSecretEnv({
+    secretEnv: input.secretEnv,
+    secretOverrides: input.secretOverrides,
+  });
+  // Universal leak masking at the output boundary: redact any user-override
+  // secret value (plus any extra `deps.maskValues`) out of
+  // result/stdout/stderr/error before it is returned to the browser, so even
+  // an override can't round-trip to the surface unmasked.
+  const maskValues = [...secretTest.maskValues, ...(deps.maskValues ?? [])];
+  const mask = (
+    res: ScriptTestResult,
+  ): ScriptTestResult => {
+    const masked = maskScriptRunOutput({
+      output: {
+        result: res.result,
+        stdout: res.stdout,
+        stderr: res.stderr,
+        error: res.error,
+      },
+      values: maskValues,
+    });
+    return { ...res, ...masked };
+  };
+
+  try {
+    if (input.kind === "shell") {
+      const runner = deps.shellRunner ?? defaultShellScriptRunner;
+      const flattened = flattenScopeToShellEnv(input.context);
+      const res = await runner.run({
+        script: input.script,
+        // Operator env, then the test secret env (placeholders/overrides)
+        // on top so a declared secret env name wins.
+        env: { ...flattened, ...input.env, ...secretTest.env },
+        cwd: input.workingDirectory,
+        timeoutMs: input.timeoutMs,
+      });
+      const durationMs = Date.now() - startedAt;
+      if (res.timedOut) {
+        return mask({
+          stdout: res.stdout,
+          stderr: res.stderr,
+          exitCode: res.exitCode,
+          durationMs,
+          timedOut: true,
+          error: "Script execution timed out",
+        });
+      }
+      return mask({
+        stdout: res.stdout,
+        stderr: res.stderr,
+        exitCode: res.exitCode,
+        durationMs,
+        timedOut: false,
+        error:
+          res.exitCode === 0
+            ? undefined
+            : `Shell script exited with code ${res.exitCode}`,
+      });
+    }
+
+    const runner = deps.esmRunner ?? defaultEsmScriptRunner;
+    const res = await runner.run({
+      script: input.script,
+      context: buildScriptContext(input.context),
+      timeoutMs: input.timeoutMs,
+      helperModuleName: "@checkstack/integration",
+      helperFunctionName: "defineIntegration",
+      ...(Object.keys(secretTest.env).length > 0
+        ? { env: secretTest.env }
+        : {}),
+      ...(deps.resolutionRoot ? { resolutionRoot: deps.resolutionRoot } : {}),
+    });
+    const durationMs = Date.now() - startedAt;
+    return mask({
+      result: res.result,
+      stdout: res.stdout,
+      stderr: res.stderr,
+      durationMs,
+      timedOut: res.timedOut,
+      error: res.timedOut ? "Script execution timed out" : res.error,
+    });
+  } catch (error) {
+    return mask({
+      stdout: "",
+      stderr: "",
+      durationMs: Date.now() - startedAt,
+      timedOut: false,
+      error: extractErrorMessage(error),
+    });
+  }
+}

package/src/trigger-registry.ts

@@ -62,8 +62,10 @@ export function createTriggerRegistry(): TriggerRegistry {
         payloadSchema: definition.payloadSchema,
         configSchema: definition.configSchema,
         contextKey: definition.contextKey,
+        contextKeyLabel: definition.contextKeyLabel,
         hook: definition.hook,
         setup: definition.setup,
+        evaluateConfig: definition.evaluateConfig,
         qualifiedId,
         ownerPluginId: pluginMetadata.pluginId,
         payloadJsonSchema,

package/src/validate-definition.test.ts

@@ -88,6 +88,7 @@ function baseDefinition(
       },
     ],
     mode: "single",
+    concurrency_scope: "automation",
     max_runs: 1,
     ...overrides,
   };

package/tsconfig.json

@@ -7,6 +7,9 @@
     {
       "path": "../automation-common"
     },
+    {
+      "path": "../backend"
+    },
     {
       "path": "../backend-api"
     },
@@ -19,6 +22,18 @@
     {
       "path": "../drizzle-helper"
     },
+    {
+      "path": "../gitops-backend"
+    },
+    {
+      "path": "../gitops-common"
+    },
+    {
+      "path": "../healthcheck-common"
+    },
+    {
+      "path": "../integration-backend"
+    },
     {
       "path": "../integration-common"
     },
@@ -28,6 +43,15 @@
     {
       "path": "../queue-api"
     },
+    {
+      "path": "../script-packages-backend"
+    },
+    {
+      "path": "../secrets-backend"
+    },
+    {
+      "path": "../secrets-common"
+    },
     {
       "path": "../signal-common"
     },