@checkstack/automation-backend 0.2.0 → 0.3.0

package/src/dispatch/run-secret-registry.ts

@@ -0,0 +1,247 @@
+import type { ServiceRef } from "@checkstack/backend-api";
+import {
+  maskSecrets,
+  maskSecretsDeep,
+} from "@checkstack/secrets-common";
+
+/**
+ * Run-scoped accumulation of every secret VALUE resolved during a dispatch
+ * run, so the run-state persistence layer can mask it out of step / run
+ * output before it is written (and therefore before any DTO / run-detail
+ * page can read it). Jenkins-style, by-value, least-privilege: a run's set
+ * holds ONLY the values that run actually resolved.
+ *
+ * Capture happens by wrapping the run's `getService` (see
+ * {@link wrapGetServiceForRun}) so that resolving the secret resolver or
+ * connection store during the run registers the resolved values here. The
+ * resolved values themselves stay in memory only — this registry is never
+ * persisted, and a run's entry is dropped when the run reaches a terminal
+ * state.
+ */
+export interface RunSecretRegistry {
+  /** Register resolved secret values for a run (deduped, in memory). */
+  register(runId: string, values: Iterable<string>): void;
+  /**
+   * Associate a step with its run, so step writes (which carry only a
+   * stepId) can find the run's mask set without threading runId through
+   * every `updateStep` caller.
+   */
+  linkStep(stepId: string, runId: string): void;
+  /** Mask every registered value out of `text` for a run. No-op if none. */
+  maskText(runId: string, text: string): string;
+  /** Mask every registered value out of a JSON-like payload for a run. */
+  maskDeep(runId: string, value: unknown): unknown;
+  /** Mask `text` using the run that owns `stepId`. */
+  maskTextForStep(stepId: string, text: string): string;
+  /** Mask a payload using the run that owns `stepId`. */
+  maskDeepForStep(stepId: string, value: unknown): unknown;
+  /** Drop a run's accumulated values + its step links (terminal status). */
+  drop(runId: string): void;
+}
+
+export function createRunSecretRegistry(): RunSecretRegistry {
+  const byRun = new Map<string, Set<string>>();
+  const stepToRun = new Map<string, string>();
+
+  const maskTextForRun = (runId: string, text: string): string => {
+    const set = byRun.get(runId);
+    if (!set || set.size === 0) return text;
+    return maskSecrets({ text, values: set });
+  };
+  const maskDeepForRun = (runId: string, value: unknown): unknown => {
+    const set = byRun.get(runId);
+    if (!set || set.size === 0) return value;
+    return maskSecretsDeep({ value, values: set });
+  };
+
+  return {
+    register(runId, values) {
+      let set = byRun.get(runId);
+      if (!set) {
+        set = new Set<string>();
+        byRun.set(runId, set);
+      }
+      for (const v of values) {
+        if (typeof v === "string" && v.length > 0) set.add(v);
+      }
+    },
+
+    linkStep(stepId, runId) {
+      stepToRun.set(stepId, runId);
+    },
+
+    maskText: maskTextForRun,
+    maskDeep: maskDeepForRun,
+
+    maskTextForStep(stepId, text) {
+      const runId = stepToRun.get(stepId);
+      return runId ? maskTextForRun(runId, text) : text;
+    },
+    maskDeepForStep(stepId, value) {
+      const runId = stepToRun.get(stepId);
+      return runId ? maskDeepForRun(runId, value) : value;
+    },
+
+    drop(runId) {
+      byRun.delete(runId);
+      for (const [stepId, rid] of stepToRun) {
+        if (rid === runId) stepToRun.delete(stepId);
+      }
+    },
+  };
+}
+
+/**
+ * Wrap a run's `getService` so resolving the secret resolver or the
+ * connection store registers every resolved value into the
+ * {@link RunSecretRegistry} for this run. This is the single capture point
+ * for run-wide masking: any action that resolves secrets (script
+ * `secretEnv`, a `${{ secrets.NAME }}` template, or a provider connection
+ * credential) contributes its resolved values to the run's mask set,
+ * least-privilege.
+ *
+ * `refs` carries the ids to intercept (the resolver + connection-store
+ * refs); other services pass through untouched. Ids are passed in (rather
+ * than imported) to avoid a hard dependency cycle on the secrets / integration
+ * packages from the dispatch core.
+ */
+export function wrapGetServiceForRun(input: {
+  getService: <T>(ref: ServiceRef<T>) => Promise<T>;
+  runId: string;
+  registry: RunSecretRegistry;
+  /** Service-ref id of the secret resolver (`secretResolverRef`). */
+  resolverRefId: string;
+  /** Service-ref id of the connection store (`connectionStoreRef`). */
+  connectionStoreRefId: string;
+}): <T>(ref: ServiceRef<T>) => Promise<T> {
+  const { getService, runId, registry, resolverRefId, connectionStoreRefId } =
+    input;
+
+  return async <T>(ref: ServiceRef<T>): Promise<T> => {
+    const service = await getService(ref);
+    if (ref.id === resolverRefId) {
+      return wrapResolver(service, runId, registry);
+    }
+    if (ref.id === connectionStoreRefId) {
+      return wrapConnectionStore(service, runId, registry);
+    }
+    return service;
+  };
+}
+
+// ─── Service proxies that register resolved values ─────────────────────────
+
+// Minimal structural shapes of the methods that return secret values. We
+// avoid importing the concrete service types to keep the dispatch core free
+// of a dependency on the secrets / integration packages.
+
+interface ResolverLike {
+  resolveSecret(input: { name: string }): Promise<string>;
+  resolveForRun(input: { secretEnv: Record<string, string> }): Promise<{
+    env: Record<string, string>;
+    masking: unknown;
+  }>;
+  resolveBySchema(input: {
+    value: unknown;
+    schema: unknown;
+  }): Promise<{ resolved: unknown; warnings: string[] }>;
+}
+
+interface ConnectionStoreLike {
+  getConnectionWithCredentials(
+    connectionId: string,
+  ): Promise<{ config: Record<string, unknown> } | undefined>;
+}
+
+function hasResolverShape(s: unknown): s is ResolverLike {
+  return (
+    typeof s === "object" &&
+    s !== null &&
+    typeof (s as ResolverLike).resolveForRun === "function" &&
+    typeof (s as ResolverLike).resolveSecret === "function"
+  );
+}
+
+function hasConnectionStoreShape(s: unknown): s is ConnectionStoreLike {
+  return (
+    typeof s === "object" &&
+    s !== null &&
+    typeof (s as ConnectionStoreLike).getConnectionWithCredentials ===
+      "function"
+  );
+}
+
+/** Collect every string leaf in a JSON-like value (resolved cred values). */
+function collectStrings(value: unknown, out: string[]): void {
+  if (typeof value === "string") {
+    out.push(value);
+  } else if (Array.isArray(value)) {
+    for (const v of value) collectStrings(v, out);
+  } else if (value !== null && typeof value === "object") {
+    for (const v of Object.values(value)) collectStrings(v, out);
+  }
+}
+
+function wrapResolver<T>(service: T, runId: string, registry: RunSecretRegistry): T {
+  if (!hasResolverShape(service)) return service;
+  const inner = service as ResolverLike & Record<string, unknown>;
+  // Proxy that intercepts ONLY the three value-returning methods (to
+  // register resolved secrets) and forwards every other method / property
+  // untouched via Reflect.get. Mirrors `wrapConnectionStore` — a hand-built
+  // literal would silently drop any resolver method we didn't re-declare.
+  return new Proxy(inner, {
+    get(target, prop, receiver) {
+      if (prop === "resolveSecret") {
+        return async (input: { name: string }) => {
+          const value = await inner.resolveSecret(input);
+          registry.register(runId, [value]);
+          return value;
+        };
+      }
+      if (prop === "resolveForRun") {
+        return async (input: { secretEnv: Record<string, string> }) => {
+          const result = await inner.resolveForRun(input);
+          registry.register(runId, Object.values(result.env));
+          return result;
+        };
+      }
+      if (prop === "resolveBySchema") {
+        return async (input: { value: unknown; schema: unknown }) => {
+          const result = await inner.resolveBySchema(input);
+          const strings: string[] = [];
+          collectStrings(result.resolved, strings);
+          registry.register(runId, strings);
+          return result;
+        };
+      }
+      return Reflect.get(target, prop, receiver);
+    },
+  }) as unknown as T;
+}
+
+function wrapConnectionStore<T>(
+  service: T,
+  runId: string,
+  registry: RunSecretRegistry,
+): T {
+  if (!hasConnectionStoreShape(service)) return service;
+  const inner = service as ConnectionStoreLike & Record<string, unknown>;
+  // Preserve all the store's other methods; only the credential resolver
+  // registers values.
+  return new Proxy(inner, {
+    get(target, prop, receiver) {
+      if (prop === "getConnectionWithCredentials") {
+        return async (connectionId: string) => {
+          const conn = await inner.getConnectionWithCredentials(connectionId);
+          if (conn) {
+            const strings: string[] = [];
+            collectStrings(conn.config, strings);
+            registry.register(runId, strings);
+          }
+          return conn;
+        };
+      }
+      return Reflect.get(target, prop, receiver);
+    },
+  }) as unknown as T;
+}

package/src/dispatch/run-state-masking.test.ts

@@ -0,0 +1,376 @@
+import { describe, it, expect } from "bun:test";
+import type { ServiceRef } from "@checkstack/backend-api";
+import { AutomationDefinitionSchema } from "@checkstack/automation-common";
+import { createRunStore } from "./run-state";
+import { createRunStateStore } from "./run-state-store";
+import { createArtifactStore } from "../artifact-store";
+import {
+  createRunSecretRegistry,
+  wrapGetServiceForRun,
+} from "./run-secret-registry";
+import { reseedRunSecretRegistry } from "./reseed-run-secrets";
+import type { AdvisoryLockService } from "@checkstack/backend-api";
+
+/**
+ * Asserts the run-state persistence CHOKE POINT masks resolved secret
+ * values out of step / run output before they are written — so any
+ * downstream read / DTO / run-detail page is safe by construction.
+ *
+ * Uses a capturing fake `db` (the established boundary avoids real-DB
+ * tests); we assert on the values the store hands to the DB layer.
+ */
+
+interface Captured {
+  inserts: Array<{ table: string; values: Record<string, unknown> }>;
+  updates: Array<Record<string, unknown>>;
+}
+
+function capturingDb(captured: Captured) {
+  // The store calls: insert(table).values(v).returning() and
+  // update(table).set(v).where(). We capture v and synthesize ids.
+  let lastInsertTable = "";
+  const db = {
+    insert(table: { [k: string]: unknown }) {
+      lastInsertTable = String(
+        (table as { _?: { name?: string } })._?.name ?? "table",
+      );
+      return {
+        values(v: Record<string, unknown>) {
+          captured.inserts.push({ table: lastInsertTable, values: v });
+          return {
+            returning() {
+              return Promise.resolve([{ id: "generated-id", ...v }]);
+            },
+            onConflictDoUpdate() {
+              return Promise.resolve();
+            },
+          };
+        },
+      };
+    },
+    update() {
+      return {
+        set(v: Record<string, unknown>) {
+          captured.updates.push(v);
+          return {
+            where() {
+              return Promise.resolve();
+            },
+          };
+        },
+      };
+    },
+  };
+  // The store's typed signature wants a SafeDatabase; this fake implements
+  // only the chains exercised here.
+  return db as unknown as Parameters<typeof createRunStore>[0];
+}
+
+describe("run-state masking choke point", () => {
+  it("masks a step resultPayload + errorMessage that contain a resolved secret before persist", async () => {
+    const captured: Captured = { inserts: [], updates: [] };
+    const registry = createRunSecretRegistry();
+    const store = createRunStore(capturingDb(captured), undefined, registry);
+
+    // A run resolved this credential during execution.
+    registry.register("run-1", ["resolved-cred-XYZ"]);
+
+    const stepId = await store.createStep({
+      runId: "run-1",
+      actionPath: "actions[0]",
+      actionId: "a1",
+      actionKind: "action",
+      providerActionId: "integration-jira.create_issue",
+    });
+
+    // A provider HTTP error embedding the credential + a payload echoing it.
+    await store.updateStep(stepId, {
+      status: "failed",
+      errorMessage: "401 from Jira using token resolved-cred-XYZ",
+      resultPayload: { detail: { auth: "Bearer resolved-cred-XYZ" } },
+    });
+
+    const stepUpdate = captured.updates.at(-1)!;
+    expect(stepUpdate.errorMessage).toBe("401 from Jira using token ****");
+    expect(stepUpdate.resultPayload).toEqual({
+      detail: { auth: "Bearer ****" },
+    });
+    expect(JSON.stringify(stepUpdate)).not.toContain("resolved-cred-XYZ");
+  });
+
+  it("masks the run-level errorMessage before persist", async () => {
+    const captured: Captured = { inserts: [], updates: [] };
+    const registry = createRunSecretRegistry();
+    const store = createRunStore(capturingDb(captured), undefined, registry);
+    registry.register("run-2", ["run-cred-999"]);
+
+    await store.updateRunStatus(
+      "run-2",
+      "failed",
+      "run failed: leaked run-cred-999 in error",
+    );
+
+    const runUpdate = captured.updates.at(-1)!;
+    expect(runUpdate.errorMessage).toBe("run failed: leaked **** in error");
+    expect(JSON.stringify(runUpdate)).not.toContain("run-cred-999");
+  });
+
+  it("least-privilege: a value not resolved in the run is left intact", async () => {
+    const captured: Captured = { inserts: [], updates: [] };
+    const registry = createRunSecretRegistry();
+    const store = createRunStore(capturingDb(captured), undefined, registry);
+    // run-3 resolved nothing.
+    const stepId = await store.createStep({
+      runId: "run-3",
+      actionPath: "actions[0]",
+      actionId: "a1",
+      actionKind: "log",
+      providerActionId: null,
+    });
+    await store.updateStep(stepId, {
+      status: "success",
+      resultPayload: { message: "not-a-secret-value" },
+    });
+    const stepUpdate = captured.updates.at(-1)!;
+    expect(stepUpdate.resultPayload).toEqual({ message: "not-a-secret-value" });
+  });
+
+  it("drops the run's mask set once the run reaches a terminal status", async () => {
+    const captured: Captured = { inserts: [], updates: [] };
+    const registry = createRunSecretRegistry();
+    const store = createRunStore(capturingDb(captured), undefined, registry);
+    registry.register("run-4", ["transient-cred"]);
+    await store.updateRunStatus("run-4", "success");
+    // After terminal, the value is no longer in the registry (memory-only).
+    expect(registry.maskText("run-4", "x=transient-cred")).toBe(
+      "x=transient-cred",
+    );
+  });
+});
+
+// ─── L2 cross-pod masking ───────────────────────────────────────────────
+
+const RESOLVER_REF_ID = "secrets.resolver";
+const CONNECTION_REF_ID = "integration.connectionStore";
+const RUN_ID = "run-suspended";
+const POD_A_CRED = "podA-resolved-cred-XYZ";
+
+/**
+ * A SHARED secret backend both pods read through — the value lives in the
+ * cluster's secret/connection store, not on any one pod. Pod A and pod B get
+ * their own (per-process) mask registry, but resolve against this same store.
+ */
+function sharedSecretBackend() {
+  return {
+    resolver: {
+      resolveSecret: async () => POD_A_CRED,
+      resolveForRun: async () => ({
+        env: { API_TOKEN: POD_A_CRED },
+        masking: {},
+      }),
+      resolveBySchema: async () => ({ resolved: {}, warnings: [] }),
+    },
+    connectionStore: {
+      getConnectionWithCredentials: async () => ({
+        config: { baseUrl: "https://jira.example", apiToken: POD_A_CRED },
+      }),
+    },
+  };
+}
+
+/** A getService over the shared backend, keyed by the two intercepted refs. */
+function sharedGetService(backend: ReturnType<typeof sharedSecretBackend>) {
+  return async <T>(refArg: ServiceRef<T>): Promise<T> => {
+    if (refArg.id === RESOLVER_REF_ID) {
+      return backend.resolver as unknown as T;
+    }
+    if (refArg.id === CONNECTION_REF_ID) {
+      return backend.connectionStore as unknown as T;
+    }
+    throw new Error(`unexpected ref ${refArg.id}`);
+  };
+}
+
+/**
+ * Definition whose action uses BOTH a `secretEnv` mapping and a connection
+ * — the declared secret refs the resume path must re-resolve.
+ */
+function definitionUsingSecrets() {
+  return AutomationDefinitionSchema.parse({
+    name: "Cross-pod masking",
+    triggers: [{ event: "incident.created" }],
+    conditions: [],
+    actions: [
+      {
+        action: "integration-jira.create_issue",
+        config: {
+          connectionId: "conn-1",
+          secretEnv: { API_TOKEN: "${{ secrets.jira_token }}" },
+        },
+      },
+      { wait_for_trigger: { event: "incident.resolved" } },
+      {
+        action: "integration-script.run",
+        config: { script: "echo done" },
+      },
+    ],
+    mode: "single",
+    max_runs: 1,
+  });
+}
+
+describe("L2 cross-pod masking — resume on a fresh pod re-seeds the mask set", () => {
+  it("WITHOUT re-seed: a fresh pod-B registry persists pod-A's secret UNMASKED (the leak)", async () => {
+    // Pod B comes up cold: it never resolved this run's secrets, so its mask
+    // set is empty.
+    const podBRegistry = createRunSecretRegistry();
+    const captured: Captured = { inserts: [], updates: [] };
+    const store = createRunStore(capturingDb(captured), undefined, podBRegistry);
+
+    // Pod B persists a step whose output still carries pod-A's credential
+    // (e.g. a carried-over value surfaced post-resume).
+    const stepId = await store.createStep({
+      runId: RUN_ID,
+      actionPath: "actions[2]",
+      actionId: "a3",
+      actionKind: "action",
+      providerActionId: "integration-script.run",
+    });
+    await store.updateStep(stepId, {
+      status: "success",
+      resultPayload: { echoed: `token=${POD_A_CRED}` },
+    });
+
+    const stepUpdate = captured.updates.at(-1)!;
+    // The leak: an empty mask set leaves the value intact in the persisted row.
+    expect(JSON.stringify(stepUpdate)).toContain(POD_A_CRED);
+  });
+
+  it("WITH re-seed: pod B re-resolves declared refs, so the same persist is masked", async () => {
+    const backend = sharedSecretBackend();
+    const podBRegistry = createRunSecretRegistry();
+    const wrappedGetService = wrapGetServiceForRun({
+      getService: sharedGetService(backend),
+      runId: RUN_ID,
+      registry: podBRegistry,
+      resolverRefId: RESOLVER_REF_ID,
+      connectionStoreRefId: CONNECTION_REF_ID,
+    });
+
+    // The fix: before walking/persisting, the resuming pod re-seeds its mask
+    // set from the automation's declared secret refs.
+    await reseedRunSecretRegistry({
+      getService: wrappedGetService,
+      registry: podBRegistry,
+      runId: RUN_ID,
+      definition: definitionUsingSecrets(),
+      resolverRefId: RESOLVER_REF_ID,
+      connectionStoreRefId: CONNECTION_REF_ID,
+    });
+
+    // Now every choke point on pod B masks pod-A's value.
+    const captured: Captured = { inserts: [], updates: [] };
+    const runStore = createRunStore(
+      capturingDb(captured),
+      undefined,
+      podBRegistry,
+    );
+    const stepId = await runStore.createStep({
+      runId: RUN_ID,
+      actionPath: "actions[2]",
+      actionId: "a3",
+      actionKind: "action",
+      providerActionId: "integration-script.run",
+    });
+    await runStore.updateStep(stepId, {
+      status: "success",
+      resultPayload: { echoed: `token=${POD_A_CRED}` },
+      errorMessage: undefined,
+    });
+    const stepUpdate = captured.updates.at(-1)!;
+    expect(stepUpdate.resultPayload).toEqual({ echoed: "token=****" });
+    expect(JSON.stringify(stepUpdate)).not.toContain(POD_A_CRED);
+
+    // The scope-snapshot choke point (run-state) is masked too.
+    const noopAdvisoryLock: AdvisoryLockService = {
+      tryAcquire: async () => ({ release: async () => {} }),
+    };
+    const stateCaptured: Captured = { inserts: [], updates: [] };
+    const stateStore = createRunStateStore(
+      capturingDb(stateCaptured) as unknown as Parameters<
+        typeof createRunStateStore
+      >[0],
+      noopAdvisoryLock,
+      podBRegistry,
+    );
+    await stateStore.upsert({
+      runId: RUN_ID,
+      scopeSnapshot: { variables: { carried: POD_A_CRED }, artifacts: {} },
+      lastActionPath: "actions[2]",
+    });
+    const persistedScope = stateCaptured.inserts.at(-1)!.values;
+    expect(JSON.stringify(persistedScope.scopeSnapshot)).not.toContain(
+      POD_A_CRED,
+    );
+    expect(persistedScope.scopeSnapshot).toEqual({
+      variables: { carried: "****" },
+      artifacts: {},
+    });
+
+    // And a produced artifact echoing the credential is masked.
+    const artCaptured: Captured = { inserts: [], updates: [] };
+    const artifactStore = createArtifactStore(
+      capturingDb(artCaptured) as unknown as Parameters<
+        typeof createArtifactStore
+      >[0],
+      podBRegistry,
+    );
+    await artifactStore.record({
+      automationId: "auto-1",
+      runId: RUN_ID,
+      stepId,
+      actionId: "a3",
+      artifactType: "integration-jira.issue",
+      data: { url: "https://x", auth: `Bearer ${POD_A_CRED}` },
+      contextKey: "incident-1",
+    });
+    const persistedArtifact = artCaptured.inserts.at(-1)!.values;
+    expect(JSON.stringify(persistedArtifact.data)).not.toContain(POD_A_CRED);
+    expect(persistedArtifact.data).toEqual({
+      url: "https://x",
+      auth: "Bearer ****",
+    });
+  });
+
+  it("re-seed is a no-op for a definition that declares no secret refs", async () => {
+    const backend = sharedSecretBackend();
+    const registry = createRunSecretRegistry();
+    const wrapped = wrapGetServiceForRun({
+      getService: sharedGetService(backend),
+      runId: "run-clean",
+      registry,
+      resolverRefId: RESOLVER_REF_ID,
+      connectionStoreRefId: CONNECTION_REF_ID,
+    });
+    const definition = AutomationDefinitionSchema.parse({
+      name: "No secrets",
+      triggers: [{ event: "x.y" }],
+      conditions: [],
+      actions: [{ variables: { greeting: "hi" } }],
+      mode: "single",
+      max_runs: 1,
+    });
+    await reseedRunSecretRegistry({
+      getService: wrapped,
+      registry,
+      runId: "run-clean",
+      definition,
+      resolverRefId: RESOLVER_REF_ID,
+      connectionStoreRefId: CONNECTION_REF_ID,
+    });
+    // Nothing registered, so an unrelated value is left intact.
+    expect(registry.maskText("run-clean", `x=${POD_A_CRED}`)).toBe(
+      `x=${POD_A_CRED}`,
+    );
+  });
+});