@checkstack/automation-backend 0.2.0 → 0.3.0

package/src/dispatch/scope-artifact-masking.test.ts

@@ -0,0 +1,138 @@
+import { describe, it, expect } from "bun:test";
+import { createRunStateStore } from "./run-state-store";
+import { createRunSecretRegistry } from "./run-secret-registry";
+import { createArtifactStore } from "../artifact-store";
+import type { AdvisoryLockService } from "@checkstack/backend-api";
+
+/**
+ * H2 — the run-wide masking choke point must ALSO cover the scope snapshot
+ * (run-state) and produced artifacts. A resolved connection credential
+ * threaded into `scope.variables` / an artifact's `data` must be redacted
+ * BEFORE persist, so `getRunScopeForReplay` (which reads those rows
+ * verbatim, gated only on automationAccess.read) can never hand a
+ * read-only user the live value.
+ *
+ * Uses a capturing fake `db` (the established boundary for these store
+ * tests) so we assert on exactly what the store writes to the DB layer —
+ * which is what the replay endpoint later reads back.
+ */
+
+interface Captured {
+  inserts: Array<{ values: Record<string, unknown> }>;
+}
+
+function capturingDb(captured: Captured) {
+  const chain = {
+    values(v: Record<string, unknown>) {
+      captured.inserts.push({ values: v });
+      return {
+        returning() {
+          return Promise.resolve([{ id: "generated-id", ...v }]);
+        },
+        onConflictDoUpdate() {
+          return Promise.resolve();
+        },
+      };
+    },
+  };
+  const db = {
+    insert() {
+      return chain;
+    },
+  };
+  return db;
+}
+
+const noopAdvisoryLock: AdvisoryLockService = {
+  tryAcquire: async () => ({ release: async () => {} }),
+};
+
+describe("H2 — scope snapshot masking (run-state choke point)", () => {
+  it("masks a resolved credential surfaced into scope.variables before persist", async () => {
+    const captured: Captured = { inserts: [] };
+    const registry = createRunSecretRegistry();
+    // The run resolved this connection credential during execution (the
+    // wrapped connection store registered it).
+    registry.register("run-1", ["super-secret-token-123"]);
+
+    const store = createRunStateStore(
+      capturingDb(captured) as unknown as Parameters<
+        typeof createRunStateStore
+      >[0],
+      noopAdvisoryLock,
+      registry,
+    );
+
+    await store.upsert({
+      runId: "run-1",
+      scopeSnapshot: {
+        variables: { apiKey: "super-secret-token-123" },
+        artifacts: {},
+      },
+      lastActionPath: "actions[0]",
+    });
+
+    const persisted = captured.inserts.at(-1)!.values;
+    // This is exactly the row getRunScopeForReplay would read back.
+    expect(JSON.stringify(persisted.scopeSnapshot)).not.toContain(
+      "super-secret-token-123",
+    );
+    expect(persisted.scopeSnapshot).toEqual({
+      variables: { apiKey: "****" },
+      artifacts: {},
+    });
+  });
+
+  it("least-privilege: a value not resolved in the run is left intact", async () => {
+    const captured: Captured = { inserts: [] };
+    const registry = createRunSecretRegistry();
+    const store = createRunStateStore(
+      capturingDb(captured) as unknown as Parameters<
+        typeof createRunStateStore
+      >[0],
+      noopAdvisoryLock,
+      registry,
+    );
+    await store.upsert({
+      runId: "run-x",
+      scopeSnapshot: { variables: { note: "not-a-secret" } },
+      lastActionPath: null,
+    });
+    const persisted = captured.inserts.at(-1)!.values;
+    expect(persisted.scopeSnapshot).toEqual({
+      variables: { note: "not-a-secret" },
+    });
+  });
+});
+
+describe("H2 — artifact data masking (artifact-store choke point)", () => {
+  it("masks a resolved credential surfaced into a produced artifact before insert", async () => {
+    const captured: Captured = { inserts: [] };
+    const registry = createRunSecretRegistry();
+    registry.register("run-2", ["resolved-cred-XYZ"]);
+
+    const store = createArtifactStore(
+      capturingDb(captured) as unknown as Parameters<
+        typeof createArtifactStore
+      >[0],
+      registry,
+    );
+
+    await store.record({
+      automationId: "auto-1",
+      runId: "run-2",
+      stepId: "step-1",
+      actionId: "a1",
+      artifactType: "integration-jira.issue",
+      data: { url: "https://x", auth: "Bearer resolved-cred-XYZ" },
+      contextKey: "incident-1",
+    });
+
+    const persisted = captured.inserts.at(-1)!.values;
+    expect(JSON.stringify(persisted.data)).not.toContain("resolved-cred-XYZ");
+    expect(persisted.data).toEqual({
+      url: "https://x",
+      auth: "Bearer ****",
+    });
+  });
+});

package/src/dispatch/secret-ref-ids.test.ts

@@ -0,0 +1,19 @@
+import { describe, it, expect } from "bun:test";
+import { secretResolverRef } from "@checkstack/secrets-backend";
+import { connectionStoreRef } from "@checkstack/integration-backend";
+import {
+  SECRET_RESOLVER_REF_ID,
+  CONNECTION_STORE_REF_ID,
+} from "./secret-ref-ids";
+
+/**
+ * Drift guard: the dispatch core intercepts these service refs by literal
+ * id (to avoid a hard dependency cycle). If either ref is renamed, this
+ * test fails so run-wide masking can't silently stop capturing values.
+ */
+describe("secret ref id constants", () => {
+  it("match the real secretResolverRef / connectionStoreRef ids", () => {
+    expect(SECRET_RESOLVER_REF_ID).toBe(secretResolverRef.id);
+    expect(CONNECTION_STORE_REF_ID).toBe(connectionStoreRef.id);
+  });
+});

package/src/dispatch/secret-ref-ids.ts

@@ -0,0 +1,17 @@
+/**
+ * Service-ref ids the dispatch engine intercepts to capture resolved
+ * secret values for run-wide output masking.
+ *
+ * These are kept as plain string constants (rather than importing the
+ * `secretResolverRef` / `connectionStoreRef` objects) so the dispatch core
+ * does NOT take a hard dependency on `@checkstack/secrets-backend` or
+ * `@checkstack/integration-backend` (which would risk a dependency cycle).
+ * A unit test asserts these match the real ref ids, so a rename in either
+ * package fails CI rather than silently disabling masking.
+ */
+
+/** Must equal `secretResolverRef.id` in `@checkstack/secrets-backend`. */
+export const SECRET_RESOLVER_REF_ID = "secrets.resolver";
+
+/** Must equal `connectionStoreRef.id` in `@checkstack/integration-backend`. */
+export const CONNECTION_STORE_REF_ID = "integration.connectionStore";

package/src/dispatch/snapshots.test.ts

@@ -0,0 +1,86 @@
+import { describe, it, expect } from "bun:test";
+import { SYSTEM_ACTOR } from "@checkstack/common";
+import { parseActorSnapshot, parseWaitConfig } from "./snapshots";
+
+const silentLogger = {
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as unknown as Parameters<typeof parseActorSnapshot>[0]["logger"];
+
+describe("parseActorSnapshot", () => {
+  it("returns a valid stored actor unchanged", () => {
+    const actor = { type: "user" as const, id: "u-1" };
+    const parsed = parseActorSnapshot({
+      value: actor,
+      logger: silentLogger,
+      context: "test",
+    });
+    expect(parsed).toEqual(actor);
+  });
+
+  it("degrades a malformed/drifted snapshot to the system actor", () => {
+    // A hand-edited / drifted row: wrong shape. Must NOT flow through as a
+    // bogus Actor — degrades safely (and would log a warning).
+    const parsed = parseActorSnapshot({
+      value: { type: "not-a-real-type", whoops: true },
+      logger: silentLogger,
+      context: "Dwell d-1",
+    });
+    expect(parsed).toEqual(SYSTEM_ACTOR);
+  });
+
+  it("degrades a non-object snapshot to the system actor", () => {
+    expect(
+      parseActorSnapshot({ value: "garbage", context: "x" }),
+    ).toEqual(SYSTEM_ACTOR);
+    expect(parseActorSnapshot({ value: null, context: "x" })).toEqual(
+      SYSTEM_ACTOR,
+    );
+  });
+});
+
+describe("parseWaitConfig", () => {
+  const valid = {
+    condition: "trigger.payload.x == 1",
+    continueOnTimeout: false,
+  };
+
+  it("returns a valid stored wait config unchanged", () => {
+    const parsed = parseWaitConfig({
+      value: valid,
+      logger: silentLogger,
+      context: "test",
+    });
+    expect(parsed).toEqual(valid);
+  });
+
+  it("resumes an old suspended wait that still carries pollSeconds", () => {
+    // Back-compat: rows written before the reactive engine dropped polling
+    // still carry a `pollSeconds` key. The schema is a non-strict z.object,
+    // so the extra key is stripped and the wait resumes cleanly.
+    const parsed = parseWaitConfig({
+      value: { ...valid, pollSeconds: 30 },
+      logger: silentLogger,
+      context: "test",
+    });
+    expect(parsed).toEqual(valid);
+    expect(parsed).not.toHaveProperty("pollSeconds");
+  });
+
+  it("treats null/undefined as no config", () => {
+    expect(parseWaitConfig({ value: null, context: "x" })).toBeNull();
+    expect(parseWaitConfig({ value: undefined, context: "x" })).toBeNull();
+  });
+
+  it("returns null for a malformed/drifted config (engine treats as gone)", () => {
+    // Missing required fields — must not be trusted as an UntilWaitConfig.
+    const parsed = parseWaitConfig({
+      value: { continueOnTimeout: "nope" },
+      logger: silentLogger,
+      context: "Wait lock w-1",
+    });
+    expect(parsed).toBeNull();
+  });
+});

package/src/dispatch/snapshots.ts

@@ -0,0 +1,79 @@
+/**
+ * Validation for snapshots that were written to jsonb columns and are read
+ * back later (dwell `actorSnapshot`, wait-lock `waitConfig`).
+ *
+ * These columns are typed `unknown`/`Record<string, unknown>` at the DB
+ * boundary, so the engine previously trusted them via `as unknown as ...`
+ * casts on LOAD. A drifted schema or a hand-edited row would then flow
+ * through as the wrong type and blow up far from the cause. We instead
+ * `safeParse` on load and degrade safely (the write side stays as-is — it
+ * writes known-good typed data).
+ */
+import { z } from "zod";
+import { type Actor, ActorSchema, SYSTEM_ACTOR } from "@checkstack/common";
+import { ConditionSchema } from "@checkstack/automation-common";
+import type { Logger } from "@checkstack/backend-api";
+
+import type { UntilWaitConfig } from "./types";
+
+/**
+ * Stored config for a `kind: "until"` wait lock (jsonb).
+ *
+ * NOTE: this is a non-strict `z.object`, so old persisted rows that still
+ * carry a `pollSeconds` key (written before the reactive engine dropped
+ * polling) parse cleanly — zod silently strips the unknown key. Do NOT make
+ * this `.strict()` or resuming an already-suspended wait would fail.
+ */
+export const UntilWaitConfigSchema: z.ZodType<UntilWaitConfig> = z.object({
+  condition: ConditionSchema,
+  continueOnTimeout: z.boolean(),
+});
+
+/**
+ * Parse a stored `actorSnapshot`. Degrades to {@link SYSTEM_ACTOR} on a
+ * malformed/drifted row rather than crashing the fire — actor is used for
+ * attribution, and dropping a legitimate alert because of a bad snapshot is
+ * worse than attributing it to the system. Logs a warning so the drift is
+ * visible.
+ */
+export function parseActorSnapshot({
+  value,
+  logger,
+  context,
+}: {
+  value: unknown;
+  logger?: Logger;
+  context: string;
+}): Actor {
+  const result = ActorSchema.safeParse(value);
+  if (result.success) return result.data;
+  logger?.warn(
+    `${context}: stored actorSnapshot is malformed (${result.error.message}); ` +
+      `falling back to the system actor`,
+  );
+  return SYSTEM_ACTOR;
+}
+
+/**
+ * Parse a stored wait-lock `waitConfig`. Returns `null` (which the engine
+ * treats as a gone/invalid `until` lock and fails the run cleanly) on a
+ * malformed/drifted row, rather than trusting wrongly-typed data downstream.
+ */
+export function parseWaitConfig({
+  value,
+  logger,
+  context,
+}: {
+  value: unknown;
+  logger?: Logger;
+  context: string;
+}): UntilWaitConfig | null {
+  if (value === null || value === undefined) return null;
+  const result = UntilWaitConfigSchema.safeParse(value);
+  if (result.success) return result.data;
+  logger?.warn(
+    `${context}: stored waitConfig is malformed (${result.error.message}); ` +
+      `treating the wait lock as invalid`,
+  );
+  return null;
+}

package/src/dispatch/stage1-router.test.ts

@@ -0,0 +1,324 @@
+import { describe, it, expect } from "bun:test";
+import { AutomationDefinitionSchema } from "@checkstack/automation-common";
+import type { EntityChanged, DispatchJob } from "@checkstack/automation-common";
+import { SYSTEM_ACTOR } from "@checkstack/common";
+
+import { createActionRegistry } from "../action-registry";
+import { createChangeDeriverRegistry } from "../entity/change-derivers";
+import { makeDispatchDeps, makeRecordingAction, testPlugin } from "./test-fixtures";
+import { routeEntityChange, ENTITY_ROUTE_WORKER_GROUP } from "./stage1-router";
+import { DISPATCH_QUEUE_NAME } from "./stage2-dispatch";
+import type { AutomationStore } from "../automation-store";
+import type { LoadedAutomation } from "./types";
+
+function change(overrides: Partial<EntityChanged> = {}): EntityChanged {
+  return {
+    kind: "fake",
+    id: "ent-1",
+    prev: null,
+    next: { status: "open" },
+    delta: { status: "open" },
+    changedFields: ["status"],
+    actor: SYSTEM_ACTOR,
+    occurredAt: new Date().toISOString(),
+    ...overrides,
+  };
+}
+
+function fakeAutomation(): LoadedAutomation {
+  const definition = AutomationDefinitionSchema.parse({
+    name: "A",
+    triggers: [{ event: "fake.opened" }],
+    conditions: [],
+    actions: [{ action: "test.record", config: { value: "ran" } }],
+    mode: "single",
+    max_runs: 10,
+  });
+  return { id: "auto-1", name: "A", status: "enabled", definition };
+}
+
+function storeWith(autos: LoadedAutomation[]): AutomationStore {
+  return {
+    create: async () => {
+      throw new Error("nope");
+    },
+    update: async () => {
+      throw new Error("nope");
+    },
+    delete: async () => {},
+    toggle: async () => {
+      throw new Error("nope");
+    },
+    getById: async (id) => {
+      const a = autos.find((x) => x.id === id);
+      return a
+        ? {
+            id: a.id,
+            name: a.name,
+            description: undefined,
+            status: a.status,
+            definition: a.definition,
+            managedBy: undefined,
+            createdAt: new Date(),
+            updatedAt: new Date(),
+          }
+        : undefined;
+    },
+    list: async () => ({ items: [], total: 0 }),
+    listGroups: async () => [],
+    findEnabledByTriggerEvent: async (eventId) =>
+      autos.filter((a) => a.definition.triggers.some((t) => t.event === eventId)),
+    listEnabled: async () => autos,
+  };
+}
+
+describe("ENTITY_ROUTE_WORKER_GROUP", () => {
+  it("is the documented Stage-1 worker group", () => {
+    expect(ENTITY_ROUTE_WORKER_GROUP).toBe("automation-entity-route");
+  });
+});
+
+describe("Stage-1 routeEntityChange — fresh triggers via deriver", () => {
+  it("routes a change to a Stage-2 trigger job via a registered fake deriver", async () => {
+    const actions = createActionRegistry();
+    actions.register(makeRecordingAction().definition, testPlugin);
+    const { deps, queue } = makeDispatchDeps({ actions });
+
+    const changeDerivers = createChangeDeriverRegistry();
+    changeDerivers.register({
+      kind: "fake",
+      derive: (c) => (c.next?.status === "open" ? ["fake.opened"] : []),
+    });
+
+    const auto = fakeAutomation();
+    const jobs = await routeEntityChange({
+      deps,
+      automationStore: storeWith([auto]),
+      changeDerivers,
+      changed: change(),
+    });
+
+    expect(jobs).toHaveLength(1);
+    const job = jobs[0]!;
+    expect(job.reason).toBe("trigger");
+    if (job.reason === "trigger") {
+      expect(job.automationId).toBe("auto-1");
+      expect(job.triggerId).toBe("fake.opened");
+      expect(job.ref).toBe("fake:ent-1");
+    }
+    // The job was enqueued onto the dispatch queue.
+    expect(queue.jobs.some((j) => j.queue === DISPATCH_QUEUE_NAME)).toBe(true);
+  });
+
+  it("routes nothing when no deriver is registered (Phase-5 production default)", async () => {
+    const { deps } = makeDispatchDeps();
+    const changeDerivers = createChangeDeriverRegistry();
+    const jobs = await routeEntityChange({
+      deps,
+      automationStore: storeWith([fakeAutomation()]),
+      changeDerivers,
+      changed: change(),
+    });
+    expect(jobs).toHaveLength(0);
+  });
+
+  it("derives multiple event ids and fans out to each referencing automation", async () => {
+    const { deps } = makeDispatchDeps();
+    const changeDerivers = createChangeDeriverRegistry();
+    changeDerivers.register({
+      kind: "fake",
+      derive: () => ["fake.opened", "fake.touched"],
+    });
+
+    const a1 = fakeAutomation();
+    const a2def = AutomationDefinitionSchema.parse({
+      name: "B",
+      triggers: [{ event: "fake.touched" }],
+      conditions: [],
+      actions: [{ action: "test.record", config: { value: "ran" } }],
+      mode: "single",
+      max_runs: 10,
+    });
+    const a2: LoadedAutomation = {
+      id: "auto-2",
+      name: "B",
+      status: "enabled",
+      definition: a2def,
+    };
+
+    const jobs = await routeEntityChange({
+      deps,
+      automationStore: storeWith([a1, a2]),
+      changeDerivers,
+      changed: change(),
+    });
+    const triggerJobs = jobs.filter((j): j is Extract<DispatchJob, { reason: "trigger" }> => j.reason === "trigger");
+    expect(triggerJobs.map((j) => j.automationId).toSorted()).toEqual([
+      "auto-1",
+      "auto-2",
+    ]);
+  });
+});
+
+describe("Stage-1 trigger jobId — dedup by changeId, not occurredAt", () => {
+  /** Re-route the same logical change and collect the trigger job ids. */
+  async function triggerJobIdsFor(changed: EntityChanged): Promise<string[]> {
+    const actions = createActionRegistry();
+    actions.register(makeRecordingAction().definition, testPlugin);
+    const { deps, queue } = makeDispatchDeps({ actions });
+    const changeDerivers = createChangeDeriverRegistry();
+    changeDerivers.register({
+      kind: "fake",
+      derive: (c) => (c.next?.status === "open" ? ["fake.opened"] : []),
+    });
+    await routeEntityChange({
+      deps,
+      automationStore: storeWith([fakeAutomation()]),
+      changeDerivers,
+      changed,
+    });
+    return queue.jobs
+      .filter((j) => j.queue === DISPATCH_QUEUE_NAME && j.jobId?.startsWith("trigger:"))
+      .map((j) => j.jobId!)
+      .filter((id): id is string => id !== undefined);
+  }
+
+  it("two DISTINCT changes within one ms (distinct changeId) get distinct jobIds", async () => {
+    // Same entity, same occurredAt (ms granularity collides), distinct change.
+    const at = new Date().toISOString();
+    const idsA = await triggerJobIdsFor(
+      change({ occurredAt: at, changeId: "chg-A" }),
+    );
+    const idsB = await triggerJobIdsFor(
+      change({ occurredAt: at, changeId: "chg-B" }),
+    );
+    expect(idsA).toHaveLength(1);
+    expect(idsB).toHaveLength(1);
+    // The wall-clock occurredAt is identical, but the changeId distinguishes
+    // them → distinct jobIds → BullMQ keeps both runs.
+    expect(idsA[0]).not.toBe(idsB[0]);
+    expect(idsA[0]).toContain("chg-A");
+    expect(idsB[0]).toContain("chg-B");
+  });
+
+  it("a REDELIVERY of one change (same changeId) derives the SAME jobId (deduped)", async () => {
+    const changed = change({ changeId: "chg-redelivered" });
+    const first = await triggerJobIdsFor(changed);
+    // A redelivery carries the same payload (changeId travels with it).
+    const second = await triggerJobIdsFor(changed);
+    expect(first[0]).toBe(second[0]);
+  });
+
+  it("falls back to occurredAt for legacy payloads without a changeId", async () => {
+    const at = new Date().toISOString();
+    const ids = await triggerJobIdsFor(change({ occurredAt: at }));
+    expect(ids[0]).toContain(at);
+  });
+});
+
+describe("Stage-1 routeEntityChange — waiting runs via wake-index", () => {
+  it("routes a Stage-2 wake job for a wait whose ref matches", async () => {
+    const { deps, runs } = makeDispatchDeps();
+
+    // Seed a reactive until-lock depending on `fake:ent-1`.
+    await deps.runStore.createRun({
+      automationId: "auto-1",
+      triggerId: "t",
+      triggerEventId: "test.event",
+      triggerPayload: {},
+      contextKey: "ent-1",
+    });
+    const lockId = await deps.runStore.createWaitLockWithWakeRefs({
+      runId: "run-1",
+      actionPath: "actions[0]",
+      eventId: "@@until",
+      contextKey: "ent-1",
+      timeoutAt: null,
+      waitConfig: {
+        condition: "state.fake['ent-1'].status == 'open'",
+        continueOnTimeout: true,
+      },
+      wakeRefs: ["fake:ent-1"],
+    });
+    void lockId;
+
+    const changeDerivers = createChangeDeriverRegistry();
+    const jobs = await routeEntityChange({
+      deps,
+      automationStore: storeWith([]),
+      changeDerivers,
+      changed: change(),
+    });
+
+    expect(jobs).toHaveLength(1);
+    expect(jobs[0]?.reason).toBe("wake");
+    if (jobs[0]?.reason === "wake") {
+      expect(jobs[0].runId).toBe("run-1");
+      expect(jobs[0].ref).toBe("fake:ent-1");
+    }
+    expect(runs.waitLocks.size).toBe(1);
+  });
+
+  it("matches a kind-level wildcard wait", async () => {
+    const { deps } = makeDispatchDeps();
+    await deps.runStore.createRun({
+      automationId: "auto-1",
+      triggerId: "t",
+      triggerEventId: "test.event",
+      triggerPayload: {},
+      contextKey: null,
+    });
+    await deps.runStore.createWaitLockWithWakeRefs({
+      runId: "run-1",
+      actionPath: "actions[0]",
+      eventId: "@@until",
+      contextKey: null,
+      timeoutAt: null,
+      waitConfig: {
+        condition: "state.fake[trigger.id].status == 'open'",
+        continueOnTimeout: true,
+      },
+      wakeRefs: ["fake:*"],
+    });
+
+    const jobs = await routeEntityChange({
+      deps,
+      automationStore: storeWith([]),
+      changeDerivers: createChangeDeriverRegistry(),
+      changed: change({ id: "ent-99" }),
+    });
+    expect(jobs).toHaveLength(1);
+    expect(jobs[0]?.reason).toBe("wake");
+  });
+
+  it("does not match a wait depending on a different ref", async () => {
+    const { deps } = makeDispatchDeps();
+    await deps.runStore.createRun({
+      automationId: "auto-1",
+      triggerId: "t",
+      triggerEventId: "test.event",
+      triggerPayload: {},
+      contextKey: null,
+    });
+    await deps.runStore.createWaitLockWithWakeRefs({
+      runId: "run-1",
+      actionPath: "actions[0]",
+      eventId: "@@until",
+      contextKey: null,
+      timeoutAt: null,
+      waitConfig: {
+        condition: "state.fake['other'].status == 'open'",
+        continueOnTimeout: true,
+      },
+      wakeRefs: ["fake:other"],
+    });
+
+    const jobs = await routeEntityChange({
+      deps,
+      automationStore: storeWith([]),
+      changeDerivers: createChangeDeriverRegistry(),
+      changed: change({ id: "ent-1" }),
+    });
+    expect(jobs).toHaveLength(0);
+  });
+});