@checkstack/automation-backend 0.2.0 → 0.3.0

package/src/dispatch/stage1-router.ts

@@ -0,0 +1,152 @@
+/**
+ * Stage-1 routing (reactive automation engine §7, §13).
+ *
+ * Subscribes to the internal `ENTITY_CHANGED` hook in work-queue mode
+ * (`workerGroup: "automation-entity-route"`) so exactly ONE instance claims
+ * each change. The claimer does only cheap, indexed routing:
+ *
+ *   (a) Waiting runs — wake-index intersection lookup (§8.2): every
+ *       `kind:"until"` wait lock whose dependency set includes the changed
+ *       `${kind}:${id}` ref (or the kind-level wildcard). One Stage-2 `wake`
+ *       job is enqueued per matching lock.
+ *   (b) Fresh-run triggers — the change is run through the per-kind
+ *       trigger-event deriver registry (§ change-derivers) to get the
+ *       qualified trigger event id(s); each id is fanned out to the enabled
+ *       automations referencing it (`findEnabledByTriggerEvent`). One
+ *       Stage-2 `trigger` job is enqueued per (automation, event) match.
+ *
+ * Stage 1 never executes a run — it only enqueues Stage-2 jobs onto the
+ * `automation-dispatch` queue, keeping the claim fast.
+ */
+import type { HookEventMeta, Logger } from "@checkstack/backend-api";
+import {
+  EntityChangedSchema,
+  type DispatchJob,
+  type EntityChanged,
+} from "@checkstack/automation-common";
+
+import type { AutomationStore } from "../automation-store";
+import type { ChangeDeriverRegistry } from "../entity/change-derivers";
+import { ENTITY_CHANGED_HOOK } from "../entity/hook";
+import { DISPATCH_QUEUE_NAME } from "./stage2-dispatch";
+import type { DispatchDeps } from "./types";
+
+/** Worker group for the Stage-1 claim (reactive automation engine §13.1). */
+export const ENTITY_ROUTE_WORKER_GROUP = "automation-entity-route";
+
+/** Type of `onHook` injected in afterPluginsReady (mirrors backend-api). */
+export type OnHookFn = <T>(
+  hook: { id: string; _type?: T },
+  listener: (payload: T, meta?: HookEventMeta) => Promise<void>,
+  options?:
+    | { mode?: "broadcast"; maxRetries?: number }
+    | { mode: "work-queue"; workerGroup: string; maxRetries?: number }
+    | { mode: "instance-local" },
+) => () => Promise<void>;
+
+export interface Stage1RouterArgs {
+  deps: DispatchDeps;
+  automationStore: AutomationStore;
+  changeDerivers: ChangeDeriverRegistry;
+  onHook: OnHookFn;
+  logger: Logger;
+}
+
+export interface Stage1Router {
+  dispose: () => Promise<void>;
+}
+
+/**
+ * Route one validated entity change to Stage-2 jobs. Exported so tests can
+ * drive routing directly (without a real hook bus); returns the jobs it
+ * enqueued for assertion.
+ */
+export async function routeEntityChange(args: {
+  deps: DispatchDeps;
+  automationStore: AutomationStore;
+  changeDerivers: ChangeDeriverRegistry;
+  changed: EntityChanged;
+}): Promise<DispatchJob[]> {
+  const { deps, automationStore, changeDerivers, changed } = args;
+  const ref = `${changed.kind}:${changed.id}`;
+  const queue = deps.queueManager.getQueue<DispatchJob>(DISPATCH_QUEUE_NAME);
+  const enqueued: DispatchJob[] = [];
+
+  // (a) Waiting runs depending on this ref (or the kind wildcard).
+  const waits = await deps.runStore.findWaitLocksByWakeRef(ref);
+  for (const lock of waits) {
+    const job: DispatchJob = {
+      reason: "wake",
+      runId: lock.runId,
+      waitLockId: lock.id,
+      ref,
+      changed,
+    };
+    await queue.enqueue(job, {
+      // One in-flight wake per (lock, ref) — a duplicate change for the same
+      // ref collapses onto the same job id.
+      jobId: `wake:${lock.id}:${ref}`,
+    });
+    enqueued.push(job);
+  }
+
+  // (b) Fresh-run triggers — derive the qualified trigger event id(s).
+  const eventIds = changeDerivers.derive(changed);
+  for (const eventId of eventIds) {
+    const automations = await automationStore.findEnabledByTriggerEvent(eventId);
+    for (const automation of automations) {
+      const job: DispatchJob = {
+        reason: "trigger",
+        automationId: automation.id,
+        triggerId: eventId,
+        ref,
+        changed,
+      };
+      await queue.enqueue(job, {
+        // Dedup REDELIVERIES of one change (stable `changeId`) but distinguish
+        // two DISTINCT changes to the same entity — even within one
+        // millisecond, where `occurredAt` collides (§13.2). Fall back to
+        // `occurredAt` only for legacy payloads emitted before `changeId`.
+        jobId: `trigger:${automation.id}:${eventId}:${ref}:${changed.changeId ?? changed.occurredAt}`,
+      });
+      enqueued.push(job);
+    }
+  }
+
+  if (enqueued.length > 0) {
+    deps.logger.debug(
+      `stage1: routed ${ref} → ${enqueued.length} dispatch job(s) (${waits.length} wake, ${enqueued.length - waits.length} trigger)`,
+    );
+  }
+  return enqueued;
+}
+
+export async function startStage1Router(
+  args: Stage1RouterArgs,
+): Promise<Stage1Router> {
+  const unsub = args.onHook(
+    ENTITY_CHANGED_HOOK,
+    async (payload) => {
+      const parsed = EntityChangedSchema.safeParse(payload);
+      if (!parsed.success) {
+        args.logger.warn(
+          `stage1: dropping malformed ENTITY_CHANGED payload: ${parsed.error.message}`,
+        );
+        return;
+      }
+      await routeEntityChange({
+        deps: args.deps,
+        automationStore: args.automationStore,
+        changeDerivers: args.changeDerivers,
+        changed: parsed.data,
+      });
+    },
+    { mode: "work-queue", workerGroup: ENTITY_ROUTE_WORKER_GROUP },
+  );
+
+  return {
+    dispose: async () => {
+      await unsub();
+    },
+  };
+}

package/src/dispatch/stage1.it.test.ts

@@ -0,0 +1,84 @@
+/**
+ * Integration test (real Redis / BullMQ) for Stage-1 routing exactly-once.
+ *
+ * Part of the surgical integration lane (plan §14.4 #4). The Stage-1 router
+ * subscribes to `ENTITY_CHANGED` in work-queue mode
+ * (`workerGroup: "automation-entity-route"`); the event-bus realises that as
+ * a BullMQ consumer group, so exactly one of N competing workers claims a
+ * given message. This pins that third-party contract directly against
+ * BullMQ: two workers sharing one consumer group on one queue → a single
+ * enqueued job is processed EXACTLY ONCE.
+ *
+ * Gated behind `CHECKSTACK_IT=1` so the default `bun test` never runs it. The
+ * `integration` CI job sets the flag and provides a real Redis service.
+ * Connection comes from `CHECKSTACK_IT_REDIS_URL` (defaulting to the
+ * `docker-compose-dev.yml` Redis port). Each run uses a unique queue name +
+ * key prefix and tears the queue down afterwards.
+ */
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import { Queue, Worker, type ConnectionOptions } from "bullmq";
+
+function redisConnection(): ConnectionOptions {
+  const url = new URL(
+    process.env.CHECKSTACK_IT_REDIS_URL ?? "redis://localhost:6379",
+  );
+  return {
+    host: url.hostname,
+    port: Number(url.port || 6379),
+    password: url.password || undefined,
+  };
+}
+
+const QUEUE = `it_stage1_${crypto.randomUUID().replace(/-/g, "")}`;
+const PREFIX = `it:${crypto.randomUUID().replace(/-/g, "")}`;
+
+describe.skipIf(!process.env.CHECKSTACK_IT)(
+  "Stage-1 routing exactly-once (real Redis)",
+  () => {
+    let queue: Queue;
+    const workers: Worker[] = [];
+
+    beforeAll(() => {
+      queue = new Queue(QUEUE, {
+        connection: redisConnection(),
+        prefix: PREFIX,
+      });
+    });
+
+    afterAll(async () => {
+      for (const w of workers) await w.close();
+      await queue.obliterate({ force: true }).catch(() => {});
+      await queue.close();
+    });
+
+    it("one ENTITY_CHANGED-style job runs the routing handler exactly once across two workers", async () => {
+      let processed = 0;
+      const seen: string[] = [];
+
+      // Two competing workers in the SAME consumer group (the event-bus uses
+      // the worker-group name as the BullMQ group; two workers on the same
+      // queue compete, so only one claims a given job).
+      const makeWorker = () =>
+        new Worker(
+          QUEUE,
+          async (job) => {
+            processed += 1;
+            seen.push(String(job.data.ref));
+          },
+          { connection: redisConnection(), prefix: PREFIX },
+        );
+      workers.push(makeWorker(), makeWorker());
+
+      // Wait for both workers to be ready before enqueuing.
+      await Promise.all(workers.map((w) => w.waitUntilReady()));
+
+      await queue.add("entity-changed", { ref: "health:sys-1" });
+
+      // Give the workers time to claim + process.
+      await new Promise((r) => setTimeout(r, 1500));
+
+      expect(processed).toBe(1);
+      expect(seen).toEqual(["health:sys-1"]);
+    });
+  },
+);

package/src/dispatch/stage2-dispatch.test.ts

@@ -0,0 +1,285 @@
+import { describe, it, expect } from "bun:test";
+import { AutomationDefinitionSchema } from "@checkstack/automation-common";
+import type { EntityChanged, DispatchJob } from "@checkstack/automation-common";
+import { SYSTEM_ACTOR } from "@checkstack/common";
+
+import { createActionRegistry } from "../action-registry";
+import { makeDispatchDeps, makeRecordingAction, testPlugin } from "./test-fixtures";
+import { handleDispatchJob } from "./stage2-dispatch";
+import { createChangeDeriverRegistry } from "../entity/change-derivers";
+import type { AutomationStore } from "../automation-store";
+import type { LoadedAutomation } from "./types";
+
+/** An empty registry: every change falls back to the generic payload shape. */
+function emptyDerivers() {
+  return createChangeDeriverRegistry();
+}
+
+function change(overrides: Partial<EntityChanged> = {}): EntityChanged {
+  return {
+    kind: "fake",
+    id: "ent-1",
+    prev: null,
+    next: { status: "open", severity: "high" },
+    delta: { status: "open" },
+    changedFields: ["status"],
+    actor: SYSTEM_ACTOR,
+    occurredAt: new Date().toISOString(),
+    ...overrides,
+  };
+}
+
+function automationFor(event: string): LoadedAutomation {
+  const definition = AutomationDefinitionSchema.parse({
+    name: "A",
+    triggers: [{ event }],
+    conditions: [],
+    actions: [{ action: "test.record", config: { value: "{{ trigger.payload.status }}" } }],
+    mode: "single",
+    max_runs: 10,
+  });
+  return { id: "auto-1", name: "A", status: "enabled", definition };
+}
+
+function storeFor(auto: LoadedAutomation): AutomationStore {
+  return {
+    create: async () => {
+      throw new Error("nope");
+    },
+    update: async () => {
+      throw new Error("nope");
+    },
+    delete: async () => {},
+    toggle: async () => {
+      throw new Error("nope");
+    },
+    getById: async (id) =>
+      id === auto.id
+        ? {
+            id: auto.id,
+            name: auto.name,
+            description: undefined,
+            status: auto.status,
+            definition: auto.definition,
+            managedBy: undefined,
+            createdAt: new Date(),
+            updatedAt: new Date(),
+          }
+        : undefined,
+    list: async () => ({ items: [], total: 0 }),
+    listGroups: async () => [],
+    findEnabledByTriggerEvent: async () => [auto],
+    listEnabled: async () => [auto],
+  };
+}
+
+describe("Stage-2 handleDispatchJob — reason: trigger", () => {
+  it("starts a fresh run for the matched automation, with the change as payload", async () => {
+    const actions = createActionRegistry();
+    const rec = makeRecordingAction();
+    actions.register(rec.definition, testPlugin);
+    const { deps, runs } = makeDispatchDeps({ actions });
+
+    const auto = automationFor("fake.opened");
+    const job: DispatchJob = {
+      reason: "trigger",
+      automationId: "auto-1",
+      triggerId: "fake.opened",
+      ref: "fake:ent-1",
+      changed: change(),
+    };
+
+    await handleDispatchJob({ deps, automationStore: storeFor(auto), changeDerivers: emptyDerivers(), job });
+
+    // The action ran with the entity-change's `next.status` in payload.
+    expect(rec.calls.map((c) => c.value)).toEqual(["open"]);
+    // A run row exists and reached a terminal status.
+    expect([...runs.runs.values()].map((r) => r.status)).toEqual(["success"]);
+  });
+
+  it("drops the job when the automation is gone", async () => {
+    const { deps, runs } = makeDispatchDeps();
+    const job: DispatchJob = {
+      reason: "trigger",
+      automationId: "missing",
+      triggerId: "fake.opened",
+      ref: "fake:ent-1",
+      changed: change(),
+    };
+    await handleDispatchJob({
+      deps,
+      automationStore: storeFor(automationFor("fake.opened")),
+      changeDerivers: emptyDerivers(),
+      job,
+    });
+    expect(runs.runs.size).toBe(0);
+  });
+
+  it("does not run a disabled automation", async () => {
+    const actions = createActionRegistry();
+    const rec = makeRecordingAction();
+    actions.register(rec.definition, testPlugin);
+    const { deps, runs } = makeDispatchDeps({ actions });
+    const auto = automationFor("fake.opened");
+    auto.status = "disabled";
+    const job: DispatchJob = {
+      reason: "trigger",
+      automationId: "auto-1",
+      triggerId: "fake.opened",
+      ref: "fake:ent-1",
+      changed: change(),
+    };
+    await handleDispatchJob({ deps, automationStore: storeFor(auto), changeDerivers: emptyDerivers(), job });
+    expect(rec.calls).toHaveLength(0);
+    expect(runs.runs.size).toBe(0);
+  });
+});
+
+describe("Stage-2 handleDispatchJob — reason: wake", () => {
+  it("resumes a suspended wait_until whose condition now holds", async () => {
+    const actions = createActionRegistry();
+    const rec = makeRecordingAction();
+    actions.register(rec.definition, testPlugin);
+
+    // Health client that reports healthy, so the wait re-eval passes.
+    const healthClient = {
+      getHealthState: async () => ({
+        status: "healthy",
+        inStatusSince: new Date(),
+        inStatusForMs: 0,
+        inMaintenance: false,
+        evaluatedAt: new Date(),
+      }),
+      getBulkHealthState: async ({ systemIds }: { systemIds: string[] }) => {
+        const states: Record<string, unknown> = {};
+        for (const id of systemIds) {
+          states[id] = {
+            status: "healthy",
+            inStatusSince: new Date(),
+            inStatusForMs: 0,
+            inMaintenance: false,
+            evaluatedAt: new Date(),
+          };
+        }
+        return { states };
+      },
+    } as never;
+
+    const { deps, runs } = makeDispatchDeps({ actions, healthCheckClient: healthClient });
+
+    // Build an automation with a wait_until then a recording action, suspend
+    // it by dispatching while unhealthy is irrelevant — we seed the lock
+    // directly and drive the wake.
+    const definition = AutomationDefinitionSchema.parse({
+      name: "WU",
+      triggers: [{ event: "test.event" }],
+      conditions: [],
+      actions: [
+        { wait_until: { condition: "health.system.status == 'healthy'" } },
+        { action: "test.record", config: { value: "woke" } },
+      ],
+      mode: "single",
+      max_runs: 10,
+    });
+    const auto: LoadedAutomation = {
+      id: "auto-1",
+      name: "WU",
+      status: "enabled",
+      definition,
+    };
+
+    // Suspend the run via the engine so the lock + scope snapshot exist.
+    const unhealthyDeps = makeDispatchDeps({
+      actions,
+      healthCheckClient: {
+        getHealthState: async () => ({
+          status: "unhealthy",
+          inStatusSince: new Date(),
+          inStatusForMs: 0,
+          inMaintenance: false,
+          evaluatedAt: new Date(),
+        }),
+        getBulkHealthState: async ({ systemIds }: { systemIds: string[] }) => {
+          const states: Record<string, unknown> = {};
+          for (const id of systemIds) {
+            states[id] = {
+              status: "unhealthy",
+              inStatusSince: new Date(),
+              inStatusForMs: 0,
+              inMaintenance: false,
+              evaluatedAt: new Date(),
+            };
+          }
+          return { states };
+        },
+      } as never,
+    });
+    // Use one shared deps so the run/lock seeded by the engine is the one the
+    // wake reads. Re-dispatch on `deps` (healthy) would immediately satisfy,
+    // so instead seed with unhealthy deps and then wake with healthy deps
+    // pointing at the SAME stores.
+    void unhealthyDeps;
+
+    // Simplest faithful path: dispatch on `deps` but with health unhealthy
+    // first is not possible (deps is healthy). Instead seed the lock + run
+    // manually and drive the wake re-eval (healthy → resume).
+    await deps.runStore.createRun({
+      automationId: "auto-1",
+      triggerId: "t",
+      triggerEventId: "test.event",
+      triggerPayload: { id: "sys-1" },
+      contextKey: "sys-1",
+    });
+    await deps.runStore.updateRunStatus("run-1", "waiting");
+    await deps.runStateStore.upsert({
+      runId: "run-1",
+      scopeSnapshot: { trigger: { payload: { id: "sys-1" } } },
+      lastActionPath: null,
+    });
+    const lockId = await deps.runStore.createWaitLockWithWakeRefs({
+      runId: "run-1",
+      actionPath: "actions[0]",
+      eventId: "@@until",
+      contextKey: "sys-1",
+      timeoutAt: null,
+      waitConfig: {
+        condition: "health.system.status == 'healthy'",
+        continueOnTimeout: true,
+      },
+      wakeRefs: ["health:*"],
+    });
+
+    const job: DispatchJob = {
+      reason: "wake",
+      runId: "run-1",
+      waitLockId: lockId,
+      ref: "health:sys-1",
+      changed: change({ kind: "health", id: "sys-1" }),
+    };
+    await handleDispatchJob({ deps, automationStore: storeFor(auto), changeDerivers: emptyDerivers(), job });
+
+    // Re-eval passed (healthy) → resumed → recording action ran.
+    expect(rec.calls.map((c) => c.value)).toEqual(["woke"]);
+    expect(runs.runs.get("run-1")?.status).toBe("success");
+    expect(runs.waitLocks.size).toBe(0);
+  });
+
+  it("drops a wake job whose lock is gone", async () => {
+    const { deps } = makeDispatchDeps();
+    const job: DispatchJob = {
+      reason: "wake",
+      runId: "run-x",
+      waitLockId: "missing",
+      ref: "fake:ent-1",
+      changed: change(),
+    };
+    // No throw, no-op.
+    await handleDispatchJob({
+      deps,
+      automationStore: storeFor(automationFor("x")),
+      changeDerivers: emptyDerivers(),
+      job,
+    });
+    expect(true).toBe(true);
+  });
+});