@checkstack/automation-backend 0.2.0 → 0.3.0

package/src/dispatch/test-fixtures.ts

@@ -5,12 +5,9 @@
  * registries so engine tests can run without a real database or queue.
  */
 import { z } from "zod";
-import { Versioned, createHook } from "@checkstack/backend-api";
+import { Versioned } from "@checkstack/backend-api";
 import { createDefaultFilterRegistry } from "@checkstack/template-engine";
-import type {
-  ActionDefinition,
-  TriggerDefinition,
-} from "../action-types";
+import type { ActionDefinition } from "../action-types";
 import { createActionRegistry, type ActionRegistry } from "../action-registry";
 import {
   createArtifactTypeRegistry,
@@ -23,14 +20,28 @@ import type {
   CreateRunInput,
   CreateStepInput,
   CreateWaitLockInput,
+  CreateWaitLockWithRefsInput,
   DispatchDeps,
+  DwellStore,
+  LoadedDwell,
   LoadedRun,
   LoadedWaitLock,
+  RecordWindowInput,
   RunStore,
+  UpsertDwellInput,
+  WindowStore,
 } from "./types";
 import type { RunStateSnapshot, RunStateStore } from "./run-state-store";
 
-export function createInMemoryRunStore(): {
+export function createInMemoryRunStore(opts?: {
+  /**
+   * Called with the ids of runs cancelled by `cancelActiveRuns`, so a
+   * caller (makeDispatchDeps) can clear the corresponding run-state rows -
+   * faithfully modelling the real store, which deletes wait locks AND
+   * run-state in the same operation.
+   */
+  onCancel?: (cancelledRunIds: string[]) => void;
+}): {
   store: RunStore;
   runs: Map<string, LoadedRun>;
   steps: Array<{
@@ -46,6 +57,8 @@ export function createInMemoryRunStore(): {
     resultPayload?: Record<string, unknown>;
   }>;
   waitLocks: Map<string, LoadedWaitLock>;
+  /** Wake-index rows keyed by wait-lock id → set of refs (reactive §8). */
+  wakeRefs: Map<string, Set<string>>;
 } {
   const runs = new Map<string, LoadedRun>();
   const steps: Array<{
@@ -61,6 +74,8 @@ export function createInMemoryRunStore(): {
     resultPayload?: Record<string, unknown>;
   }> = [];
   const waitLocks = new Map<string, LoadedWaitLock>();
+  const wakeRefs = new Map<string, Set<string>>();
+  const onCancel = opts?.onCancel;
   let runCounter = 0;
   let stepCounter = 0;
   let lockCounter = 0;
@@ -96,27 +111,29 @@ export function createInMemoryRunStore(): {
     async loadRun(runId) {
       return runs.get(runId);
     },
-    async countActiveRuns(automationId) {
+    async countActiveRuns(automationId, contextKey?) {
       let count = 0;
       for (const r of runs.values()) {
         if (
           r.automationId === automationId &&
-          ["pending", "running", "waiting"].includes(r.status)
+          ["pending", "running", "waiting"].includes(r.status) &&
+          (contextKey === undefined || r.contextKey === contextKey)
         ) {
           count += 1;
         }
       }
       return count;
     },
-    async hasActiveRun(automationId) {
-      return (await this.countActiveRuns(automationId)) > 0;
+    async hasActiveRun(automationId, contextKey?) {
+      return (await this.countActiveRuns(automationId, contextKey)) > 0;
     },
-    async cancelActiveRuns(automationId, reason) {
+    async cancelActiveRuns(automationId, reason, contextKey?) {
       const cancelled: string[] = [];
       for (const r of runs.values()) {
         if (
           r.automationId === automationId &&
-          ["pending", "running", "waiting"].includes(r.status)
+          ["pending", "running", "waiting"].includes(r.status) &&
+          (contextKey === undefined || r.contextKey === contextKey)
         ) {
           r.status = "cancelled";
           r.errorMessage = reason;
@@ -124,6 +141,19 @@ export function createInMemoryRunStore(): {
           cancelled.push(r.id);
         }
       }
+      // Faithful model of the real store: tear down the cancelled runs'
+      // wait locks in the same operation so a later wake can't resurrect
+      // them. (Run-state clearing happens via the optional hook wired in
+      // makeDispatchDeps.)
+      if (cancelled.length > 0) {
+        for (const [id, lock] of waitLocks) {
+          if (cancelled.includes(lock.runId)) {
+            waitLocks.delete(id);
+            wakeRefs.delete(id);
+          }
+        }
+        onCancel?.(cancelled);
+      }
       return cancelled;
     },
 
@@ -181,8 +211,26 @@ export function createInMemoryRunStore(): {
         contextKey: input.contextKey,
         filterTemplate: input.filterTemplate,
         timeoutAt: input.timeoutAt,
+        waitConfig: input.waitConfig ?? null,
+        createdAt: new Date(),
+      });
+      return id;
+    },
+    async createWaitLockWithWakeRefs(input: CreateWaitLockWithRefsInput) {
+      const id = `lock-${++lockCounter}`;
+      waitLocks.set(id, {
+        id,
+        runId: input.runId,
+        actionPath: input.actionPath,
+        kind: "until",
+        eventId: input.eventId,
+        contextKey: input.contextKey,
+        filterTemplate: null,
+        timeoutAt: input.timeoutAt,
+        waitConfig: input.waitConfig,
         createdAt: new Date(),
       });
+      wakeRefs.set(id, new Set(input.wakeRefs));
       return id;
     },
     async loadWaitLock(id) {
@@ -197,8 +245,27 @@ export function createInMemoryRunStore(): {
       }
       return matches;
     },
+    async findWaitLocksByWakeRef(ref) {
+      const colon = ref.indexOf(":");
+      const kind = colon === -1 ? ref : ref.slice(0, colon);
+      const wildcard = `${kind}:*`;
+      const matches: LoadedWaitLock[] = [];
+      for (const [lockId, refs] of wakeRefs) {
+        const lock = waitLocks.get(lockId);
+        if (!lock || lock.kind !== "until") continue;
+        if (refs.has(ref) || refs.has(wildcard)) matches.push(lock);
+      }
+      return matches;
+    },
+    async findWaitLocksByKind(kind) {
+      return [...waitLocks.values()].filter((lock) => lock.kind === kind);
+    },
+    async findWaitLocksByRun(runId) {
+      return [...waitLocks.values()].filter((lock) => lock.runId === runId);
+    },
     async deleteWaitLock(id) {
       waitLocks.delete(id);
+      wakeRefs.delete(id);
     },
     async sweepExpiredWaitLocks(now) {
       const expired: LoadedWaitLock[] = [];
@@ -211,7 +278,7 @@ export function createInMemoryRunStore(): {
     },
   };
 
-  return { store, runs, steps, waitLocks };
+  return { store, runs, steps, waitLocks, wakeRefs };
 }
 
 export function createInMemoryArtifactStore(): {
@@ -267,7 +334,15 @@ export function createInMemoryArtifactStore(): {
   return { store, artifacts };
 }
 
-export function createInMemoryRunStateStore(): {
+/**
+ * @param runs - the run store's `runs` map, so `findStalledRunIds`
+ *   faithfully models the real DB join that filters to `status =
+ *   'running'`. Without it the fake would skip the status filter and a
+ *   test couldn't observe the C1 fix (a `waiting` run must NOT be swept).
+ */
+export function createInMemoryRunStateStore(
+  runs?: Map<string, LoadedRun>,
+): {
   store: RunStateStore;
   states: Map<string, RunStateSnapshot>;
   locks: Set<string>;
@@ -277,9 +352,16 @@ export function createInMemoryRunStateStore(): {
 
   const store: RunStateStore = {
     async upsert(input) {
+      const existing = states.get(input.runId);
+      // Mirror the real store: omitting `lastActionPath` on an existing
+      // row preserves the prior checkpoint; passing it (incl. null) sets it.
+      const lastActionPath =
+        input.lastActionPath === undefined
+          ? (existing?.lastActionPath ?? null)
+          : input.lastActionPath;
       states.set(input.runId, {
         scopeSnapshot: input.scopeSnapshot,
-        lastActionPath: input.lastActionPath,
+        lastActionPath,
         lastHeartbeatAt: new Date(),
       });
     },
@@ -296,23 +378,169 @@ export function createInMemoryRunStateStore(): {
     async findStalledRunIds(threshold) {
       const ids: string[] = [];
       for (const [runId, s] of states.entries()) {
-        if (s.lastHeartbeatAt < threshold) ids.push(runId);
+        if (s.lastHeartbeatAt >= threshold) continue;
+        // Faithful model of the DB join: only `running` runs are stalled.
+        if (runs && runs.get(runId)?.status !== "running") continue;
+        ids.push(runId);
       }
       return ids;
     },
     async tryAdvisoryLock(runId) {
-      if (locks.has(runId)) return false;
+      if (locks.has(runId)) return null;
       locks.add(runId);
-      return true;
-    },
-    async releaseAdvisoryLock(runId) {
-      locks.delete(runId);
+      let released = false;
+      return {
+        async release() {
+          if (released) return;
+          released = true;
+          locks.delete(runId);
+        },
+      };
     },
   };
 
   return { store, states, locks };
 }
 
+export function createInMemoryDwellStore(): {
+  store: DwellStore;
+  dwells: Map<string, LoadedDwell>;
+} {
+  const dwells = new Map<string, LoadedDwell>();
+  let counter = 0;
+
+  const matchesKey = (
+    d: LoadedDwell,
+    automationId: string,
+    triggerId: string,
+    contextKey: string | null,
+  ) =>
+    d.automationId === automationId &&
+    d.triggerId === triggerId &&
+    d.contextKey === contextKey;
+
+  const store: DwellStore = {
+    async arm(input: UpsertDwellInput) {
+      // Insert-if-absent: preserve an existing dwell's original fireAt.
+      const existing = [...dwells.values()].find((d) =>
+        matchesKey(d, input.automationId, input.triggerId, input.contextKey),
+      );
+      if (existing) {
+        return { id: existing.id, created: false, fireAt: existing.fireAt };
+      }
+      const id = `dwell-${++counter}`;
+      dwells.set(id, {
+        id,
+        automationId: input.automationId,
+        triggerId: input.triggerId,
+        eventId: input.eventId,
+        contextKey: input.contextKey,
+        armedStatus: input.armedStatus,
+        payloadSnapshot: input.payloadSnapshot,
+        actorSnapshot: input.actorSnapshot,
+        fireAt: input.fireAt,
+        createdAt: new Date(),
+      });
+      return { id, created: true, fireAt: input.fireAt };
+    },
+    async load(id) {
+      return dwells.get(id);
+    },
+    async findByKey(automationId, triggerId, contextKey) {
+      return [...dwells.values()].find((d) =>
+        matchesKey(d, automationId, triggerId, contextKey),
+      );
+    },
+    async delete(id) {
+      // Map.delete returns true only if the key existed — a faithful
+      // model of the DB's `DELETE ... RETURNING` atomic claim.
+      return dwells.delete(id);
+    },
+    async deleteByKey(automationId, triggerId, contextKey) {
+      for (const [id, d] of dwells.entries()) {
+        if (matchesKey(d, automationId, triggerId, contextKey)) dwells.delete(id);
+      }
+    },
+    async deleteForAutomation(automationId) {
+      for (const [id, d] of dwells.entries()) {
+        if (d.automationId === automationId) dwells.delete(id);
+      }
+    },
+    async sweepExpired(now) {
+      return [...dwells.values()].filter(
+        (d) => d.fireAt.getTime() <= now.getTime(),
+      );
+    },
+  };
+
+  return { store, dwells };
+}
+
+/**
+ * In-memory `WindowStore` mirroring the SQL append-log semantics: each
+ * `recordAndCount` appends an occurrence, counts rows for the key within the
+ * trailing window (inclusive), and applies the re-fire policy. Faithful to
+ * `window-store.ts` so the gate's behaviour is testable without a DB.
+ */
+export function createInMemoryWindowStore(): {
+  store: WindowStore;
+  events: Array<{
+    automationId: string;
+    triggerId: string;
+    eventId: string;
+    contextKey: string | null;
+    occurredAt: Date;
+  }>;
+} {
+  const events: Array<{
+    automationId: string;
+    triggerId: string;
+    eventId: string;
+    contextKey: string | null;
+    occurredAt: Date;
+  }> = [];
+
+  const store: WindowStore = {
+    async recordAndCount(input: RecordWindowInput) {
+      const {
+        automationId,
+        triggerId,
+        eventId,
+        contextKey,
+        occurredAt,
+        windowMinutes,
+        threshold,
+        refire,
+      } = input;
+      events.push({ automationId, triggerId, eventId, contextKey, occurredAt });
+      const windowStart = occurredAt.getTime() - windowMinutes * 60_000;
+      const newCount = events.filter(
+        (e) =>
+          e.automationId === automationId &&
+          e.triggerId === triggerId &&
+          e.contextKey === contextKey &&
+          e.occurredAt.getTime() >= windowStart,
+      ).length;
+      if (refire === "once") return newCount === threshold;
+      return newCount >= threshold;
+    },
+    async sweepExpired(cutoff) {
+      for (let i = events.length - 1; i >= 0; i--) {
+        if (events[i]!.occurredAt.getTime() < cutoff.getTime()) {
+          events.splice(i, 1);
+        }
+      }
+    },
+    async deleteForAutomation(automationId) {
+      for (let i = events.length - 1; i >= 0; i--) {
+        if (events[i]!.automationId === automationId) events.splice(i, 1);
+      }
+    },
+  };
+
+  return { store, events };
+}
+
 /**
  * Minimal in-memory queue manager stub for engine tests. Records enqueued
  * jobs so a test can fire them synchronously to simulate the delay
@@ -329,7 +557,7 @@ export interface FakeQueueManager {
   fireAll: () => Promise<void>;
 }
 
-export function createFakeQueueManager(opts?: {
+function createFakeQueueManager(opts?: {
   onJob?: (queue: string, data: unknown) => Promise<void> | void;
 }): FakeQueueManager {
   const jobs: FakeQueueManager["jobs"] = [];
@@ -442,16 +670,45 @@ export function makeDispatchDeps(opts?: {
   actions?: ActionRegistry;
   artifactTypes?: ArtifactTypeRegistry;
   triggers?: TriggerRegistry;
+  /** Optional health-check client for sensing-layer enrichment tests. */
+  healthCheckClient?: DispatchDeps["healthCheckClient"];
+  /**
+   * Optional kind-agnostic entity resolver for reactive `wait_until` wake
+   * re-eval tests (resolves `state.<kind>.<id>` for non-health entity kinds).
+   */
+  entityResolverFor?: DispatchDeps["entityResolverFor"];
+  /**
+   * Wire a faithful in-memory serializing lock for the concurrency-mode
+   * check-then-create (models the real transaction-scoped advisory lock:
+   * keyed, blocks until granted). Off by default so the natural
+   * check-then-create race is observable in tests that don't opt in.
+   */
+  withConcurrencyLock?: boolean;
 }): {
   deps: DispatchDeps;
   runs: ReturnType<typeof createInMemoryRunStore>;
   artifacts: ReturnType<typeof createInMemoryArtifactStore>;
   state: ReturnType<typeof createInMemoryRunStateStore>;
+  dwells: ReturnType<typeof createInMemoryDwellStore>;
+  windows: ReturnType<typeof createInMemoryWindowStore>;
   queue: FakeQueueManager;
 } {
-  const runs = createInMemoryRunStore();
+  // `cancelActiveRuns` clears the cancelled runs' run-state rows too (the
+  // real store deletes wait locks + run-state in one op). The run-state map
+  // doesn't exist yet, so let the cancel hook reach it lazily via a holder.
+  const stateHolder: {
+    store?: ReturnType<typeof createInMemoryRunStateStore>;
+  } = {};
+  const runs = createInMemoryRunStore({
+    onCancel: (ids) => {
+      for (const id of ids) stateHolder.store?.states.delete(id);
+    },
+  });
   const artifacts = createInMemoryArtifactStore();
-  const state = createInMemoryRunStateStore();
+  const state = createInMemoryRunStateStore(runs.runs);
+  stateHolder.store = state;
+  const dwells = createInMemoryDwellStore();
+  const windows = createInMemoryWindowStore();
   const queue = createFakeQueueManager();
   const noopLogger = {
     debug: () => {},
@@ -470,12 +727,36 @@ export function makeDispatchDeps(opts?: {
     runStore: runs.store,
     artifactStore: artifacts.store,
     runStateStore: state.store,
+    dwellStore: dwells.store,
+    windowStore: windows.store,
     queueManager: queue.manager,
+    healthCheckClient: opts?.healthCheckClient,
+    entityResolverFor: opts?.entityResolverFor,
     getService: async () => {
       throw new Error("getService not stubbed for this test");
     },
   };
-  return { deps, runs, artifacts, state, queue };
+  if (opts?.withConcurrencyLock) {
+    // A faithful keyed async mutex: a second caller for the same key awaits
+    // the first's completion, exactly like pg_advisory_xact_lock blocking
+    // until COMMIT. Distinct keys never contend.
+    const chains = new Map<string, Promise<unknown>>();
+    deps.withConcurrencyLock = <T>(key: string, fn: () => Promise<T>) => {
+      const prior = chains.get(key) ?? Promise.resolve();
+      const next = prior.then(() => fn());
+      // Keep the chain alive even if fn rejects, so the lock still releases
+      // (catch swallows both outcomes into a settled void promise).
+      chains.set(
+        key,
+        next.then(
+          () => {},
+          () => {},
+        ),
+      );
+      return next;
+    };
+  }
+  return { deps, runs, artifacts, state, dwells, windows, queue };
 }
 
 // ─── Shared fixtures ────────────────────────────────────────────────────
@@ -543,16 +824,3 @@ export function makeFailingAction(): ActionDefinition<{ reason: string }> {
     }),
   };
 }
-
-/** Hook used by tests that need a registered hook reference. */
-export const testHook = createHook<{ id: string }>("test.event");
-
-export function makeTrigger(): TriggerDefinition<{ id: string }> {
-  return {
-    id: "event",
-    displayName: "Test event",
-    payloadSchema: z.object({ id: z.string() }),
-    hook: testHook,
-    contextKey: (p) => p.id,
-  };
-}

package/src/dispatch/trigger-fanin.test.ts

@@ -0,0 +1,111 @@
+import { describe, expect, it } from "bun:test";
+import { z } from "zod";
+import { createHook, type HookSubscribeOptions } from "@checkstack/backend-api";
+import type { TriggerDefinition } from "../action-types";
+import { createTriggerRegistry } from "../trigger-registry";
+import type { AutomationStore } from "../automation-store";
+import {
+  setupTriggerSubscriptions,
+  type OnHookFn,
+} from "./trigger-subscriber";
+import { makeDispatchDeps } from "./test-fixtures";
+
+/**
+ * Tier-1 contract regression guard for the horizontal-scale fan-in.
+ *
+ * Every hook-backed trigger MUST be subscribed in `mode: "work-queue"` with
+ * a per-trigger `workerGroup`. That is what makes exactly ONE pod consume a
+ * given trigger emission across the fleet; dropping it would silently
+ * revert to broadcast delivery, double-firing every automation once per
+ * running pod.
+ *
+ * This guard can't prove exactly-once delivery (that needs a real BullMQ
+ * work queue — Tier 2), but it catches an accidental removal / mistyping of
+ * the work-queue options at the wiring boundary.
+ */
+
+function triggerFor(id: string): TriggerDefinition<{ id: string }> {
+  return {
+    id,
+    displayName: id,
+    payloadSchema: z.object({ id: z.string() }),
+    hook: createHook<{ id: string }>(`test.${id}`),
+    contextKey: (p) => p.id,
+  };
+}
+
+const emptyStore: AutomationStore = {
+  create: async () => {
+    throw new Error("nope");
+  },
+  update: async () => {
+    throw new Error("nope");
+  },
+  delete: async () => {},
+  toggle: async () => {
+    throw new Error("nope");
+  },
+  getById: async () => undefined,
+  list: async () => ({ items: [], total: 0 }),
+  listGroups: async () => [],
+  findEnabledByTriggerEvent: async () => [],
+  listEnabled: async () => [],
+};
+
+const noopLogger = {
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as unknown as Parameters<typeof setupTriggerSubscriptions>[0]["logger"];
+
+describe("trigger fan-in — single-consumer (work-queue) contract", () => {
+  it("subscribes every hook-backed trigger in work-queue mode with a per-trigger workerGroup", async () => {
+    const triggers = createTriggerRegistry();
+    triggers.register(triggerFor("alpha"), { pluginId: "plug" });
+    triggers.register(triggerFor("beta"), { pluginId: "plug" });
+
+    const { deps } = makeDispatchDeps({ triggers });
+
+    // Spy onHook: capture (hookId, options) per subscription and hand back a
+    // no-op teardown so setup completes.
+    const captured: Array<{
+      hookId: string;
+      options: HookSubscribeOptions | undefined;
+    }> = [];
+    const onHook: OnHookFn = (hook, _listener, options) => {
+      captured.push({ hookId: hook.id, options });
+      return async () => {};
+    };
+
+    await setupTriggerSubscriptions({
+      deps,
+      onHook,
+      automationStore: emptyStore,
+      logger: noopLogger,
+    });
+
+    expect(captured).toHaveLength(2);
+    for (const { hookId, options } of captured) {
+      expect(options).toEqual({
+        mode: "work-queue",
+        workerGroup: `automation-trigger-${qualifiedIdForHook(hookId)}`,
+      });
+    }
+
+    // Each per-trigger workerGroup is distinct so the two triggers fan in on
+    // independent consumer groups (a shared group would serialize unrelated
+    // triggers).
+    const groups = captured.map((c) => c.options?.workerGroup);
+    expect(new Set(groups).size).toBe(2);
+  });
+});
+
+/**
+ * The hook id is `test.<triggerId>` and the qualified id is
+ * `plug.<triggerId>`; map one to the other for the assertion.
+ */
+function qualifiedIdForHook(hookId: string): string {
+  const triggerId = hookId.replace(/^test\./, "");
+  return `plug.${triggerId}`;
+}