@checkstack/automation-backend 0.2.0

package/src/dispatch/run-state.ts

@@ -0,0 +1,298 @@
+/**
+ * Drizzle-backed implementation of `RunStore`. The dispatch engine uses
+ * this for every run / step / wait-lock write so durability survives
+ * process restarts.
+ *
+ * Kept thin: each method maps almost 1:1 to a DB statement. Concurrency
+ * and consistency concerns live in the calling code (the dispatcher and
+ * trigger subscriber).
+ */
+import { and, desc, eq, inArray, isNotNull, isNull, lte, sql } from "drizzle-orm";
+import type { SafeDatabase } from "@checkstack/backend-api";
+
+import {
+  automationRunSteps,
+  automationRuns,
+  automationWaitLocks,
+} from "../schema";
+import type {
+  CreateRunInput,
+  CreateStepInput,
+  CreateWaitLockInput,
+  LoadedRun,
+  LoadedStep,
+  LoadedWaitLock,
+  RunStore,
+} from "./types";
+
+type Schema = {
+  automationRuns: typeof automationRuns;
+  automationRunSteps: typeof automationRunSteps;
+  automationWaitLocks: typeof automationWaitLocks;
+};
+
+const ACTIVE_STATUSES = ["pending", "running", "waiting"] as const;
+
+export function createRunStore(db: SafeDatabase<Schema>): RunStore {
+  return {
+    async createRun(input: CreateRunInput): Promise<string> {
+      const [row] = await db
+        .insert(automationRuns)
+        .values({
+          automationId: input.automationId,
+          triggerId: input.triggerId,
+          triggerEventId: input.triggerEventId,
+          triggerPayload: input.triggerPayload,
+          contextKey: input.contextKey,
+          status: "running",
+        })
+        .returning({ id: automationRuns.id });
+      if (!row) throw new Error("createRun: insert returned no rows");
+      return row.id;
+    },
+
+    async updateRunStatus(runId, status, errorMessage): Promise<void> {
+      const isTerminal =
+        status === "success" ||
+        status === "failed" ||
+        status === "cancelled" ||
+        status === "skipped";
+      await db
+        .update(automationRuns)
+        .set({
+          status,
+          errorMessage: errorMessage ?? null,
+          finishedAt: isTerminal ? new Date() : null,
+        })
+        .where(eq(automationRuns.id, runId));
+    },
+
+    async loadRun(runId: string): Promise<LoadedRun | undefined> {
+      const rows = await db
+        .select()
+        .from(automationRuns)
+        .where(eq(automationRuns.id, runId))
+        .limit(1);
+      const row = rows[0];
+      if (!row) return undefined;
+      return {
+        id: row.id,
+        automationId: row.automationId,
+        triggerId: row.triggerId,
+        triggerEventId: row.triggerEventId,
+        triggerPayload: row.triggerPayload,
+        contextKey: row.contextKey,
+        status: row.status,
+        errorMessage: row.errorMessage,
+        startedAt: row.startedAt,
+        finishedAt: row.finishedAt,
+      };
+    },
+
+    async countActiveRuns(automationId: string): Promise<number> {
+      const rows = await db
+        .select({ count: sql<number>`count(*)::int` })
+        .from(automationRuns)
+        .where(
+          and(
+            eq(automationRuns.automationId, automationId),
+            inArray(automationRuns.status, [...ACTIVE_STATUSES]),
+          ),
+        );
+      return rows[0]?.count ?? 0;
+    },
+
+    async hasActiveRun(automationId: string): Promise<boolean> {
+      const rows = await db
+        .select({ id: automationRuns.id })
+        .from(automationRuns)
+        .where(
+          and(
+            eq(automationRuns.automationId, automationId),
+            inArray(automationRuns.status, [...ACTIVE_STATUSES]),
+          ),
+        )
+        .limit(1);
+      return rows.length > 0;
+    },
+
+    async cancelActiveRuns(
+      automationId: string,
+      reason: string,
+    ): Promise<string[]> {
+      const rows = await db
+        .update(automationRuns)
+        .set({
+          status: "cancelled",
+          errorMessage: reason,
+          finishedAt: new Date(),
+        })
+        .where(
+          and(
+            eq(automationRuns.automationId, automationId),
+            inArray(automationRuns.status, [...ACTIVE_STATUSES]),
+          ),
+        )
+        .returning({ id: automationRuns.id });
+      return rows.map((r) => r.id);
+    },
+
+    async createStep(input: CreateStepInput): Promise<string> {
+      const [row] = await db
+        .insert(automationRunSteps)
+        .values({
+          runId: input.runId,
+          actionPath: input.actionPath,
+          actionId: input.actionId,
+          actionKind: input.actionKind,
+          providerActionId: input.providerActionId,
+          status: "running",
+          attempts: 1,
+        })
+        .returning({ id: automationRunSteps.id });
+      if (!row) throw new Error("createStep: insert returned no rows");
+      return row.id;
+    },
+
+    async updateStep(stepId, patch): Promise<void> {
+      const isTerminal =
+        patch.status === "success" ||
+        patch.status === "failed" ||
+        patch.status === "skipped";
+      const set: Record<string, unknown> = {
+        status: patch.status,
+        errorMessage: patch.errorMessage ?? null,
+        resultPayload: patch.resultPayload ?? null,
+      };
+      if (isTerminal) set.finishedAt = new Date();
+      if (patch.incrementAttempts) {
+        set.attempts = sql`${automationRunSteps.attempts} + 1`;
+      }
+      await db
+        .update(automationRunSteps)
+        .set(set)
+        .where(eq(automationRunSteps.id, stepId));
+    },
+
+    async findStepByPath(runId, actionPath): Promise<LoadedStep | undefined> {
+      const rows = await db
+        .select()
+        .from(automationRunSteps)
+        .where(
+          and(
+            eq(automationRunSteps.runId, runId),
+            eq(automationRunSteps.actionPath, actionPath),
+          ),
+        )
+        .orderBy(desc(automationRunSteps.startedAt))
+        .limit(1);
+      const row = rows[0];
+      if (!row) return;
+      return {
+        id: row.id,
+        runId: row.runId,
+        actionPath: row.actionPath,
+        actionId: row.actionId,
+        actionKind: row.actionKind,
+        status: row.status,
+        attempts: row.attempts,
+        errorMessage: row.errorMessage,
+        resultPayload: row.resultPayload,
+        startedAt: row.startedAt,
+        finishedAt: row.finishedAt,
+      };
+    },
+
+    async createWaitLock(input: CreateWaitLockInput): Promise<string> {
+      const [row] = await db
+        .insert(automationWaitLocks)
+        .values({
+          runId: input.runId,
+          actionPath: input.actionPath,
+          kind: input.kind,
+          eventId: input.eventId,
+          contextKey: input.contextKey,
+          filterTemplate: input.filterTemplate,
+          timeoutAt: input.timeoutAt,
+        })
+        .returning({ id: automationWaitLocks.id });
+      if (!row) throw new Error("createWaitLock: insert returned no rows");
+      return row.id;
+    },
+
+    async loadWaitLock(id) {
+      const rows = await db
+        .select()
+        .from(automationWaitLocks)
+        .where(eq(automationWaitLocks.id, id))
+        .limit(1);
+      const row = rows[0];
+      if (!row) return;
+      return {
+        id: row.id,
+        runId: row.runId,
+        actionPath: row.actionPath,
+        kind: row.kind as "trigger" | "delay",
+        eventId: row.eventId,
+        contextKey: row.contextKey,
+        filterTemplate: row.filterTemplate,
+        timeoutAt: row.timeoutAt,
+        createdAt: row.createdAt,
+      };
+    },
+
+    async findWaitLocksFor(
+      eventId: string,
+      contextKey: string | null,
+    ): Promise<LoadedWaitLock[]> {
+      const filters = [
+        eq(automationWaitLocks.eventId, eventId),
+        contextKey === null
+          ? isNull(automationWaitLocks.contextKey)
+          : eq(automationWaitLocks.contextKey, contextKey),
+      ];
+      const rows = await db
+        .select()
+        .from(automationWaitLocks)
+        .where(and(...filters));
+      return rows.map((r) => ({
+        id: r.id,
+        runId: r.runId,
+        actionPath: r.actionPath,
+        kind: r.kind as "trigger" | "delay",
+        eventId: r.eventId,
+        contextKey: r.contextKey,
+        filterTemplate: r.filterTemplate,
+        timeoutAt: r.timeoutAt,
+        createdAt: r.createdAt,
+      }));
+    },
+
+    async deleteWaitLock(id: string): Promise<void> {
+      await db.delete(automationWaitLocks).where(eq(automationWaitLocks.id, id));
+    },
+
+    async sweepExpiredWaitLocks(now: Date): Promise<LoadedWaitLock[]> {
+      const rows = await db
+        .select()
+        .from(automationWaitLocks)
+        .where(
+          and(
+            isNotNull(automationWaitLocks.timeoutAt),
+            lte(automationWaitLocks.timeoutAt, now),
+          ),
+        );
+      return rows.map((r) => ({
+        id: r.id,
+        runId: r.runId,
+        actionPath: r.actionPath,
+        kind: r.kind as "trigger" | "delay",
+        eventId: r.eventId,
+        contextKey: r.contextKey,
+        filterTemplate: r.filterTemplate,
+        timeoutAt: r.timeoutAt,
+        createdAt: r.createdAt,
+      }));
+    },
+  };
+}

package/src/dispatch/scope.test.ts

@@ -0,0 +1,40 @@
+import { describe, it, expect } from "bun:test";
+import { SYSTEM_ACTOR } from "@checkstack/common";
+import { buildInitialScope } from "./scope";
+
+describe("buildInitialScope — trigger.actor", () => {
+  const base = {
+    triggerId: "t1",
+    triggerEventId: "incident.created",
+    payload: { incidentId: "i1" },
+    startedAt: new Date("2026-05-30T00:00:00.000Z"),
+  };
+
+  it("defaults trigger.actor to the system actor when none is supplied", () => {
+    const scope = buildInitialScope(base);
+    const trigger = scope.trigger as { actor: unknown };
+    expect(trigger.actor).toEqual(SYSTEM_ACTOR);
+  });
+
+  it("exposes the supplied actor under trigger.actor alongside the payload", () => {
+    const actor = { type: "user", id: "user-1", name: "Nico" } as const;
+    const scope = buildInitialScope({ ...base, actor });
+    const trigger = scope.trigger as { actor: unknown; payload: unknown };
+    expect(trigger.actor).toEqual(actor);
+    expect(trigger.payload).toEqual({ incidentId: "i1" });
+  });
+
+  it("exposes trigger.id and the canonical trigger.event (with eventId alias)", () => {
+    const scope = buildInitialScope(base);
+    const trigger = scope.trigger as {
+      id: unknown;
+      event: unknown;
+      eventId: unknown;
+    };
+    expect(trigger.id).toBe("t1");
+    // `event` is canonical (matches the editor + script contract); `eventId`
+    // stays as a back-compat alias.
+    expect(trigger.event).toBe("incident.created");
+    expect(trigger.eventId).toBe("incident.created");
+  });
+});

package/src/dispatch/scope.ts

@@ -0,0 +1,125 @@
+/**
+ * Variable scope construction for the dispatch engine.
+ *
+ * The shape exposed to templates is:
+ *
+ *   trigger.id, trigger.event, trigger.payload.*
+ *   trigger.actor.{type,id,name}                 (who/what caused the event)
+ *   variables.*                                  (from `variables` blocks)
+ *   artifacts.<actionId>.<localArtifactName>.*  (set when an action produces)
+ *   repeat.item, repeat.index                    (only inside a repeat)
+ *   now (helper, ISO string of dispatch start)
+ *
+ * Keep this shape stable — the editor's intellisense reads it.
+ */
+import { SYSTEM_ACTOR, type Actor } from "@checkstack/common";
+import type { DispatchContext } from "./types";
+
+/**
+ * Build the initial scope at run start. Subsequent blocks (variables,
+ * repeat) clone-and-extend this scope rather than mutating it, so a
+ * variable defined inside a nested block does not leak to siblings.
+ *
+ * `actor` carries who/what caused the originating event (a user, an
+ * application/API client, a service, or the system). It defaults to the
+ * system actor so callers that don't have one still produce a complete scope.
+ */
+export function buildInitialScope(args: {
+  triggerId: string;
+  triggerEventId: string;
+  payload: Record<string, unknown>;
+  actor?: Actor;
+  startedAt: Date;
+}): Record<string, unknown> {
+  return {
+    trigger: {
+      id: args.triggerId,
+      event: args.triggerEventId,
+      // Back-compat alias for the former internal key. Templates and the
+      // editor use `trigger.event`; `eventId` stays so older saved scope
+      // snapshots / automations referencing it keep resolving.
+      eventId: args.triggerEventId,
+      actor: args.actor ?? SYSTEM_ACTOR,
+      payload: args.payload,
+    },
+    variables: {},
+    artifacts: {},
+    now: args.startedAt.toISOString(),
+  };
+}
+
+/**
+ * Clone a scope, optionally extending it with new fields. Used to push
+ * down a child scope when entering nested blocks (variables, repeat
+ * iteration body). The structural clone is shallow at the top level —
+ * inner objects are shared, which is fine because primitives never
+ * mutate them.
+ */
+export function extendScope(
+  scope: Record<string, unknown>,
+  patch: Record<string, unknown>,
+): Record<string, unknown> {
+  return { ...scope, ...patch };
+}
+
+/**
+ * Push `variables.*` updates. Variable defs run before any downstream
+ * action, so we merge into the existing `variables` namespace.
+ */
+export function extendVariables(
+  scope: Record<string, unknown>,
+  newVars: Record<string, unknown>,
+): Record<string, unknown> {
+  const existing = (scope.variables as Record<string, unknown>) ?? {};
+  return { ...scope, variables: { ...existing, ...newVars } };
+}
+
+/**
+ * Push `repeat.*` info onto the scope for the iteration body.
+ */
+export function withRepeatContext(
+  scope: Record<string, unknown>,
+  repeat: { index: number; item?: unknown },
+): Record<string, unknown> {
+  return { ...scope, repeat };
+}
+
+/**
+ * Resolve and attach upstream artifacts the calling action declared in
+ * `consumes`. Looks up each type within the current run's scope and
+ * exposes the data under `artifacts.<type>`.
+ *
+ * Conflict policy: if multiple actions in the same automation produced
+ * the same artifact type, the most-recent open artifact wins. Operators
+ * who want explicit producer pinning should reference by action id in a
+ * template (`artifacts.<id>.<name>`), which the engine auto-populates for
+ * every producing action.
+ */
+export async function resolveConsumedArtifacts(
+  ctx: DispatchContext,
+  consumes: ReadonlyArray<string>,
+  ownerPluginId: string,
+): Promise<Record<string, unknown>> {
+  if (consumes.length === 0) return {};
+  const result: Record<string, unknown> = {};
+
+  for (const localType of consumes) {
+    // `consumes` carries local artifact ids; the stored artifact type is
+    // the fully-qualified `${pluginId}.${id}` the producing action wrote.
+    // Qualify against the consuming action's own plugin (same-plugin
+    // handoff) for the lookup, but key the result by the local id so the
+    // action's `execute` reads `consumedArtifacts[localId]`.
+    const qualifiedType = `${ownerPluginId}.${localType}`;
+    const found = await ctx.deps.artifactStore.find({
+      automationId: ctx.run.automation.id,
+      contextKey: ctx.run.contextKey,
+      artifactType: qualifiedType,
+      onlyOpen: true,
+    });
+    if (found) {
+      result[localType] = found.data;
+    }
+  }
+
+  return result;
+}

package/src/dispatch/stalled-sweeper.ts

@@ -0,0 +1,164 @@
+/**
+ * Stalled-run sweeper.
+ *
+ * Periodically scans for runs whose heartbeat is older than a
+ * configurable threshold and resumes them. Combined with the per-run
+ * Postgres advisory lock, this gives the platform restart safety + safe
+ * horizontal scaling: when an instance crashes mid-execution, another
+ * instance picks up the dropped runs after the heartbeat threshold
+ * elapses.
+ *
+ * Also sweeps expired wait locks:
+ *   - `kind: "delay"` locks past `timeoutAt` resume the run (in case
+ *     the queue scheduler lost the job).
+ *   - `kind: "trigger"` locks past `timeoutAt` fail the run with a
+ *     clear "wait timed out" error.
+ */
+import type { Logger } from "@checkstack/backend-api";
+
+import type { AutomationStore } from "../automation-store";
+import { recoverStalledRun, resumeRun } from "./engine";
+import type { DispatchDeps } from "./types";
+
+export interface StalledSweeperArgs {
+  deps: DispatchDeps;
+  automationStore: AutomationStore;
+  logger: Logger;
+  /** Heartbeat age (ms) above which a run is considered stalled. */
+  staleAfterMs?: number;
+  /** Poll interval (ms). */
+  intervalMs?: number;
+}
+
+export interface StalledSweeper {
+  /** Run one sweep cycle. Useful in tests. */
+  sweep: () => Promise<void>;
+  /** Stop the periodic polling. */
+  stop: () => void;
+}
+
+const DEFAULT_STALE_MS = 60_000; // 1 minute
+const DEFAULT_INTERVAL_MS = 30_000; // every 30 seconds
+
+export function startStalledSweeper(
+  args: StalledSweeperArgs,
+): StalledSweeper {
+  const staleMs = args.staleAfterMs ?? DEFAULT_STALE_MS;
+  const intervalMs = args.intervalMs ?? DEFAULT_INTERVAL_MS;
+
+  const sweep = async (): Promise<void> => {
+    await sweepStalledRuns(args, staleMs);
+    await sweepExpiredWaitLocks(args);
+  };
+
+  let timer: ReturnType<typeof setInterval> | undefined = setInterval(() => {
+    sweep().catch((error) => {
+      args.logger.warn(
+        `automation stalled sweeper failed: ${(error as Error).message}`,
+      );
+    });
+  }, intervalMs);
+
+  return {
+    sweep,
+    stop: () => {
+      if (timer) {
+        clearInterval(timer);
+        timer = undefined;
+      }
+    },
+  };
+}
+
+async function sweepStalledRuns(
+  args: StalledSweeperArgs,
+  staleMs: number,
+): Promise<void> {
+  const threshold = new Date(Date.now() - staleMs);
+  const stalled = await args.deps.runStateStore.findStalledRunIds(threshold);
+  if (stalled.length === 0) return;
+  args.logger.debug(
+    `automation sweeper: ${stalled.length} stalled run(s) detected`,
+  );
+
+  for (const runId of stalled) {
+    const acquired = await args.deps.runStateStore.tryAdvisoryLock(runId);
+    if (!acquired) continue; // another instance already on it
+    try {
+      const run = await args.deps.runStore.loadRun(runId);
+      if (!run) continue;
+      const automation = await args.automationStore.getById(run.automationId);
+      if (!automation) {
+        await args.deps.runStore.updateRunStatus(
+          runId,
+          "failed",
+          "automation deleted while run was stalled",
+        );
+        await args.deps.runStateStore.clear(runId);
+        continue;
+      }
+      args.logger.info(`automation sweeper: recovering run ${runId}`);
+      await recoverStalledRun(args.deps, {
+        runId,
+        automation: {
+          id: automation.id,
+          name: automation.name,
+          status: automation.status,
+          definition: automation.definition,
+        },
+      });
+    } catch (error) {
+      args.logger.warn(
+        `automation sweeper failed to recover ${runId}: ${(error as Error).message}`,
+      );
+    } finally {
+      await args.deps.runStateStore.releaseAdvisoryLock(runId);
+    }
+  }
+}
+
+async function sweepExpiredWaitLocks(
+  args: StalledSweeperArgs,
+): Promise<void> {
+  const now = new Date();
+  const expired = await args.deps.runStore.sweepExpiredWaitLocks(now);
+  if (expired.length === 0) return;
+
+  for (const lock of expired) {
+    if (lock.kind === "delay") {
+      // The queue scheduler may have lost the job — wake the run
+      // ourselves. Idempotent: resumeRun takes the advisory lock and
+      // skips if someone else already resumed.
+      const run = await args.deps.runStore.loadRun(lock.runId);
+      if (!run) {
+        await args.deps.runStore.deleteWaitLock(lock.id);
+        continue;
+      }
+      const automation = await args.automationStore.getById(run.automationId);
+      if (!automation) {
+        await args.deps.runStore.deleteWaitLock(lock.id);
+        continue;
+      }
+      await args.deps.runStore.deleteWaitLock(lock.id);
+      await resumeRun(args.deps, {
+        runId: lock.runId,
+        automation: {
+          id: automation.id,
+          name: automation.name,
+          status: automation.status,
+          definition: automation.definition,
+        },
+        waitedAtPath: lock.actionPath,
+      });
+      continue;
+    }
+    // Trigger lock expired without firing — fail the run.
+    await args.deps.runStore.deleteWaitLock(lock.id);
+    await args.deps.runStore.updateRunStatus(
+      lock.runId,
+      "failed",
+      `wait_for_trigger timed out waiting for ${lock.eventId}`,
+    );
+    await args.deps.runStateStore.clear(lock.runId);
+  }
+}