@checkstack/automation-backend 0.2.0 → 0.3.0

package/src/dispatch/types.ts

@@ -5,9 +5,14 @@
  * `@checkstack/automation-common` and the package's index re-exports.
  */
 import type { Logger, ServiceRef } from "@checkstack/backend-api";
-import type { AutomationDefinition } from "@checkstack/automation-common";
+import type {
+  AutomationDefinition,
+  Condition,
+} from "@checkstack/automation-common";
 import type { QueueManager } from "@checkstack/queue-api";
 import type { FilterRegistry } from "@checkstack/template-engine";
+import type { InferClient } from "@checkstack/common";
+import type { HealthCheckApi } from "@checkstack/healthcheck-common";
 
 import type { ActionRegistry } from "../action-registry";
 import type { ArtifactTypeRegistry } from "../artifact-type-registry";
@@ -15,6 +20,8 @@ import type { TriggerRegistry } from "../trigger-registry";
 import type { ArtifactStore } from "../artifact-store";
 
 import type { RunStateStore } from "./run-state-store";
+import type { RunSecretRegistry } from "./run-secret-registry";
+import type { EntityKindResolver } from "./state-scope";
 
 /**
  * Persistent dependency bundle threaded through the dispatch engine.
@@ -40,6 +47,52 @@ export interface DispatchDeps {
    * and any future queue-backed continuations.
    */
   queueManager: QueueManager;
+  /**
+   * Health-check client for the sensing layer's scope pre-resolution
+   * (`enrichScopeWithState`) and the `for:` dwell re-confirm. Optional so
+   * test harnesses and minimal installs that don't sense live state can
+   * omit it — enrichment then fails open to an empty `health` namespace,
+   * and a dwell with no client re-confirms without a status gate.
+   */
+  healthCheckClient?: InferClient<typeof HealthCheckApi>;
+  /**
+   * Kind-agnostic entity resolver for reactive `wait_until` wake re-eval
+   * (reactive automation engine §3.6, §8). On wake, the engine re-enriches
+   * scope with EVERY `state.<kind>.<id>` ref the wait depends on, resolved
+   * through the framework entity store. Returns a batched `getMany`-style
+   * resolver for a registered kind, or `undefined` for an unknown kind (the
+   * enrichment then leaves that kind unresolved, fail-open). Optional so test
+   * harnesses / minimal installs without the entity store omit it — non-health
+   * waits then can't re-resolve their state (health waits still work via
+   * `healthCheckClient`).
+   */
+  entityResolverFor?: (kind: string) => EntityKindResolver | undefined;
+  /** Persistence backend for pre-run `for:` dwell timers. */
+  dwellStore: DwellStore;
+  /** Persistence backend for windowed-count / rate trigger gates. */
+  windowStore: WindowStore;
+  /**
+   * Run-scoped secret registry. When set, the engine wraps each run's
+   * `getService` so resolving the secret resolver / connection store
+   * registers the resolved values, and the run store masks step / run
+   * output before persistence. Optional so tests / minimal installs skip
+   * masking. `secretResolverRefId` / `connectionStoreRefId` carry the
+   * service-ref ids to intercept (passed as strings to avoid a dependency
+   * cycle from the dispatch core onto the secrets / integration packages).
+   */
+  secretRegistry?: RunSecretRegistry;
+  secretResolverRefId?: string;
+  connectionStoreRefId?: string;
+  /**
+   * Serialize a short critical section under a transaction-scoped advisory
+   * lock keyed on `key` (blocks until granted, auto-releases at COMMIT).
+   * Used to make the concurrency-mode check-then-create atomic per
+   * `(automationId, scopeKey)` so two concurrent fires / a dwell-fire /
+   * cross-pod can't both pass a `single`-mode "no active run" check and
+   * both create a run. Optional so tests / minimal installs degrade to the
+   * unserialized check (correct under a single in-process caller).
+   */
+  withConcurrencyLock?: <T>(key: string, fn: () => Promise<T>) => Promise<T>;
 }
 
 /**
@@ -152,11 +205,27 @@ export interface RunStore {
     errorMessage?: string,
   ): Promise<void>;
   loadRun(runId: string): Promise<LoadedRun | undefined>;
-  countActiveRuns(automationId: string): Promise<number>;
-  /** Used by `mode: "single"` to detect a pre-existing run. */
-  hasActiveRun(automationId: string): Promise<boolean>;
-  /** Used by `mode: "restart"` to abort prior runs. */
-  cancelActiveRuns(automationId: string, reason: string): Promise<string[]>;
+  /**
+   * Active-run count. When `contextKey` is omitted, counts across the
+   * whole automation (the default per-automation concurrency scope);
+   * when provided (including `null`), counts only runs with that context
+   * key (the per-context-key scope).
+   */
+  countActiveRuns(
+    automationId: string,
+    contextKey?: string | null,
+  ): Promise<number>;
+  /** Used by `mode: "single"` to detect a pre-existing run. See `countActiveRuns` for `contextKey`. */
+  hasActiveRun(
+    automationId: string,
+    contextKey?: string | null,
+  ): Promise<boolean>;
+  /** Used by `mode: "restart"` to abort prior runs. See `countActiveRuns` for `contextKey`. */
+  cancelActiveRuns(
+    automationId: string,
+    reason: string,
+    contextKey?: string | null,
+  ): Promise<string[]>;
 
   // Steps
   createStep(input: CreateStepInput): Promise<string>;
@@ -180,13 +249,35 @@ export interface RunStore {
     actionPath: string,
   ): Promise<LoadedStep | undefined>;
 
-  // Wait locks (for wait_for_trigger + delay durability)
+  // Wait locks (for wait_for_trigger + delay + wait_until durability)
   createWaitLock(input: CreateWaitLockInput): Promise<string>;
+  /**
+   * Insert a `kind: "until"` wait lock PLUS its wake-index dependency rows
+   * (one per ref) atomically (reactive automation engine §8.2). The
+   * `(waitLockId, ref)` pair is unique, so duplicate refs collapse via
+   * `ON CONFLICT DO NOTHING`. Returns the new wait-lock id.
+   */
+  createWaitLockWithWakeRefs(input: CreateWaitLockWithRefsInput): Promise<string>;
   loadWaitLock(id: string): Promise<LoadedWaitLock | undefined>;
   findWaitLocksFor(
     eventId: string,
     contextKey: string | null,
   ): Promise<LoadedWaitLock[]>;
+  /**
+   * Wake-index intersection lookup (reactive automation engine §8.2): every
+   * `kind: "until"` wait lock that depends on `ref` (exact match) OR on the
+   * kind-level wildcard for `ref`'s kind. Generalizes `findWaitLocksFor`.
+   */
+  findWaitLocksByWakeRef(ref: string): Promise<LoadedWaitLock[]>;
+  /** All wait locks of a given kind — powers the sweeper's `until` re-tick. */
+  findWaitLocksByKind(kind: WaitLockKind): Promise<LoadedWaitLock[]>;
+  /**
+   * All wait locks belonging to a run. Used by stalled-recovery to detect
+   * a live wait (refuse to from-top recover a genuinely-suspended run) and
+   * to delete leftover locks before a from-top re-walk (so recovery
+   * doesn't leak a second lock + duplicate delay job).
+   */
+  findWaitLocksByRun(runId: string): Promise<LoadedWaitLock[]>;
   deleteWaitLock(id: string): Promise<void>;
   sweepExpiredWaitLocks(now: Date): Promise<LoadedWaitLock[]>;
 }
@@ -234,7 +325,17 @@ export interface LoadedStep {
   finishedAt: Date | null;
 }
 
-export type WaitLockKind = "trigger" | "delay";
+export type WaitLockKind = "trigger" | "delay" | "until";
+
+/**
+ * Persisted config for a `kind: "until"` wait lock. The condition can be a
+ * structured object, so it rides in the lock's `waitConfig` jsonb rather
+ * than `filterTemplate`.
+ */
+export interface UntilWaitConfig {
+  condition: Condition;
+  continueOnTimeout: boolean;
+}
 
 export interface CreateWaitLockInput {
   runId: string;
@@ -244,6 +345,25 @@ export interface CreateWaitLockInput {
   contextKey: string | null;
   filterTemplate: string | null;
   timeoutAt: Date | null;
+  /** Only set for `kind: "until"`. */
+  waitConfig?: UntilWaitConfig | null;
+}
+
+/**
+ * A reactive `wait_until` suspend: the wait lock plus the wake-index refs
+ * its condition depends on (reactive automation engine §8.2). `kind` is
+ * always `"until"`; the refs are the `${kind}:${id}` (or `${kind}:*`)
+ * strings the condition reads.
+ */
+export interface CreateWaitLockWithRefsInput {
+  runId: string;
+  actionPath: string;
+  eventId: string;
+  contextKey: string | null;
+  timeoutAt: Date | null;
+  waitConfig: UntilWaitConfig;
+  /** Wake-index dependency refs (`${kind}:${id}` or `${kind}:*`). */
+  wakeRefs: ReadonlyArray<string>;
 }
 
 export interface LoadedWaitLock {
@@ -255,5 +375,140 @@ export interface LoadedWaitLock {
   contextKey: string | null;
   filterTemplate: string | null;
   timeoutAt: Date | null;
+  waitConfig: UntilWaitConfig | null;
+  createdAt: Date;
+}
+
+// ─── Dwell-timer store interface ─────────────────────────────────────────
+
+/** Candidate dwell to arm. `fireAt` is used only on a fresh INSERT. */
+export interface UpsertDwellInput {
+  automationId: string;
+  triggerId: string;
+  eventId: string;
+  contextKey: string | null;
+  armedStatus: string | null;
+  payloadSnapshot: Record<string, unknown>;
+  actorSnapshot: Record<string, unknown>;
+  fireAt: Date;
+}
+
+export interface LoadedDwell {
+  id: string;
+  automationId: string;
+  triggerId: string;
+  eventId: string;
+  contextKey: string | null;
+  armedStatus: string | null;
+  payloadSnapshot: Record<string, unknown>;
+  actorSnapshot: Record<string, unknown>;
+  fireAt: Date;
   createdAt: Date;
 }
+
+/** Result of an idempotent {@link DwellStore.arm}. */
+export interface ArmDwellResult {
+  id: string;
+  /**
+   * True when a NEW dwell row was inserted; false when a dwell already
+   * existed for the key and was preserved unchanged.
+   */
+  created: boolean;
+  /**
+   * The row's `fireAt` — for a continuation this is the ORIGINAL arm
+   * deadline, NOT the incoming candidate, so a `for:` window measures
+   * "continuously matched since first arm" (HA semantics).
+   */
+  fireAt: Date;
+}
+
+/**
+ * Persistence for pre-run `for:` dwell timers. The dwell row is the
+ * source of truth; the `automation-dwell` queue job is just the wake
+ * signal, and cancellation is a row delete (constraint 2).
+ */
+export interface DwellStore {
+  /**
+   * Arm a dwell idempotently on the unique
+   * `(automationId, triggerId, contextKey)` key.
+   *
+   *   - No row exists → INSERT with the supplied `fireAt` + snapshot.
+   *   - A row already exists → preserve it UNCHANGED (same `fireAt`).
+   *
+   * This is the correct `for:` semantic: a re-fire while the dwell is
+   * still armed must NOT push the deadline forward, or a continuously
+   * re-firing trigger (e.g. a level-triggered numeric_state) would never
+   * elapse. A genuine recover-then-recur deletes the row first (via
+   * inverse-cancel / re-confirm), so a fresh window starts then.
+   */
+  arm(input: UpsertDwellInput): Promise<ArmDwellResult>;
+  load(id: string): Promise<LoadedDwell | undefined>;
+  /** Look up the single dwell for a key, if armed. */
+  findByKey(
+    automationId: string,
+    triggerId: string,
+    contextKey: string | null,
+  ): Promise<LoadedDwell | undefined>;
+  /**
+   * Delete one dwell (cancellation / fire). Idempotent. Returns `true`
+   * only if THIS call deleted the row, so a caller can use the delete as
+   * an atomic claim: whoever gets `true` owns the fire, racing callers
+   * (a second pod, or the sweeper vs the queue consumer) get `false`.
+   */
+  delete(id: string): Promise<boolean>;
+  /** Delete by key (early cancel on the inverse signal). Idempotent. */
+  deleteByKey(
+    automationId: string,
+    triggerId: string,
+    contextKey: string | null,
+  ): Promise<void>;
+  /** Drop every dwell for an automation (disabled / deleted). */
+  deleteForAutomation(automationId: string): Promise<void>;
+  /** Dwells whose `fireAt` has passed — the sweeper fallback. */
+  sweepExpired(now: Date): Promise<LoadedDwell[]>;
+}
+
+// ─── Windowed-count store interface ──────────────────────────────────────
+
+/** Re-fire policy for a windowed-count gate. */
+export type WindowRefire = "every" | "once";
+
+/** Inputs to {@link WindowStore.recordAndCount}. */
+export interface RecordWindowInput {
+  automationId: string;
+  triggerId: string;
+  eventId: string;
+  contextKey: string | null;
+  /** When the qualifying occurrence happened (the recorded row's timestamp). */
+  occurredAt: Date;
+  /** Trailing sliding window, in minutes. */
+  windowMinutes: number;
+  /** Occurrences within the window that arm the trigger. */
+  threshold: number;
+  /** `every` fires at/over the threshold; `once` fires only on the crossing edge. */
+  refire: WindowRefire;
+}
+
+/**
+ * Persistence for windowed-count / rate trigger gates. The append log is the
+ * source of truth; the in-window COUNT is a pure DB read, so every pod agrees.
+ */
+export interface WindowStore {
+  /**
+   * Append one qualifying occurrence and decide whether the trigger fires:
+   *
+   *  - records the row at `occurredAt`,
+   *  - counts rows for `(automationId, triggerId, contextKey)` whose
+   *    `occurredAt >= occurredAt - windowMinutes` (the new row is included),
+   *  - returns `true` iff the re-fire policy fires for this occurrence:
+   *      `every` → `newCount >= threshold`,
+   *      `once`  → `newCount === threshold` (the crossing edge).
+   *
+   * Single INSERT per claimed emission ⇒ no double-count across pods.
+   */
+  recordAndCount(input: RecordWindowInput): Promise<boolean>;
+  /** Delete occurrences older than `cutoff` — the sweeper's TTL prune. */
+  sweepExpired(cutoff: Date): Promise<void>;
+  /** Drop every occurrence for an automation (disabled / deleted). */
+  deleteForAutomation(automationId: string): Promise<void>;
+}

package/src/dispatch/wait-timeout-queue.ts

@@ -0,0 +1,89 @@
+/**
+ * `wait_until` timeout-timer consumer (reactive automation engine §7).
+ *
+ * A reactive `wait_until` is event-driven: a relevant `ENTITY_CHANGED` wakes
+ * it via Stage 1. The ONLY queue job a suspended wait holds is a single
+ * timeout timer scheduled at `timeoutAt` (when a timeout is configured) —
+ * NOT a poll loop. When that timer fires this consumer runs `checkWaitUntil`,
+ * which re-evaluates the condition one last time and then applies the
+ * continue/fail-on-timeout policy.
+ *
+ * Idempotent: `checkWaitUntil` deletes the lock before resuming and
+ * `resumeRun` takes the per-run advisory lock, so a wake that already
+ * resumed the run makes this a no-op (`loadWaitLock` finds nothing).
+ * Mirrors the delay / dwell consumer wiring.
+ */
+import type { Logger } from "@checkstack/backend-api";
+
+import type { AutomationStore } from "../automation-store";
+import {
+  checkWaitUntil,
+  WAIT_TIMEOUT_QUEUE_NAME,
+  type WaitTimeoutJob,
+} from "./engine";
+import type { DispatchDeps } from "./types";
+
+export interface WaitTimeoutQueueConsumerArgs {
+  deps: DispatchDeps;
+  automationStore: AutomationStore;
+  logger: Logger;
+}
+
+export interface WaitTimeoutQueueConsumer {
+  stop: () => Promise<void>;
+}
+
+export async function startWaitTimeoutQueueConsumer(
+  args: WaitTimeoutQueueConsumerArgs,
+): Promise<WaitTimeoutQueueConsumer> {
+  const queue = args.deps.queueManager.getQueue<WaitTimeoutJob>(
+    WAIT_TIMEOUT_QUEUE_NAME,
+  );
+
+  await queue.consume(
+    async (job) => {
+      const { runId, waitLockId } = job.data;
+      const lock = await args.deps.runStore.loadWaitLock(waitLockId);
+      if (!lock || lock.kind !== "until") {
+        args.logger.debug(
+          `wait-timeout: lock ${waitLockId} gone (woken / cancelled)`,
+        );
+        return;
+      }
+      const run = await args.deps.runStore.loadRun(runId);
+      if (!run) {
+        await args.deps.runStore.deleteWaitLock(waitLockId);
+        return;
+      }
+      const automation = await args.automationStore.getById(run.automationId);
+      if (!automation) {
+        await args.deps.runStore.deleteWaitLock(waitLockId);
+        await args.deps.runStore.updateRunStatus(
+          runId,
+          "failed",
+          "automation deleted while run was suspended on wait_until",
+        );
+        await args.deps.runStateStore.clear(runId);
+        return;
+      }
+
+      await checkWaitUntil(args.deps, {
+        runId,
+        waitLockId,
+        automation: {
+          id: automation.id,
+          name: automation.name,
+          status: automation.status,
+          definition: automation.definition,
+        },
+      });
+    },
+    { consumerGroup: "automation-wait-timeout", maxRetries: 3 },
+  );
+
+  return {
+    stop: async () => {
+      await queue.stop();
+    },
+  };
+}