@checkstack/automation-backend 0.2.0

package/src/validate-definition.test.ts

@@ -0,0 +1,306 @@
+/**
+ * Tests for the deep automation-definition validator.
+ *
+ * Builds a tiny trigger + action registry, then asserts that
+ * `collectDefinitionIssues` surfaces structural errors, unknown
+ * trigger/action ids, and — the gap this module closes — invalid
+ * provider-action config values + unknown config keys.
+ */
+import { describe, expect, it } from "bun:test";
+import { z } from "zod";
+import { Versioned } from "@checkstack/backend-api";
+import { definePluginMetadata } from "@checkstack/common";
+import type { AutomationDefinition } from "@checkstack/automation-common";
+import { createActionRegistry } from "./action-registry";
+import { createTriggerRegistry } from "./trigger-registry";
+import { collectDefinitionIssues } from "./validate-definition";
+
+const meta = definePluginMetadata({ pluginId: "test" });
+
+function makeDeps() {
+  const triggerRegistry = createTriggerRegistry();
+  const actionRegistry = createActionRegistry();
+
+  triggerRegistry.register(
+    {
+      id: "fired",
+      displayName: "Fired",
+      payloadSchema: z.object({ id: z.string() }),
+      configSchema: z.object({ intervalSeconds: z.number().int().min(1) }),
+      // A trigger must be reachable via a hook or setup; this validator
+      // only cares about the config schema, so a no-op setup suffices.
+      setup: async () => async () => {},
+    },
+    meta,
+  );
+
+  actionRegistry.register(
+    {
+      id: "log",
+      displayName: "Log",
+      config: new Versioned({
+        version: 1,
+        schema: z.object({
+          message: z.string().min(1),
+          level: z.enum(["debug", "info", "warn", "error"]).default("info"),
+        }),
+      }),
+      execute: async () => ({ success: true }),
+    },
+    meta,
+  );
+
+  return { triggerRegistry, actionRegistry };
+}
+
+/**
+ * Deps whose action registry includes a producing action (`test.create`,
+ * `produces: "thing"`) so the artifact-id invariants can be exercised.
+ */
+function makeProducerDeps() {
+  const { triggerRegistry, actionRegistry } = makeDeps();
+  actionRegistry.register(
+    {
+      id: "create",
+      displayName: "Create Thing",
+      config: new Versioned({ version: 1, schema: z.object({}) }),
+      produces: "thing",
+      execute: async () => ({ success: true, artifact: { ok: true } }),
+    },
+    meta,
+  );
+  return { triggerRegistry, actionRegistry };
+}
+
+function baseDefinition(
+  overrides: Partial<AutomationDefinition> = {},
+): AutomationDefinition {
+  return {
+    name: "A",
+    triggers: [{ event: "test.fired", config: { intervalSeconds: 5 } }],
+    conditions: [],
+    actions: [
+      {
+        action: "test.log",
+        config: { message: "hi", level: "info" },
+        enabled: true,
+        continue_on_error: false,
+      },
+    ],
+    mode: "single",
+    max_runs: 1,
+    ...overrides,
+  };
+}
+
+describe("collectDefinitionIssues", () => {
+  it("returns no issues for a fully valid definition", () => {
+    const issues = collectDefinitionIssues(baseDefinition(), makeDeps());
+    expect(issues).toEqual([]);
+  });
+
+  it("flags an invalid enum value in a provider action config", () => {
+    const def = baseDefinition({
+      actions: [
+        {
+          action: "test.log",
+          config: { message: "hi", level: "debugthisiswrong" },
+          enabled: true,
+          continue_on_error: false,
+        },
+      ],
+    });
+    const issues = collectDefinitionIssues(def, makeDeps());
+    expect(issues.length).toBeGreaterThan(0);
+    const levelIssue = issues.find((i) => i.path.join(".") === "actions.0.config.level");
+    expect(levelIssue).toBeDefined();
+  });
+
+  it("flags a missing required config field", () => {
+    const def = baseDefinition({
+      actions: [
+        {
+          action: "test.log",
+          config: { level: "info" },
+          enabled: true,
+          continue_on_error: false,
+        },
+      ],
+    });
+    const issues = collectDefinitionIssues(def, makeDeps());
+    expect(
+      issues.some((i) => i.path.join(".") === "actions.0.config.message"),
+    ).toBe(true);
+  });
+
+  it("flags an unknown config key (strict)", () => {
+    const def = baseDefinition({
+      actions: [
+        {
+          action: "test.log",
+          config: { message: "hi", level: "info", levle: "typo" },
+          enabled: true,
+          continue_on_error: false,
+        },
+      ],
+    });
+    const issues = collectDefinitionIssues(def, makeDeps());
+    expect(issues.length).toBeGreaterThan(0);
+    // The unknown key is reported under the action's config path.
+    expect(issues.some((i) => i.path[0] === "actions" && i.path.includes("config"))).toBe(true);
+  });
+
+  it("flags an unknown action id", () => {
+    const def = baseDefinition({
+      actions: [
+        {
+          action: "test.does_not_exist",
+          config: {},
+          enabled: true,
+          continue_on_error: false,
+        },
+      ],
+    });
+    const issues = collectDefinitionIssues(def, makeDeps());
+    const issue = issues.find((i) => i.path.join(".") === "actions.0.action");
+    expect(issue?.message).toMatch(/Unknown action/);
+  });
+
+  it("flags an unknown trigger event", () => {
+    const def = baseDefinition({
+      triggers: [{ event: "nope.gone" }],
+    });
+    const issues = collectDefinitionIssues(def, makeDeps());
+    const issue = issues.find((i) => i.path.join(".") === "triggers.0.event");
+    expect(issue?.message).toMatch(/Unknown trigger/);
+  });
+
+  it("flags an invalid trigger config value", () => {
+    const def = baseDefinition({
+      triggers: [{ event: "test.fired", config: { intervalSeconds: 0 } }],
+    });
+    const issues = collectDefinitionIssues(def, makeDeps());
+    expect(
+      issues.some(
+        (i) => i.path.join(".") === "triggers.0.config.intervalSeconds",
+      ),
+    ).toBe(true);
+  });
+
+  it("validates configs nested inside a choose branch", () => {
+    const def = baseDefinition({
+      actions: [
+        {
+          choose: [
+            {
+              when: "true",
+              sequence: [
+                {
+                  action: "test.log",
+                  config: { message: "hi", level: "bogus" },
+                  enabled: true,
+                  continue_on_error: false,
+                },
+              ],
+            },
+          ],
+          enabled: true,
+          continue_on_error: false,
+        },
+      ],
+    });
+    const issues = collectDefinitionIssues(def, makeDeps());
+    expect(
+      issues.some(
+        (i) =>
+          i.path.join(".") === "actions.0.choose.0.sequence.0.config.level",
+      ),
+    ).toBe(true);
+  });
+
+  it("returns structural issues for a malformed top-level shape", () => {
+    const issues = collectDefinitionIssues(
+      { name: "", triggers: [] },
+      makeDeps(),
+    );
+    expect(issues.length).toBeGreaterThan(0);
+  });
+
+  it("rejects a duplicate action id", () => {
+    const def = baseDefinition({
+      actions: [
+        {
+          id: "dup",
+          action: "test.log",
+          config: { message: "a", level: "info" },
+          enabled: true,
+          continue_on_error: false,
+        },
+        {
+          id: "dup",
+          action: "test.log",
+          config: { message: "b", level: "info" },
+          enabled: true,
+          continue_on_error: false,
+        },
+      ],
+    });
+    const issues = collectDefinitionIssues(def, makeDeps());
+    const dupIssue = issues.find(
+      (i) => i.path.join(".") === "actions.1.id",
+    );
+    expect(dupIssue?.message).toMatch(/must be unique/);
+  });
+
+  it("rejects a producing action that has no id", () => {
+    const deps = makeProducerDeps();
+    const def = baseDefinition({
+      actions: [
+        {
+          action: "test.create",
+          config: {},
+          enabled: true,
+          continue_on_error: false,
+        },
+      ],
+    });
+    const issues = collectDefinitionIssues(def, deps);
+    const idIssue = issues.find((i) => i.path.join(".") === "actions.0.id");
+    expect(idIssue?.message).toMatch(/must have an id/);
+  });
+
+  it("accepts a producing action that has an id", () => {
+    const deps = makeProducerDeps();
+    const def = baseDefinition({
+      actions: [
+        {
+          id: "make_thing",
+          action: "test.create",
+          config: {},
+          enabled: true,
+          continue_on_error: false,
+        },
+      ],
+    });
+    const issues = collectDefinitionIssues(def, deps);
+    expect(issues).toEqual([]);
+  });
+
+  it("rejects a hyphenated action id via the structural pass", () => {
+    const def = baseDefinition({
+      actions: [
+        {
+          id: "bad-id",
+          action: "test.log",
+          config: { message: "hi", level: "info" },
+          enabled: true,
+          continue_on_error: false,
+        },
+      ],
+    });
+    const issues = collectDefinitionIssues(def, makeDeps());
+    expect(
+      issues.some((i) => i.path.includes("id")),
+    ).toBe(true);
+  });
+});

package/src/validate-definition.ts

@@ -0,0 +1,304 @@
+/**
+ * Deep validation of an automation definition.
+ *
+ * `AutomationDefinitionSchema` only validates the structural shape —
+ * action `config` is typed as `z.record(z.unknown())`, so it never
+ * checks a provider action's config against that action's own schema.
+ * This walker fills the gap so the editor can surface *any* wrong
+ * content, not just structural errors:
+ *
+ *   - unknown trigger `event` / action `action` ids,
+ *   - per-trigger `config` that violates the trigger's `configSchema`,
+ *   - per-action `config` that violates the action's config schema
+ *     (wrong enum value, missing required field, wrong type, AND —
+ *     because we validate in strict mode — unknown/typo'd keys).
+ *
+ * Returned issue `path`s are dot-joinable for display, e.g.
+ * `actions.0.config.level` or `triggers.1.event`.
+ */
+import { z } from "zod";
+import {
+  AutomationDefinitionSchema,
+  type ActionInput,
+  type AutomationDefinition,
+} from "@checkstack/automation-common";
+import type { ActionRegistry } from "./action-registry";
+import type { TriggerRegistry } from "./trigger-registry";
+
+export interface DefinitionIssue {
+  path: Array<string | number>;
+  message: string;
+}
+
+export interface ValidateDefinitionDeps {
+  triggerRegistry: TriggerRegistry;
+  actionRegistry: ActionRegistry;
+}
+
+/**
+ * Validate a definition both structurally and semantically. Returns an
+ * empty array when the definition is fully valid.
+ *
+ * Structural errors short-circuit the semantic pass: if the top-level
+ * shape is wrong we can't reliably walk the action tree, so we return
+ * the structural issues alone and let the operator fix those first.
+ */
+export function collectDefinitionIssues(
+  definition: unknown,
+  deps: ValidateDefinitionDeps,
+): DefinitionIssue[] {
+  const parsed = AutomationDefinitionSchema.safeParse(definition);
+  if (!parsed.success) {
+    return parsed.error.issues.map((issue) => ({
+      path: issue.path.map((segment) => toPathSegment(segment)),
+      message: issue.message,
+    }));
+  }
+
+  const issues: DefinitionIssue[] = [];
+  validateTriggers(parsed.data, deps, issues);
+  validateActionList(parsed.data.actions, ["actions"], deps, issues);
+  validateActionIds(parsed.data.actions, ["actions"], deps, issues);
+  return issues;
+}
+
+// ─── Semantic action-id validation ──────────────────────────────────────
+
+/**
+ * Walk the entire action tree and enforce the two artifact-reference
+ * invariants the structural zod pass can't:
+ *
+ *   1. Every action `id` is unique within the automation (so
+ *      `artifacts.<id>.<name>` is unambiguous).
+ *   2. Any provider action whose registered action declares a truthy
+ *      `produces` MUST carry an `id` (so the produced artifact is
+ *      referenceable).
+ *
+ * Identifier-format is already enforced by the zod schema in the
+ * structural pass, so we don't re-check it here.
+ */
+function validateActionIds(
+  actions: ActionInput[],
+  basePath: Array<string | number>,
+  deps: ValidateDefinitionDeps,
+  issues: DefinitionIssue[],
+): void {
+  const seen = new Set<string>();
+  walkActionIds(actions, basePath, deps, seen, issues);
+}
+
+function walkActionIds(
+  actions: ActionInput[],
+  basePath: Array<string | number>,
+  deps: ValidateDefinitionDeps,
+  seen: Set<string>,
+  issues: DefinitionIssue[],
+): void {
+  for (const [index, action] of actions.entries()) {
+    walkActionId(action, [...basePath, index], deps, seen, issues);
+  }
+}
+
+function walkActionId(
+  action: ActionInput,
+  path: Array<string | number>,
+  deps: ValidateDefinitionDeps,
+  seen: Set<string>,
+  issues: DefinitionIssue[],
+): void {
+  if (typeof action.id === "string") {
+    if (seen.has(action.id)) {
+      issues.push({
+        path: [...path, "id"],
+        message: `Action id "${action.id}" must be unique within the automation`,
+      });
+    } else {
+      seen.add(action.id);
+    }
+  }
+
+  if ("action" in action) {
+    const registered = deps.actionRegistry.getAction(action.action);
+    if (registered?.produces && !action.id) {
+      issues.push({
+        path: [...path, "id"],
+        message:
+          "Actions that produce an artifact must have an id so the artifact can be referenced as artifacts.<id>.<name>",
+      });
+    }
+    return;
+  }
+
+  if ("choose" in action) {
+    for (const [branchIndex, branch] of action.choose.entries()) {
+      walkActionIds(
+        branch.sequence,
+        [...path, "choose", branchIndex, "sequence"],
+        deps,
+        seen,
+        issues,
+      );
+    }
+    if (action.else) {
+      walkActionIds(action.else, [...path, "else"], deps, seen, issues);
+    }
+    return;
+  }
+
+  if ("parallel" in action) {
+    walkActionIds(action.parallel, [...path, "parallel"], deps, seen, issues);
+    return;
+  }
+
+  if ("repeat" in action) {
+    walkActionIds(
+      action.repeat.sequence,
+      [...path, "repeat", "sequence"],
+      deps,
+      seen,
+      issues,
+    );
+    return;
+  }
+
+  if ("sequence" in action) {
+    walkActionIds(action.sequence, [...path, "sequence"], deps, seen, issues);
+    return;
+  }
+
+  // delay / variables / condition / stop / wait_for_trigger have no child
+  // action lists and don't produce artifacts — nothing more to walk.
+}
+
+function validateTriggers(
+  definition: AutomationDefinition,
+  deps: ValidateDefinitionDeps,
+  issues: DefinitionIssue[],
+): void {
+  for (const [index, trigger] of definition.triggers.entries()) {
+    const registered = deps.triggerRegistry.getTrigger(trigger.event);
+    if (!registered) {
+      issues.push({
+        path: ["triggers", index, "event"],
+        message: `Unknown trigger event "${trigger.event}"`,
+      });
+      continue;
+    }
+    if (registered.configSchema) {
+      const result = strictParse(registered.configSchema, trigger.config ?? {});
+      if (!result.success) {
+        pushZodIssues(result.error, ["triggers", index, "config"], issues);
+      }
+    }
+  }
+}
+
+function validateActionList(
+  actions: ActionInput[],
+  basePath: Array<string | number>,
+  deps: ValidateDefinitionDeps,
+  issues: DefinitionIssue[],
+): void {
+  for (const [index, action] of actions.entries()) {
+    validateAction(action, [...basePath, index], deps, issues);
+  }
+}
+
+function validateAction(
+  action: ActionInput,
+  path: Array<string | number>,
+  deps: ValidateDefinitionDeps,
+  issues: DefinitionIssue[],
+): void {
+  if ("action" in action) {
+    const registered = deps.actionRegistry.getAction(action.action);
+    if (!registered) {
+      issues.push({
+        path: [...path, "action"],
+        message: `Unknown action "${action.action}"`,
+      });
+      return;
+    }
+    const result = strictParse(registered.config.schema, action.config);
+    if (!result.success) {
+      pushZodIssues(result.error, [...path, "config"], issues);
+    }
+    return;
+  }
+
+  if ("choose" in action) {
+    for (const [branchIndex, branch] of action.choose.entries()) {
+      validateActionList(
+        branch.sequence,
+        [...path, "choose", branchIndex, "sequence"],
+        deps,
+        issues,
+      );
+    }
+    if (action.else) {
+      validateActionList(action.else, [...path, "else"], deps, issues);
+    }
+    return;
+  }
+
+  if ("parallel" in action) {
+    validateActionList(action.parallel, [...path, "parallel"], deps, issues);
+    return;
+  }
+
+  if ("repeat" in action) {
+    validateActionList(
+      action.repeat.sequence,
+      [...path, "repeat", "sequence"],
+      deps,
+      issues,
+    );
+    return;
+  }
+
+  if ("sequence" in action) {
+    validateActionList(action.sequence, [...path, "sequence"], deps, issues);
+    return;
+  }
+
+  // delay / variables / condition / stop / wait_for_trigger carry no
+  // provider config to deep-validate — their structure is already fully
+  // covered by AutomationDefinitionSchema.
+}
+
+/**
+ * Parse against a schema in strict mode when it's a plain object schema,
+ * so unknown / typo'd config keys are reported rather than silently
+ * stripped. Non-object schemas (unions, records, primitives) fall back
+ * to a normal parse.
+ */
+function strictParse(schema: z.ZodType<unknown>, value: unknown) {
+  if (schema instanceof z.ZodObject) {
+    return schema.strict().safeParse(value);
+  }
+  return schema.safeParse(value);
+}
+
+function pushZodIssues(
+  error: z.ZodError,
+  basePath: Array<string | number>,
+  issues: DefinitionIssue[],
+): void {
+  for (const issue of error.issues) {
+    issues.push({
+      path: [...basePath, ...issue.path.map((segment) => toPathSegment(segment))],
+      message: issue.message,
+    });
+  }
+}
+
+/**
+ * Zod issue paths are `PropertyKey[]` (string | number | symbol). The
+ * automation contract's issue path is `(string | number)[]`, so coerce
+ * the rare symbol segment to its string form.
+ */
+function toPathSegment(segment: PropertyKey): string | number {
+  return typeof segment === "number" || typeof segment === "string"
+    ? segment
+    : String(segment);
+}

package/tsconfig.json

@@ -0,0 +1,41 @@
+{
+  "extends": "@checkstack/tsconfig/backend.json",
+  "include": [
+    "src"
+  ],
+  "references": [
+    {
+      "path": "../automation-common"
+    },
+    {
+      "path": "../backend-api"
+    },
+    {
+      "path": "../command-backend"
+    },
+    {
+      "path": "../common"
+    },
+    {
+      "path": "../drizzle-helper"
+    },
+    {
+      "path": "../integration-common"
+    },
+    {
+      "path": "../notification-common"
+    },
+    {
+      "path": "../queue-api"
+    },
+    {
+      "path": "../signal-common"
+    },
+    {
+      "path": "../template-engine"
+    },
+    {
+      "path": "../test-utils-backend"
+    }
+  ]
+}