automategraph 1.5.1 → 1.5.3

package/tests/conformance.test.ts

@@ -0,0 +1,442 @@
+/* ============================================================
+ * STORAGE ADAPTER CONFORMANCE PACK
+ * ------------------------------------------------------------
+ * The adapter contract (src/storage-adapter.ts) is only worth
+ * something if it is enforced. These tests do three things:
+ *
+ *   1. prove the reference adapter (InMemoryStorage) passes
+ *   2. prove the checker itself has teeth — a deliberately
+ *      broken adapter must produce specific violations
+ *   3. cover the wire format helpers (serialize/deserialize,
+ *      stable output, corruption detection, comparison)
+ * ============================================================ */
+
+import { InMemoryStorage, ExecutionSnapshot } from "../src/automategraph";
+
+import {
+  StorageAdapter,
+  compareSnapshots,
+  defineStorageAdapter,
+  deserializeSnapshot,
+  serializeSnapshot,
+  snapshotShapeViolations,
+  storageAdapterViolations
+} from "../src/storage-adapter";
+
+import {
+  ConformanceReport,
+  StorageConformanceFixture,
+  conformanceSnapshot,
+  runStorageConformance
+} from "../src/storage-conformance";
+
+import { suite } from "./harness";
+
+const conf = suite("conformance", "Storage adapter contract");
+const wire = suite("conformance", "Snapshot wire format");
+
+const BIG = { timeoutMs: 20_000 };
+
+/* ------------------------- fixtures ------------------------- */
+
+/** Reference adapter: the in-memory implementation shipped with the engine. */
+const memoryFixture: StorageConformanceFixture = {
+  name: "InMemoryStorage",
+  create: () => new InMemoryStorage()
+};
+
+/** Same adapter, but able to inject a failing save (atomicity checks). */
+class FaultInjectingAdapter implements StorageAdapter {
+  readonly id = "test.fault-injecting";
+  readonly version = "1.0.0";
+  readonly capabilities = {
+    atomic: true,
+    durable: false,
+    list: true,
+    delete: true,
+    concurrentExecutions: true
+  };
+
+  private readonly inner = new InMemoryStorage();
+  private failNext = false;
+
+  failNextSave(): void {
+    this.failNext = true;
+  }
+
+  async saveAtomic(snapshot: ExecutionSnapshot): Promise<void> {
+    if (this.failNext) {
+      this.failNext = false;
+      // Correct behaviour: reject WITHOUT touching the stored snapshot.
+      throw new Error("injected write failure");
+    }
+    return this.inner.saveAtomic(snapshot);
+  }
+
+  load(executionId: string) {
+    return this.inner.load(executionId);
+  }
+
+  listExecutionIds(): Promise<string[]> {
+    return this.inner.listExecutionIds();
+  }
+
+  delete(executionId: string): Promise<boolean> {
+    return this.inner.delete(executionId);
+  }
+}
+
+const faultFixture: StorageConformanceFixture = {
+  name: "FaultInjectingAdapter",
+  create: () => new FaultInjectingAdapter(),
+  failNextSave: adapter => (adapter as FaultInjectingAdapter).failNextSave()
+};
+
+/**
+ * Deliberately broken adapter: aliases snapshots on save AND load,
+ * ignores deletes and returns an empty list. The checker must catch all
+ * of it — otherwise "conformance" means nothing.
+ */
+class BrokenAdapter implements StorageAdapter {
+  readonly id = "test.broken";
+  readonly version = "0.0.1";
+  readonly capabilities = { list: true, delete: true };
+
+  private readonly store = new Map<string, ExecutionSnapshot>();
+
+  async saveAtomic(snapshot: ExecutionSnapshot): Promise<void> {
+    this.store.set(snapshot.executionId, snapshot); // no copy: aliasing bug
+  }
+
+  async load(executionId: string): Promise<ExecutionSnapshot | null> {
+    return this.store.get(executionId) ?? null; // no copy: aliasing bug
+  }
+
+  async listExecutionIds(): Promise<string[]> {
+    return []; // broken
+  }
+
+  async delete(): Promise<boolean> {
+    return false; // broken
+  }
+}
+
+/** Commits first, then fails — a torn write. */
+class TornSaveAdapter implements StorageAdapter {
+  readonly id = "test.torn-save";
+  readonly version = "0.0.1";
+  readonly capabilities = { atomic: false, list: true, delete: true };
+
+  private readonly inner = new InMemoryStorage();
+  private failNext = false;
+
+  failNextSave(): void {
+    this.failNext = true;
+  }
+
+  async saveAtomic(snapshot: ExecutionSnapshot): Promise<void> {
+    await this.inner.saveAtomic(snapshot); // committed too early
+    if (this.failNext) {
+      this.failNext = false;
+      throw new Error("failure after commit");
+    }
+  }
+
+  load(executionId: string) {
+    return this.inner.load(executionId);
+  }
+
+  listExecutionIds() {
+    return this.inner.listExecutionIds();
+  }
+
+  delete(executionId: string) {
+    return this.inner.delete(executionId);
+  }
+}
+
+const tornFixture: StorageConformanceFixture = {
+  name: "TornSaveAdapter",
+  create: () => new TornSaveAdapter(),
+  failNextSave: adapter => (adapter as TornSaveAdapter).failNextSave()
+};
+
+const checks = (report: ConformanceReport, code: string): boolean =>
+  report.violations.some(violation => violation.code === code);
+
+/* ------------------------- tests ------------------------- */
+
+conf.t(
+  "InMemoryStorage passes the full conformance suite with zero violations",
+  async t => {
+    const report = await runStorageConformance(memoryFixture);
+
+    if (!report.passed) {
+      throw new Error(
+        `violations: ${report.violations.map(v => `${v.code}: ${v.message}`).join(" | ")}`
+      );
+    }
+    if (report.adapterId !== "automategraph.memory-storage") throw new Error(report.adapterId);
+    if (report.adapterVersion !== "1.1.0") throw new Error(report.adapterVersion);
+    if (report.checks < 10) throw new Error(`checks=${report.checks}`);
+    if (report.violations.length !== 0) throw new Error("expected zero violations");
+
+    // the memory adapter cannot inject faults, so atomicity is reported as
+    // skipped rather than silently assumed to hold
+    if (!report.skipped.some(skip => skip.startsWith("FAILED_SAVE_PRESERVES_PREVIOUS"))) {
+      throw new Error("atomicity should be skipped when no fault injection exists");
+    }
+
+    t.log("checks executed", report.checks);
+    t.log("skipped", report.skipped.length, report.skipped);
+  },
+  BIG
+);
+
+conf.t(
+  "an adapter with fault injection proves atomicity instead of skipping it",
+  async t => {
+    const report = await runStorageConformance(faultFixture);
+
+    if (!report.passed) {
+      throw new Error(report.violations.map(v => v.message).join(" | "));
+    }
+    if (report.skipped.some(skip => skip.startsWith("FAILED_SAVE_PRESERVES_PREVIOUS"))) {
+      throw new Error("atomicity check was skipped despite failNextSave()");
+    }
+
+    t.log("checks executed", report.checks);
+    t.log("skipped", report.skipped);
+  },
+  BIG
+);
+
+conf.t(
+  "the checker catches aliasing, broken listing and broken deletion",
+  async t => {
+    const report = await runStorageConformance({
+      name: "BrokenAdapter",
+      create: () => new BrokenAdapter()
+    });
+
+    if (report.passed) throw new Error("the broken adapter was wrongly accepted");
+
+    for (const code of [
+      "NO_ALIASING_ON_SAVE",
+      "NO_ALIASING_ON_LOAD",
+      "LIST",
+      "DELETE_IDEMPOTENT"
+    ]) {
+      if (!checks(report, code)) {
+        throw new Error(
+          `missing ${code}; got ${report.violations.map(v => v.code).join(",")}`
+        );
+      }
+    }
+
+    t.log("violations detected", report.violations.map(v => v.code).join(", "));
+  },
+  BIG
+);
+
+conf.t(
+  "the checker catches a torn write that commits before failing",
+  async t => {
+    const report = await runStorageConformance(tornFixture);
+
+    if (report.passed) throw new Error("torn writes were wrongly accepted");
+    if (!checks(report, "FAILED_SAVE_PRESERVES_PREVIOUS")) {
+      throw new Error(`violations=${report.violations.map(v => v.code).join(",")}`);
+    }
+
+    t.log("finding: the conformance suite distinguishes 'rejects' from 'rejects without " +
+      "corrupting state' — adapters must checkpoint before commit, not after");
+  },
+  BIG
+);
+
+conf.t(
+  "missing required members are reported as SHAPE violations",
+  async t => {
+    const incomplete = { id: "x", version: "1" } as unknown as StorageAdapter;
+    const violations = storageAdapterViolations(incomplete);
+
+    if (!violations.some(violation => violation.includes("saveAtomic"))) throw new Error("saveAtomic");
+    if (!violations.some(violation => violation.includes("load"))) throw new Error("load");
+
+    const report = await runStorageConformance({
+      name: "Incomplete",
+      create: () => incomplete
+    });
+
+    if (report.passed) throw new Error("incomplete adapter accepted");
+    if (!checks(report, "SHAPE")) throw new Error("SHAPE not reported");
+  },
+  BIG
+);
+
+conf.t(
+  "an adapter that claims a capability it does not implement is rejected (defineStorageAdapter)",
+  () => {
+    const liar = {
+      id: "test.liar",
+      version: "1.0.0",
+      capabilities: { list: true },
+      saveAtomic: async () => undefined,
+      load: async () => null
+    };
+
+    try {
+      defineStorageAdapter(liar as unknown as StorageAdapter);
+      throw new Error("expected throw");
+    } catch (error) {
+      if (!(error as { code?: string }).code) throw error;
+      if ((error as { code: string }).code !== "STORAGE_ADAPTER_INVALID") {
+        throw new Error((error as { code: string }).code);
+      }
+    }
+
+    // and a consistent adapter is accepted, identity-preserved
+    const storage = new InMemoryStorage();
+    if (defineStorageAdapter(storage) !== storage) throw new Error("identity broken");
+  }
+);
+
+conf.t(
+  "the engine accepts any conformant adapter as its storage plugin",
+  async t => {
+    const { GraphExecutionEngine } = await import("../src/automategraph");
+    const { g, node: build } = await import("./helpers");
+
+    const adapter = new FaultInjectingAdapter();
+    const definition = g({
+      nodes: [build("a", () => "A", { next: ["b"] }), build("b", () => "B")],
+      entryNodes: ["a"],
+      terminalNodes: ["b"]
+    });
+
+    const report = await new GraphExecutionEngine(definition, { storage: adapter }).run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+
+    const stored = await adapter.load(report.executionId);
+    if (!stored) throw new Error("execution was not persisted");
+    if (stored.graphId !== definition.id) throw new Error("stored snapshot mismatch");
+    if (stored.pluginVersions && Object.keys(stored.pluginVersions).length !== 0) {
+      throw new Error("unexpected plugin versions");
+    }
+
+    t.log("snapshots stored", (await adapter.listExecutionIds()).length);
+  },
+  BIG
+);
+
+/* ------------------------- wire format ------------------------- */
+
+wire.t(
+  "serializeSnapshot is deterministic and round-trips through JSON",
+  () => {
+    const snapshot = conformanceSnapshot("wire-1");
+    const first = serializeSnapshot(snapshot);
+    const second = serializeSnapshot(conformanceSnapshot("wire-1"));
+
+    if (first !== second) throw new Error("serialization is not deterministic");
+
+    const revived = deserializeSnapshot(first);
+    const differences = compareSnapshots(snapshot, revived);
+    if (differences.length > 0) throw new Error(differences.join(","));
+
+    // key order of the source object must not matter
+    const shuffled: ExecutionSnapshot = {
+      timestamp: snapshot.timestamp,
+      pluginVersions: snapshot.pluginVersions,
+      errors: snapshot.errors,
+      results: snapshot.results,
+      outputs: snapshot.outputs,
+      status: snapshot.status,
+      graphVersion: snapshot.graphVersion,
+      graphId: snapshot.graphId,
+      executionId: snapshot.executionId
+    };
+    if (serializeSnapshot(shuffled) !== first) throw new Error("key order changed the bytes");
+  }
+);
+
+wire.t(
+  "corrupt or unsupported snapshots are rejected with typed errors",
+  () => {
+    for (const [payload, expectedCode] of [
+      ["{not json", "STORAGE_CORRUPT_SNAPSHOT"],
+      [JSON.stringify({ hello: "world" }), "STORAGE_CORRUPT_SNAPSHOT"],
+      [
+        JSON.stringify({ format: "automategraph.snapshot", formatVersion: 99, snapshot: {} }),
+        "STORAGE_FORMAT_UNSUPPORTED"
+      ],
+      [
+        JSON.stringify({
+          format: "automategraph.snapshot",
+          formatVersion: 1,
+          snapshot: { executionId: "e" }
+        }),
+        "STORAGE_CORRUPT_SNAPSHOT"
+      ]
+    ] as Array<[string, string]>) {
+      try {
+        deserializeSnapshot(payload);
+        throw new Error(`expected throw for ${payload.slice(0, 40)}`);
+      } catch (error) {
+        const code = (error as { code?: string }).code;
+        if (code !== expectedCode) throw new Error(`got ${String(code)}, expected ${expectedCode}`);
+      }
+    }
+  }
+);
+
+wire.t(
+  "snapshotShapeViolations pinpoints every malformed field",
+  () => {
+    const violations = snapshotShapeViolations({
+      executionId: "e",
+      graphId: "",
+      graphVersion: "1",
+      status: "NOT_A_STATUS",
+      outputs: null,
+      results: [{ nodeId: "a" }],
+      errors: [{}],
+      pluginVersions: {},
+      timestamp: "2026-01-01T00:00:00.000Z"
+    });
+
+    const joined = violations.join(" | ");
+    for (const expected of [
+      "graphId",
+      "status must be a GraphStatus",
+      "outputs must be an object",
+      "results[0].activationId",
+      "results[0].attempt",
+      "errors[0]"
+    ]) {
+      if (!joined.includes(expected)) throw new Error(`missing "${expected}" in: ${joined}`);
+    }
+  }
+);
+
+wire.t(
+  "compareSnapshots reports structural differences precisely",
+  () => {
+    const base = conformanceSnapshot("wire-compare");
+    if (compareSnapshots(base, conformanceSnapshot("wire-compare")).length !== 0) {
+      throw new Error("identical snapshots reported as different");
+    }
+
+    const changed: ExecutionSnapshot = {
+      ...base,
+      status: "COMPLETED",
+      outputs: { a: { n: 2 } }
+    };
+    const differences = compareSnapshots(base, changed);
+
+    if (!differences.includes("status differs")) throw new Error(differences.join(","));
+    if (!differences.includes("outputs differs")) throw new Error(differences.join(","));
+  }
+);