@aigne/ash 0.0.1

package/src/ai-dev-loop/e2e-multi-fix.test.ts

@@ -0,0 +1,473 @@
+/**
+ * Complex E2E: Multi-error correction chain.
+ *
+ * Simulates a realistic AI dev loop where the "LLM" makes multiple
+ * different mistakes, each requiring a different fix strategy.
+ * All ASH compilation, validation, and execution are REAL.
+ * Only think() is scripted.
+ *
+ * Scenario: AI tries to build an ETL pipeline that reads users,
+ * filters active ones, maps to summary, and saves to output.
+ * It makes 4 different errors before getting it right.
+ */
+
+import { describe, it, expect, vi } from "vitest";
+import { runDevLoop } from "./dev-loop.js";
+import { runStructured } from "./structured-runner.js";
+import { ashValidate } from "./ash-validate.js";
+import type { WorldInterface, JobLogger, JobContext } from "../compiler.js";
+
+// ── Real World ───────────────────────────────────────────
+
+function makeWorld(data: Record<string, unknown[]> = {}): WorldInterface & {
+  written: Record<string, unknown[]>;
+  published: Record<string, unknown[]>;
+} {
+  const written: Record<string, unknown[]> = {};
+  const published: Record<string, unknown[]> = {};
+  return {
+    read(path: string) { return data[path] ?? []; },
+    write(path: string, records: unknown[]) { written[path] = records; },
+    publish(topic: string, records: unknown[]) { published[topic] = records; },
+    written,
+    published,
+  };
+}
+
+function makeCtx(data: Record<string, unknown[]> = {}) {
+  const world = makeWorld(data);
+  return {
+    ctx: {
+      world,
+      caps: new Set(["*"]),
+      logger: { log() {} } as JobLogger,
+    } as JobContext,
+    world,
+  };
+}
+
+// ── Scripted Think ───────────────────────────────────────
+
+/**
+ * Creates a think function that returns scripts in sequence,
+ * and records what correction prompts it received.
+ */
+function scriptedThink(scripts: string[]) {
+  let i = 0;
+  const corrections: string[] = [];
+  const think = vi.fn(async (req: any) => {
+    // Record correction prompt (not the first call which is the intent)
+    if (i > 0 && req.messages?.[0]?.content) {
+      corrections.push(req.messages[0].content);
+    }
+    const script = scripts[i++] ?? "";
+    return {
+      kind: "completed" as const,
+      response: {
+        content: "",
+        tool_calls: [{
+          id: `tc-${i}`,
+          name: "ash_run",
+          arguments: JSON.stringify({ script }),
+        }],
+      },
+      toolResults: [],
+    };
+  });
+  return { think, corrections };
+}
+
+function realRunner(ctx: JobContext, mode: "dry-run" | "live" = "dry-run") {
+  return async (source: string) => runStructured(source, ctx, { mode });
+}
+
+function realValidate() {
+  return async (source: string) => ashValidate(source);
+}
+
+// ── Tests ────────────────────────────────────────────────
+
+describe("E2E: Complex multi-error correction chain", () => {
+
+  it("4 different errors → 4 corrections → 5th attempt succeeds", async () => {
+    const { ctx } = makeCtx({
+      "/world/users": [
+        { id: 1, name: "Alice", active: true, score: 90 },
+        { id: 2, name: "Bob", active: false, score: 60 },
+        { id: 3, name: "Carol", active: true, score: 75 },
+      ],
+    });
+
+    const { think, corrections } = scriptedThink([
+      // Attempt 1: unknown command "fetch" (should be "find")
+      'job etl { fetch /world/users }',
+
+      // Attempt 2: unknown command "filter" (should be "where")
+      'job etl { find /world/users | filter active == true }',
+
+      // Attempt 3: unterminated string
+      'job etl { find /world/users | where active == true | save /world/output',
+
+      // Attempt 4: type mismatch — save then where (save outputs none)
+      `job etl {
+        find /world/users
+        | save /world/output
+        | where active == true
+      }`,
+
+      // Attempt 5: correct!
+      `job etl {
+        find /world/users
+        | where active == true
+        | save /world/output
+      }`,
+    ]);
+
+    const observe = vi.fn(async () => {});
+
+    const result = await runDevLoop({
+      intent: "Build an ETL pipeline: read users, filter active ones, save to /world/output",
+      max_iterations: 6,
+      think,
+      runner: realRunner(ctx),
+      validate: realValidate(),
+      observe,
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.iterations).toBe(5);
+    expect(observe).toHaveBeenCalledTimes(5);
+
+    // Verify first 4 observations have errors
+    for (let i = 0; i < 4; i++) {
+      const obs = (observe.mock.calls as any)[i][0];
+      expect(
+        obs!.validation_errors?.length > 0 || obs!.result?.status === "error",
+      ).toBe(true);
+    }
+
+    // 5th observation is success
+    const lastObs = (observe.mock.calls as any)[4][0];
+    expect(lastObs!.result.status).toBe("ok");
+
+    // Verify correction prompts contain useful error info
+    expect(corrections).toHaveLength(4);
+
+    // Correction 1: should mention unknown command "fetch"
+    expect(corrections[0]).toContain("ParseError");
+
+    // Correction 2: should mention unknown command "filter"
+    expect(corrections[1]).toContain("ParseError");
+
+    // Correction 3: should mention syntax issue
+    expect(corrections[2]).toContain("ParseError");
+  });
+
+  it("readonly violation → correction → remove save → success", async () => {
+    const { ctx } = makeCtx({
+      "/public/data": [{ id: 1, value: "hello" }],
+    });
+
+    const { think } = scriptedThink([
+      // Attempt 1: @readonly with save → violation
+      '@readonly\njob read { find /public/data | save /public/out }',
+
+      // Attempt 2: remove save, readonly is fine
+      '@readonly\njob read { find /public/data }',
+    ]);
+
+    const result = await runDevLoop({
+      intent: "read data in readonly mode",
+      max_iterations: 3,
+      think,
+      runner: realRunner(ctx),
+      validate: realValidate(),
+      observe: vi.fn(async () => {}),
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.iterations).toBe(2);
+  });
+
+  it("progressive fix: syntax → type → success, with dry-run world isolation", async () => {
+    const { ctx, world } = makeCtx({
+      "/world/items": [
+        { id: 1, name: "Widget", price: 10 },
+        { id: 2, name: "Gadget", price: 25 },
+      ],
+    });
+
+    const { think, corrections } = scriptedThink([
+      // Attempt 1: syntax error — missing closing brace
+      'job transform { find /world/items | map { name name, cost price }',
+
+      // Attempt 2: unknown command
+      'job transform { find /world/items | transform name | save /world/out }',
+
+      // Attempt 3: correct
+      `job transform {
+        find /world/items
+        | map { name name, cost price }
+        | save /world/out
+      }`,
+    ]);
+
+    const observe = vi.fn(async () => {});
+
+    const result = await runDevLoop({
+      intent: "transform items: extract name and price as cost",
+      max_iterations: 5,
+      think,
+      runner: realRunner(ctx, "dry-run"),
+      validate: realValidate(),
+      observe,
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.iterations).toBe(3);
+
+    // INV-1: dry-run — world must NOT be written to
+    expect(Object.keys(world.written)).toHaveLength(0);
+
+    // Corrections should have error details
+    expect(corrections.length).toBeGreaterThanOrEqual(1);
+  });
+
+  it("annotation errors → fix → readonly violation → fix → success", async () => {
+    const { ctx } = makeCtx({
+      "/world/logs": [{ ts: 1, msg: "hello" }],
+    });
+
+    const { think } = scriptedThink([
+      // Attempt 1: invalid annotation argument
+      '@approval(robot)\njob audit { find /world/logs }',
+
+      // Attempt 2: readonly + save = violation
+      '@readonly\njob audit { find /world/logs | save /world/archive }',
+
+      // Attempt 3: correct — readonly with no save
+      '@readonly\njob audit { find /world/logs }',
+    ]);
+
+    const result = await runDevLoop({
+      intent: "audit logs read-only",
+      max_iterations: 4,
+      think,
+      runner: realRunner(ctx),
+      validate: realValidate(),
+      observe: vi.fn(async () => {}),
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.iterations).toBe(3);
+  });
+
+  it("undefined variable → fix → duplicate variable → fix → success", async () => {
+    const { ctx } = makeCtx({
+      "/world/scores": [{ name: "A", score: 90 }, { name: "B", score: 50 }],
+    });
+
+    const { think } = scriptedThink([
+      // Attempt 1: undefined variable $min
+      'job filter { find /world/scores | where score > $min }',
+
+      // Attempt 2: duplicate let
+      'let min = 70\nlet min = 80\njob filter { find /world/scores | where score > $min }',
+
+      // Attempt 3: correct
+      'let min = 70\njob filter { find /world/scores | where score > $min }',
+    ]);
+
+    const result = await runDevLoop({
+      intent: "filter scores above threshold",
+      max_iterations: 4,
+      think,
+      runner: realRunner(ctx),
+      validate: realValidate(),
+      observe: vi.fn(async () => {}),
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.iterations).toBe(3);
+  });
+
+  it("exhausts all iterations with different errors → returns all observations", async () => {
+    const { ctx } = makeCtx({});
+
+    const { think } = scriptedThink([
+      'job x { foobar }',                    // unknown command
+      'job x { find /a | unknowncmd }',       // unknown command
+      'job x { find /a | save /b | where x > 1 }',  // type mismatch
+    ]);
+
+    const observe = vi.fn(async () => {});
+
+    const result = await runDevLoop({
+      intent: "do something",
+      max_iterations: 3,
+      think,
+      runner: realRunner(ctx),
+      validate: realValidate(),
+      observe,
+    });
+
+    expect(result.status).toBe("error");
+    expect(result.iterations).toBe(3);
+    expect(result.observations).toHaveLength(3);
+
+    // Each observation should have distinct error info
+    const errorMessages = result.observations.map((obs: any) => {
+      if (obs.validation_errors) return obs.validation_errors[0]?.message;
+      if (obs.result?.failedAt) return obs.result.failedAt.message;
+      return obs.error;
+    });
+    // All should be defined
+    for (const msg of errorMessages) {
+      expect(msg).toBeDefined();
+    }
+    // Errors should be different (different mistakes each time)
+    const unique = new Set(errorMessages);
+    expect(unique.size).toBeGreaterThanOrEqual(2);
+  });
+
+  it("complex pipeline: find → where → map → tee → save, with mid-pipeline error fix", async () => {
+    const { ctx, world } = makeCtx({
+      "/world/products": [
+        { id: 1, name: "Laptop", category: "electronics", price: 999 },
+        { id: 2, name: "Shirt", category: "clothing", price: 29 },
+        { id: 3, name: "Phone", category: "electronics", price: 699 },
+      ],
+    });
+
+    const { think } = scriptedThink([
+      // Attempt 1: typo "wher" instead of "where"
+      `job etl {
+        find /world/products
+        | wher category == "electronics"
+        | save /world/electronics
+      }`,
+
+      // Attempt 2: correct complex pipeline
+      `job etl {
+        find /world/products
+        | where category == "electronics"
+        | map { name name, price price }
+        | save /world/electronics
+      }`,
+    ]);
+
+    const result = await runDevLoop({
+      intent: "extract electronics products",
+      max_iterations: 3,
+      mode: "live",
+      think,
+      runner: realRunner(ctx, "live"),
+      validate: realValidate(),
+      observe: vi.fn(async () => {}),
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.iterations).toBe(2);
+    expect(result.mode).toBe("live");
+    // In live mode, world should be written to
+    expect(world.written["/world/electronics"]).toBeDefined();
+  });
+
+  it("empty script from LLM → correction → valid script → success", async () => {
+    const { ctx } = makeCtx({ "/data": [{ x: 1 }] });
+
+    let callCount = 0;
+    const think = vi.fn(async () => {
+      callCount++;
+      if (callCount === 1) {
+        // LLM returns empty (no tool_calls, no content)
+        return { kind: "completed", response: { content: "", tool_calls: [] }, toolResults: [] };
+      }
+      // Second call: valid script
+      return {
+        kind: "completed",
+        response: {
+          content: "",
+          tool_calls: [{ id: "tc-2", name: "ash_run", arguments: JSON.stringify({ script: 'job x { find /data }' }) }],
+        },
+        toolResults: [],
+      };
+    });
+
+    const result = await runDevLoop({
+      intent: "read data",
+      max_iterations: 3,
+      think,
+      runner: realRunner(ctx),
+      validate: realValidate(),
+      observe: vi.fn(async () => {}),
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.iterations).toBe(2);
+  });
+
+  it("LLM sends script via content (not tool_call) → still works", async () => {
+    const { ctx } = makeCtx({ "/items": [{ a: 1 }] });
+
+    const think = vi.fn(async () => ({
+      kind: "completed",
+      response: {
+        // No tool_calls — script in content directly
+        content: 'job x { find /items }',
+        tool_calls: undefined,
+      },
+      toolResults: [],
+    }));
+
+    const result = await runDevLoop({
+      intent: "get items",
+      max_iterations: 1,
+      think,
+      runner: realRunner(ctx),
+      validate: realValidate(),
+      observe: vi.fn(async () => {}),
+    });
+
+    expect(result.status).toBe("ok");
+  });
+
+  it("stress: 5 consecutive different error types before success", async () => {
+    const { ctx } = makeCtx({ "/d": [{ v: 1 }] });
+
+    const { think, corrections } = scriptedThink([
+      // 1: unknown command
+      'job x { badcmd }',
+      // 2: unterminated string
+      'job x { find /d | output "hello }',
+      // 3: missing brace
+      'job x { find /d',
+      // 4: invalid annotation
+      '@timeout(-5)\njob x { find /d }',
+      // 5: duplicate var
+      'let a = 1\nlet a = 2\njob x { find /d }',
+      // 6: success
+      'job x { find /d }',
+    ]);
+
+    const observe = vi.fn(async () => {});
+
+    const result = await runDevLoop({
+      intent: "read data from /d",
+      max_iterations: 7,
+      think,
+      runner: realRunner(ctx),
+      validate: realValidate(),
+      observe,
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.iterations).toBe(6);
+    expect(corrections).toHaveLength(5);
+
+    // All corrections should contain "ParseError" (from validation)
+    for (const c of corrections) {
+      expect(c).toContain("previous ASH script failed");
+    }
+  });
+});