@aigne/ash 0.0.1

package/src/ai-dev-loop/dev-loop.test.ts

@@ -0,0 +1,364 @@
+import { describe, it, expect, vi } from "vitest";
+import { runDevLoop } from "./dev-loop.js";
+import type { AshRunResult } from "./ash-run-result.js";
+
+/** Fake think: returns tool calls with ASH script content */
+function fakeThink(scripts: string[]) {
+  let call = 0;
+  return vi.fn(async (_req: any) => ({
+    kind: "completed" as const,
+    response: {
+      content: scripts[call] ?? "",
+      tool_calls: [
+        { id: `tc-${call}`, name: "ash_run", arguments: JSON.stringify({ script: scripts[call++] ?? "" }) },
+      ],
+    },
+    toolResults: [],
+  }));
+}
+
+/** Fake runner: returns success or failure based on the script */
+function fakeRunner(failScripts: Set<string> = new Set()) {
+  return vi.fn(async (source: string): Promise<AshRunResult> => {
+    if (failScripts.has(source)) {
+      return {
+        status: "error",
+        steps: [{ step: 1, command: "find", status: "error", duration_ms: 5 }],
+        failedAt: { kind: "RuntimeError", message: `Failed: ${source}` },
+        duration_ms: 5,
+      };
+    }
+    return {
+      status: "ok",
+      steps: [{ step: 1, command: "find", status: "ok", duration_ms: 5, output: [{ id: 1 }] }],
+      output: [{ id: 1 }],
+      duration_ms: 5,
+    };
+  });
+}
+
+function fakeValidate() {
+  return vi.fn(async (_source: string) => []);
+}
+
+function fakeObserve() {
+  return vi.fn(async (_data: any) => {});
+}
+
+describe("AI Dev Loop Core", () => {
+  // ── Happy Path ─────────────────────────────────────────
+
+  it("success on first try", async () => {
+    const think = fakeThink(['job "test" { find /users }']);
+    const runner = fakeRunner();
+    const validate = fakeValidate();
+    const observe = fakeObserve();
+
+    const result = await runDevLoop({
+      intent: "find all users",
+      max_iterations: 3,
+      think,
+      runner,
+      validate,
+      observe,
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.iterations).toBe(1);
+    expect(result.finalResult?.status).toBe("ok");
+    expect(think).toHaveBeenCalledTimes(1);
+  });
+
+  it("first run fails → LLM corrects → second run succeeds", async () => {
+    const badScript = "bad script";
+    const goodScript = 'job "fixed" { find /users }';
+    const think = fakeThink([badScript, goodScript]);
+    const runner = fakeRunner(new Set([badScript]));
+    const validate = fakeValidate();
+    const observe = fakeObserve();
+
+    const result = await runDevLoop({
+      intent: "find users",
+      max_iterations: 5,
+      think,
+      runner,
+      validate,
+      observe,
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.iterations).toBe(2);
+    expect(think).toHaveBeenCalledTimes(2);
+  });
+
+  it("returns final AshRunResult and all observations", async () => {
+    const think = fakeThink(["script1"]);
+    const runner = fakeRunner();
+    const observe = fakeObserve();
+
+    const result = await runDevLoop({
+      intent: "do something",
+      max_iterations: 3,
+      think,
+      runner,
+      validate: fakeValidate(),
+      observe,
+    });
+
+    expect(result.finalResult).toBeDefined();
+    expect(result.observations.length).toBeGreaterThanOrEqual(1);
+  });
+
+  it("each iteration calls observe with execution result", async () => {
+    const think = fakeThink(["s1", "s2"]);
+    const runner = fakeRunner(new Set(["s1"]));
+    const observe = fakeObserve();
+
+    await runDevLoop({
+      intent: "test",
+      max_iterations: 3,
+      think,
+      runner,
+      validate: fakeValidate(),
+      observe,
+    });
+
+    expect(observe).toHaveBeenCalledTimes(2);
+  });
+
+  it("calls validate before run", async () => {
+    const callOrder: string[] = [];
+    const validate = vi.fn(async () => { callOrder.push("validate"); return []; });
+    const runner = vi.fn(async (): Promise<AshRunResult> => {
+      callOrder.push("run");
+      return { status: "ok", steps: [], output: [], duration_ms: 0 };
+    });
+
+    await runDevLoop({
+      intent: "test",
+      max_iterations: 1,
+      think: fakeThink(["script"]),
+      runner,
+      validate,
+      observe: fakeObserve(),
+    });
+
+    expect(callOrder).toEqual(["validate", "run"]);
+  });
+
+  it("successful loop returns iteration count", async () => {
+    const think = fakeThink(["a", "b", "c"]);
+    const runner = fakeRunner(new Set(["a", "b"]));
+
+    const result = await runDevLoop({
+      intent: "test",
+      max_iterations: 5,
+      think,
+      runner,
+      validate: fakeValidate(),
+      observe: fakeObserve(),
+    });
+
+    expect(result.iterations).toBe(3);
+  });
+
+  // ── Bad Path ───────────────────────────────────────────
+
+  it("exceeds max_iterations → returns last failure (INV-4)", async () => {
+    const think = fakeThink(["bad1", "bad2", "bad3"]);
+    const runner = fakeRunner(new Set(["bad1", "bad2", "bad3"]));
+
+    const result = await runDevLoop({
+      intent: "will fail",
+      max_iterations: 3,
+      think,
+      runner,
+      validate: fakeValidate(),
+      observe: fakeObserve(),
+    });
+
+    expect(result.status).toBe("error");
+    expect(result.iterations).toBe(3);
+    expect(result.finalResult?.status).toBe("error");
+    expect(result.observations).toHaveLength(3);
+  });
+
+  it("validate catches parse error → correction prompt without run", async () => {
+    const validate = vi.fn(async (source: string) => {
+      if (source === "bad syntax") {
+        return [{ kind: "ParseError" as const, message: "unexpected token", line: 1 }];
+      }
+      return [];
+    });
+    const runner = fakeRunner();
+    const think = fakeThink(["bad syntax", "good script"]);
+
+    const result = await runDevLoop({
+      intent: "test",
+      max_iterations: 3,
+      think,
+      runner,
+      validate,
+      observe: fakeObserve(),
+    });
+
+    // Runner should only be called for the good script
+    expect(runner).toHaveBeenCalledTimes(1);
+    expect(result.status).toBe("ok");
+  });
+
+  it("LLM returns empty response → treated as error", async () => {
+    const think = vi.fn(async () => ({
+      kind: "completed" as const,
+      response: { content: "", tool_calls: undefined },
+      toolResults: [],
+    }));
+
+    const result = await runDevLoop({
+      intent: "test",
+      max_iterations: 2,
+      think,
+      runner: fakeRunner(),
+      validate: fakeValidate(),
+      observe: fakeObserve(),
+    });
+
+    expect(result.status).toBe("error");
+  });
+
+  it("think throws → loop terminates with error", async () => {
+    const think = vi.fn(async () => { throw new Error("LLM down"); });
+
+    const result = await runDevLoop({
+      intent: "test",
+      max_iterations: 3,
+      think,
+      runner: fakeRunner(),
+      validate: fakeValidate(),
+      observe: fakeObserve(),
+    });
+
+    expect(result.status).toBe("error");
+    expect(result.error).toContain("LLM down");
+  });
+
+  it("budget exceeded mid-loop → loop terminates with error", async () => {
+    let callCount = 0;
+    const think = vi.fn(async () => {
+      callCount++;
+      if (callCount === 2) throw new Error("BudgetExceeded: token limit reached");
+      return {
+        kind: "completed" as const,
+        response: {
+          content: "",
+          tool_calls: [{ id: "tc-1", name: "ash_run", arguments: JSON.stringify({ script: "fail" }) }],
+        },
+        toolResults: [],
+      };
+    });
+
+    const result = await runDevLoop({
+      intent: "test",
+      max_iterations: 5,
+      think,
+      runner: fakeRunner(new Set(["fail"])),
+      validate: fakeValidate(),
+      observe: fakeObserve(),
+    });
+
+    expect(result.status).toBe("error");
+    expect(result.error).toContain("BudgetExceeded");
+    expect(result.iterations).toBe(2);
+  });
+
+  it("extractScript: script from content fallback when no tool_calls", async () => {
+    const think = vi.fn(async () => ({
+      kind: "completed" as const,
+      response: { content: 'job "fromContent" { find /x }', tool_calls: undefined },
+      toolResults: [],
+    }));
+
+    const result = await runDevLoop({
+      intent: "test",
+      max_iterations: 1,
+      think,
+      runner: fakeRunner(),
+      validate: fakeValidate(),
+      observe: fakeObserve(),
+    });
+
+    expect(result.status).toBe("ok");
+  });
+
+  // ── Edge Cases ─────────────────────────────────────────
+
+  it("max_iterations=1 → single attempt, no correction", async () => {
+    const think = fakeThink(["script"]);
+    const runner = fakeRunner(new Set(["script"]));
+
+    const result = await runDevLoop({
+      intent: "test",
+      max_iterations: 1,
+      think,
+      runner,
+      validate: fakeValidate(),
+      observe: fakeObserve(),
+    });
+
+    expect(result.status).toBe("error");
+    expect(result.iterations).toBe(1);
+    expect(think).toHaveBeenCalledTimes(1);
+  });
+
+  it("LLM fixes on last iteration → success", async () => {
+    const scripts = ["bad1", "bad2", "bad3", "bad4", "good"];
+    const think = fakeThink(scripts);
+    const runner = fakeRunner(new Set(["bad1", "bad2", "bad3", "bad4"]));
+
+    const result = await runDevLoop({
+      intent: "test",
+      max_iterations: 5,
+      think,
+      runner,
+      validate: fakeValidate(),
+      observe: fakeObserve(),
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.iterations).toBe(5);
+  });
+
+  it("all iterations fail → returns array of all failures", async () => {
+    const think = fakeThink(["f1", "f2"]);
+    const runner = fakeRunner(new Set(["f1", "f2"]));
+
+    const result = await runDevLoop({
+      intent: "test",
+      max_iterations: 2,
+      think,
+      runner,
+      validate: fakeValidate(),
+      observe: fakeObserve(),
+    });
+
+    expect(result.observations).toHaveLength(2);
+  });
+
+  // ── Security ───────────────────────────────────────────
+
+  it("default mode is dry-run (INV-1)", async () => {
+    const think = fakeThink(["script"]);
+    const runner = fakeRunner();
+
+    const result = await runDevLoop({
+      intent: "test",
+      max_iterations: 1,
+      think,
+      runner,
+      validate: fakeValidate(),
+      observe: fakeObserve(),
+    });
+
+    expect(result.mode).toBe("dry-run");
+  });
+});

package/src/ai-dev-loop/dev-loop.ts

@@ -0,0 +1,156 @@
+/**
+ * AI Dev Loop Core — bounded generate → validate → run → observe → correct loop.
+ *
+ * Invariants enforced:
+ * - INV-1: Default mode is dry-run
+ * - INV-3: Observe only, never commit to long-term memory
+ * - INV-4: Bounded by max_iterations
+ */
+
+import type { AshRunResult, AshRunFailure } from "./ash-run-result.js";
+import type { AshTypedError } from "./ash-typed-error.js";
+import { buildSystemPrompt, buildCorrectionPrompt } from "./system-prompt.js";
+
+export interface DevLoopConfig {
+  intent: string;
+  max_iterations: number;
+  mode?: "dry-run" | "live";
+  think: (request: any) => Promise<any>;
+  runner: (source: string) => Promise<AshRunResult>;
+  validate: (source: string) => Promise<AshTypedError[]>;
+  observe: (data: any) => Promise<void>;
+}
+
+export interface DevLoopResult {
+  status: "ok" | "error";
+  iterations: number;
+  mode: "dry-run" | "live";
+  finalResult?: AshRunResult;
+  observations: any[];
+  error?: string;
+}
+
+/**
+ * Run the AI dev loop: think → extract ASH → validate → run → observe → correct or return.
+ */
+export async function runDevLoop(config: DevLoopConfig): Promise<DevLoopResult> {
+  const mode = config.mode ?? "dry-run";
+  const observations: any[] = [];
+  let lastFailure: AshRunFailure | undefined;
+  let lastScript: string | undefined;
+
+  for (let i = 0; i < config.max_iterations; i++) {
+    // 1. Think — ask LLM for ASH script
+    let thinkResult: any;
+    try {
+      const messages: any[] = [];
+      if (i === 0) {
+        messages.push({ role: "system", content: buildSystemPrompt({ max_iterations: config.max_iterations }) });
+        messages.push({ role: "user", content: config.intent });
+      } else {
+        messages.push({ role: "user", content: buildCorrectionPrompt(lastFailure!, lastScript!) });
+      }
+      thinkResult = await config.think({ messages });
+    } catch (err: any) {
+      return { status: "error", iterations: i + 1, mode, observations, error: err.message };
+    }
+
+    // 2. Extract script from tool call or content
+    const script = extractScript(thinkResult);
+    if (!script) {
+      const obs = { iteration: i + 1, error: "empty_response" };
+      observations.push(obs);
+      await config.observe(obs);
+      // If no script, treat as failure for this iteration
+      if (i === config.max_iterations - 1) {
+        return { status: "error", iterations: i + 1, mode, observations, error: "LLM returned no script" };
+      }
+      // Create a synthetic failure for correction
+      lastFailure = {
+        status: "error",
+        steps: [],
+        failedAt: { kind: "RuntimeError", message: "LLM returned no ASH script" } as AshTypedError,
+        duration_ms: 0,
+      };
+      lastScript = "";
+      continue;
+    }
+
+    lastScript = script;
+
+    // 3. Validate
+    const errors = await config.validate(script);
+    if (errors.length > 0) {
+      const obs = { iteration: i + 1, script, validation_errors: errors };
+      observations.push(obs);
+      await config.observe(obs);
+
+      if (i === config.max_iterations - 1) {
+        return {
+          status: "error",
+          iterations: i + 1,
+          mode,
+          observations,
+          finalResult: {
+            status: "error",
+            steps: [],
+            failedAt: errors[0],
+            duration_ms: 0,
+          },
+        };
+      }
+
+      lastFailure = {
+        status: "error",
+        steps: [],
+        failedAt: errors[0] as AshTypedError,
+        duration_ms: 0,
+      };
+      continue;
+    }
+
+    // 4. Run
+    const result = await config.runner(script);
+
+    // 5. Observe
+    const obs = { iteration: i + 1, script, result };
+    observations.push(obs);
+    await config.observe(obs);
+
+    // 6. Check result
+    if (result.status === "ok") {
+      return { status: "ok", iterations: i + 1, mode, finalResult: result, observations };
+    }
+
+    // Failed — prepare for correction
+    lastFailure = result as AshRunFailure;
+  }
+
+  // Exhausted all iterations
+  return {
+    status: "error",
+    iterations: config.max_iterations,
+    mode,
+    finalResult: lastFailure,
+    observations,
+  };
+}
+
+function extractScript(thinkResult: any): string | undefined {
+  // Try tool_calls first (ash_run call)
+  const toolCalls = thinkResult?.response?.tool_calls;
+  if (Array.isArray(toolCalls)) {
+    for (const tc of toolCalls) {
+      if (tc.name === "ash_run") {
+        try {
+          const args = typeof tc.arguments === "string" ? JSON.parse(tc.arguments) : tc.arguments;
+          if (args?.script) return args.script;
+        } catch { /* ignore parse error */ }
+      }
+    }
+  }
+  // Fall back to content
+  const content = thinkResult?.response?.content;
+  if (typeof content === "string" && content.trim()) return content.trim();
+  return undefined;
+}

package/src/ai-dev-loop/dry-run.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect } from "vitest";
+import { runStructured } from "./structured-runner.js";
+import type { WorldInterface, JobLogger } from "../compiler.js";
+import { isAshRunSuccess, isAshRunFailure } from "./ash-run-result.js";
+
+function makeWorld(data: Record<string, unknown[]> = {}): WorldInterface & {
+  written: Record<string, unknown[]>;
+  published: Record<string, unknown[]>;
+} {
+  const written: Record<string, unknown[]> = {};
+  const published: Record<string, unknown[]> = {};
+  return {
+    read(path: string) { return data[path] ?? []; },
+    write(path: string, records: unknown[]) { written[path] = records; },
+    publish(topic: string, records: unknown[]) { published[topic] = records; },
+    written,
+    published,
+  };
+}
+
+function makeLogger(): JobLogger { return { log() {} }; }
+
+function makeCtx(data: Record<string, unknown[]> = {}) {
+  return {
+    world: makeWorld(data),
+    caps: new Set(["*"]),
+    logger: makeLogger(),
+  };
+}
+
+describe("Dry-Run Mode", () => {
+  // ── Happy Path ─────────────────────────────────────────
+
+  it("dry-run find → reads world normally", async () => {
+    const ctx = makeCtx({ "/users": [{ name: "Alice" }] });
+    const result = await runStructured('job "test" { find /users }', ctx, { mode: "dry-run" });
+    expect(isAshRunSuccess(result)).toBe(true);
+  });
+
+  it("dry-run save → does NOT write to world", async () => {
+    const ctx = makeCtx({ "/users": [{ name: "Alice" }] });
+    await runStructured('job "test" { find /users | save /output }', ctx, { mode: "dry-run" });
+    expect(ctx.world.written["/output"]).toBeUndefined();
+  });
+
+  it("dry-run publish → does NOT publish", async () => {
+    const ctx = makeCtx({ "/data": [{ x: 1 }] });
+    await runStructured('job "test" { find /data | publish /events }', ctx, { mode: "dry-run" });
+    expect(ctx.world.published["/events"]).toBeUndefined();
+  });
+
+  it("dry-run returns same AshRunResult shape as live", async () => {
+    const ctx = makeCtx({ "/data": [{ x: 1 }] });
+    const dryResult = await runStructured('job "test" { find /data }', ctx, { mode: "dry-run" });
+    const liveResult = await runStructured('job "test" { find /data }', ctx, { mode: "live" });
+    expect(dryResult.status).toBe(liveResult.status);
+  });
+
+  // ── Bad Path ───────────────────────────────────────────
+
+  it("dry-run with invalid script → same parse error as live", async () => {
+    const ctx = makeCtx();
+    const result = await runStructured('job "test" { find }', ctx, { mode: "dry-run" });
+    expect(isAshRunFailure(result)).toBe(true);
+  });
+
+  // ── Security ───────────────────────────────────────────
+
+  it("dry-run MUST NOT write to world (INV-1)", async () => {
+    const ctx = makeCtx({ "/users": [{ name: "Alice" }] });
+    await runStructured('job "test" { find /users | save /backup }', ctx, { mode: "dry-run" });
+    expect(Object.keys(ctx.world.written)).toHaveLength(0);
+  });
+
+  it("dry-run MUST NOT publish events", async () => {
+    const ctx = makeCtx({ "/data": [{ x: 1 }] });
+    await runStructured('job "test" { find /data | publish /events }', ctx, { mode: "dry-run" });
+    expect(Object.keys(ctx.world.published)).toHaveLength(0);
+  });
+
+  // ── Data Damage ────────────────────────────────────────
+
+  it("verify world state unchanged after dry-run of save+publish pipeline", async () => {
+    const data = { "/users": [{ name: "Alice" }, { name: "Bob" }] };
+    const ctx = makeCtx(data);
+    await runStructured(
+      'job "test" { find /users | save /archive }',
+      ctx,
+      { mode: "dry-run" },
+    );
+    // World reads unchanged
+    expect(ctx.world.read("/users")).toEqual([{ name: "Alice" }, { name: "Bob" }]);
+    // No writes happened
+    expect(ctx.world.written["/archive"]).toBeUndefined();
+  });
+
+  // ── Live mode comparison ───────────────────────────────
+
+  it("switching from dry-run to live on same script → live actually writes", async () => {
+    const ctx = makeCtx({ "/data": [{ x: 1 }] });
+    await runStructured('job "test" { find /data | save /out }', ctx, { mode: "dry-run" });
+    expect(ctx.world.written["/out"]).toBeUndefined();
+
+    await runStructured('job "test" { find /data | save /out }', ctx, { mode: "live" });
+    expect(ctx.world.written["/out"]).toBeDefined();
+  });
+});