ai-spec-dev 0.31.0 → 0.35.0

package/tests/dsl-validator.test.ts

@@ -0,0 +1,283 @@
+import { describe, it, expect } from "vitest";
+import { validateDsl } from "../core/dsl-validator";
+
+// ─── Fixtures ─────────────────────────────────────────────────────────────────
+
+const VALID_DSL = {
+  version: "1.0",
+  feature: {
+    id: "user-login",
+    title: "User Login",
+    description: "Allows users to authenticate with email and password",
+  },
+  models: [
+    {
+      name: "User",
+      description: "Application user",
+      fields: [
+        { name: "id", type: "String", required: true },
+        { name: "email", type: "String", required: true, unique: true },
+        { name: "password", type: "String", required: true },
+      ],
+    },
+  ],
+  endpoints: [
+    {
+      id: "EP-001",
+      method: "POST",
+      path: "/api/auth/login",
+      description: "Authenticate user and return JWT token",
+      auth: false,
+      successStatus: 200,
+      successDescription: "JWT token returned",
+      request: {
+        body: { email: "string (email format)", password: "string (min 8 chars)" },
+      },
+      errors: [
+        { status: 400, code: "INVALID_EMAIL", description: "Email format is invalid" },
+        { status: 401, code: "INVALID_CREDENTIALS", description: "Password is incorrect" },
+      ],
+    },
+  ],
+  behaviors: [
+    {
+      id: "BHV-001",
+      description: "Failed login attempts are rate-limited to 5 per minute",
+    },
+  ],
+};
+
+// ─── Valid DSL ────────────────────────────────────────────────────────────────
+
+describe("validateDsl — valid input", () => {
+  it("accepts a well-formed DSL", () => {
+    const result = validateDsl(VALID_DSL);
+    expect(result.valid).toBe(true);
+  });
+
+  it("accepts empty models array", () => {
+    const result = validateDsl({ ...VALID_DSL, models: [] });
+    expect(result.valid).toBe(true);
+  });
+
+  it("accepts empty endpoints array", () => {
+    const result = validateDsl({ ...VALID_DSL, endpoints: [] });
+    expect(result.valid).toBe(true);
+  });
+
+  it("accepts empty behaviors array", () => {
+    const result = validateDsl({ ...VALID_DSL, behaviors: [] });
+    expect(result.valid).toBe(true);
+  });
+
+  it("accepts DSL without optional components field", () => {
+    const { ...dsl } = VALID_DSL;
+    const result = validateDsl(dsl);
+    expect(result.valid).toBe(true);
+  });
+
+  it("returns the typed DSL on success", () => {
+    const result = validateDsl(VALID_DSL);
+    if (result.valid) {
+      expect(result.dsl.feature.id).toBe("user-login");
+      expect(result.dsl.endpoints[0].method).toBe("POST");
+    }
+  });
+
+  it("accepts all valid HTTP methods", () => {
+    for (const method of ["GET", "POST", "PUT", "PATCH", "DELETE"] as const) {
+      const ep = { ...VALID_DSL.endpoints[0], method };
+      const result = validateDsl({ ...VALID_DSL, endpoints: [ep] });
+      expect(result.valid).toBe(true);
+    }
+  });
+});
+
+// ─── Invalid root structure ───────────────────────────────────────────────────
+
+describe("validateDsl — root structure errors", () => {
+  it("rejects null", () => {
+    const result = validateDsl(null);
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects a plain array", () => {
+    const result = validateDsl([]);
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects a string", () => {
+    const result = validateDsl("not-an-object");
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects wrong version", () => {
+    const result = validateDsl({ ...VALID_DSL, version: "2.0" });
+    expect(result.valid).toBe(false);
+    if (!result.valid) {
+      expect(result.errors.some((e) => e.path === "version")).toBe(true);
+    }
+  });
+});
+
+// ─── Feature validation ───────────────────────────────────────────────────────
+
+describe("validateDsl — feature validation", () => {
+  it("rejects missing feature.id", () => {
+    const result = validateDsl({ ...VALID_DSL, feature: { ...VALID_DSL.feature, id: "" } });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects missing feature.title", () => {
+    const result = validateDsl({ ...VALID_DSL, feature: { ...VALID_DSL.feature, title: "" } });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects missing feature.description", () => {
+    const result = validateDsl({ ...VALID_DSL, feature: { ...VALID_DSL.feature, description: "" } });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects non-object feature", () => {
+    const result = validateDsl({ ...VALID_DSL, feature: "login" });
+    expect(result.valid).toBe(false);
+  });
+});
+
+// ─── Endpoint validation ──────────────────────────────────────────────────────
+
+describe("validateDsl — endpoint validation", () => {
+  it("rejects invalid HTTP method", () => {
+    const ep = { ...VALID_DSL.endpoints[0], method: "CONNECT" };
+    const result = validateDsl({ ...VALID_DSL, endpoints: [ep] });
+    expect(result.valid).toBe(false);
+    if (!result.valid) {
+      expect(result.errors.some((e) => e.path.includes("method"))).toBe(true);
+    }
+  });
+
+  it("rejects path not starting with /", () => {
+    const ep = { ...VALID_DSL.endpoints[0], path: "api/login" };
+    const result = validateDsl({ ...VALID_DSL, endpoints: [ep] });
+    expect(result.valid).toBe(false);
+    if (!result.valid) {
+      expect(result.errors.some((e) => e.path.includes("path"))).toBe(true);
+    }
+  });
+
+  it("rejects non-boolean auth", () => {
+    const ep = { ...VALID_DSL.endpoints[0], auth: "yes" };
+    const result = validateDsl({ ...VALID_DSL, endpoints: [ep] });
+    expect(result.valid).toBe(false);
+    if (!result.valid) {
+      expect(result.errors.some((e) => e.path.includes("auth"))).toBe(true);
+    }
+  });
+
+  it("rejects out-of-range successStatus", () => {
+    const ep = { ...VALID_DSL.endpoints[0], successStatus: 99 };
+    const result = validateDsl({ ...VALID_DSL, endpoints: [ep] });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects successStatus > 599", () => {
+    const ep = { ...VALID_DSL.endpoints[0], successStatus: 600 };
+    const result = validateDsl({ ...VALID_DSL, endpoints: [ep] });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects missing endpoint description", () => {
+    const ep = { ...VALID_DSL.endpoints[0], description: "" };
+    const result = validateDsl({ ...VALID_DSL, endpoints: [ep] });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects non-string FieldMap value", () => {
+    const ep = {
+      ...VALID_DSL.endpoints[0],
+      request: { body: { email: { nested: "object" } } },
+    };
+    const result = validateDsl({ ...VALID_DSL, endpoints: [ep] });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects error entry with invalid status code", () => {
+    const ep = {
+      ...VALID_DSL.endpoints[0],
+      errors: [{ status: 999, code: "BAD", description: "Bad" }],
+    };
+    const result = validateDsl({ ...VALID_DSL, endpoints: [ep] });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects endpoints array exceeding MAX_ENDPOINTS (100)", () => {
+    const eps = Array.from({ length: 101 }, (_, i) => ({
+      ...VALID_DSL.endpoints[0],
+      id: `EP-${i.toString().padStart(3, "0")}`,
+      path: `/api/route-${i}`,
+    }));
+    const result = validateDsl({ ...VALID_DSL, endpoints: eps });
+    expect(result.valid).toBe(false);
+    if (!result.valid) {
+      expect(result.errors.some((e) => e.path === "endpoints")).toBe(true);
+    }
+  });
+});
+
+// ─── Model validation ─────────────────────────────────────────────────────────
+
+describe("validateDsl — model validation", () => {
+  it("rejects model with missing name", () => {
+    const model = { ...VALID_DSL.models[0], name: "" };
+    const result = validateDsl({ ...VALID_DSL, models: [model] });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects model field with non-boolean required", () => {
+    const model = {
+      ...VALID_DSL.models[0],
+      fields: [{ name: "id", type: "String", required: "yes" }],
+    };
+    const result = validateDsl({ ...VALID_DSL, models: [model] });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects model with non-array fields", () => {
+    const model = { ...VALID_DSL.models[0], fields: "not-an-array" };
+    const result = validateDsl({ ...VALID_DSL, models: [model] });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects models array exceeding MAX_MODELS (50)", () => {
+    const models = Array.from({ length: 51 }, (_, i) => ({
+      ...VALID_DSL.models[0],
+      name: `Model${i}`,
+    }));
+    const result = validateDsl({ ...VALID_DSL, models });
+    expect(result.valid).toBe(false);
+  });
+
+  it("rejects non-string relation entry", () => {
+    const model = { ...VALID_DSL.models[0], relations: [42] };
+    const result = validateDsl({ ...VALID_DSL, models: [model] });
+    expect(result.valid).toBe(false);
+  });
+});
+
+// ─── Error collection (all errors in one pass) ───────────────────────────────
+
+describe("validateDsl — error accumulation", () => {
+  it("collects multiple errors in a single validation run", () => {
+    const result = validateDsl({
+      version: "2.0",                        // wrong version
+      feature: { id: "", title: "", description: "" },  // all empty
+      models: [],
+      endpoints: [],
+      behaviors: [],
+    });
+    expect(result.valid).toBe(false);
+    if (!result.valid) {
+      expect(result.errors.length).toBeGreaterThan(1);
+    }
+  });
+});

package/tests/error-feedback.test.ts

@@ -0,0 +1,292 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import * as os from "os";
+import * as path from "path";
+import * as fs from "fs-extra";
+import { execSync } from "child_process";
+import { runErrorFeedback } from "../core/error-feedback";
+import type { AIProvider } from "../core/spec-generator";
+
+// ─── Module-level mock for child_process ──────────────────────────────────────
+// execSync is non-configurable so it must be mocked at module level, not with spyOn.
+
+vi.mock("child_process", () => ({
+  execSync: vi.fn(() => Buffer.from("")),
+}));
+
+const mockExecSync = vi.mocked(execSync);
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+async function makeTempDir(): Promise<string> {
+  const dir = path.join(os.tmpdir(), `ai-spec-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+  await fs.mkdirp(dir);
+  return dir;
+}
+
+const NOOP_PROVIDER: AIProvider = { generate: vi.fn().mockResolvedValue("// fixed") };
+
+// ─── All checks skipped ───────────────────────────────────────────────────────
+
+describe("runErrorFeedback — all checks skipped", () => {
+  it("returns true immediately when skipTests + skipLint + skipBuild are all true", async () => {
+    const spy = vi.spyOn(console, "log").mockImplementation(() => {});
+    const result = await runErrorFeedback(NOOP_PROVIDER, os.tmpdir(), null, {
+      skipTests: true,
+      skipLint: true,
+      skipBuild: true,
+    });
+    spy.mockRestore();
+    expect(result).toBe(true);
+  });
+
+  it("does not call execSync when all checks are skipped", async () => {
+    mockExecSync.mockClear();
+    const spy = vi.spyOn(console, "log").mockImplementation(() => {});
+    await runErrorFeedback(NOOP_PROVIDER, os.tmpdir(), null, {
+      skipTests: true,
+      skipLint: true,
+      skipBuild: true,
+    });
+    spy.mockRestore();
+    expect(mockExecSync).not.toHaveBeenCalled();
+  });
+});
+
+// ─── Detection: empty directory ───────────────────────────────────────────────
+
+describe("runErrorFeedback — empty project directory", () => {
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = await makeTempDir();
+    mockExecSync.mockClear();
+  });
+
+  afterEach(async () => {
+    await fs.remove(tmpDir);
+  });
+
+  it("returns true when no commands can be detected", async () => {
+    const spy = vi.spyOn(console, "log").mockImplementation(() => {});
+    const result = await runErrorFeedback(NOOP_PROVIDER, tmpDir);
+    spy.mockRestore();
+    expect(result).toBe(true);
+    expect(mockExecSync).not.toHaveBeenCalled();
+  });
+});
+
+// ─── Detection: Go project ────────────────────────────────────────────────────
+
+describe("runErrorFeedback — Go project detection", () => {
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = await makeTempDir();
+    await fs.writeFile(path.join(tmpDir, "go.mod"), "module example.com/myapp\n\ngo 1.21\n");
+    mockExecSync.mockClear();
+    mockExecSync.mockReturnValue(Buffer.from("ok"));
+  });
+
+  afterEach(async () => {
+    await fs.remove(tmpDir);
+  });
+
+  it("detects go.mod and runs go test", async () => {
+    const spy = vi.spyOn(console, "log").mockImplementation(() => {});
+    await runErrorFeedback(NOOP_PROVIDER, tmpDir, null, {
+      skipBuild: true,
+      maxCycles: 1,
+    });
+    spy.mockRestore();
+
+    const calls = mockExecSync.mock.calls.map((c) => String(c[0]));
+    expect(calls.some((cmd) => cmd.includes("go test") || cmd.includes("go vet"))).toBe(true);
+  });
+});
+
+// ─── Detection: Rust project ──────────────────────────────────────────────────
+
+describe("runErrorFeedback — Rust project detection", () => {
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = await makeTempDir();
+    await fs.writeFile(path.join(tmpDir, "Cargo.toml"), "[package]\nname = \"my-crate\"\n");
+    mockExecSync.mockClear();
+    mockExecSync.mockReturnValue(Buffer.from("test result: ok"));
+  });
+
+  afterEach(async () => {
+    await fs.remove(tmpDir);
+  });
+
+  it("detects Cargo.toml and runs cargo test", async () => {
+    const spy = vi.spyOn(console, "log").mockImplementation(() => {});
+    await runErrorFeedback(NOOP_PROVIDER, tmpDir, null, {
+      skipBuild: true,
+      maxCycles: 1,
+    });
+    spy.mockRestore();
+
+    const calls = mockExecSync.mock.calls.map((c) => String(c[0]));
+    expect(calls.some((cmd) => cmd.includes("cargo test"))).toBe(true);
+  });
+});
+
+// ─── Detection: Node.js project ──────────────────────────────────────────────
+
+describe("runErrorFeedback — Node.js project detection", () => {
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = await makeTempDir();
+    mockExecSync.mockClear();
+    mockExecSync.mockReturnValue(Buffer.from(""));
+  });
+
+  afterEach(async () => {
+    await fs.remove(tmpDir);
+  });
+
+  it("uses npm test when package.json has a test script", async () => {
+    await fs.writeJson(path.join(tmpDir, "package.json"), {
+      scripts: { test: "vitest run" },
+    });
+
+    const spy = vi.spyOn(console, "log").mockImplementation(() => {});
+    await runErrorFeedback(NOOP_PROVIDER, tmpDir, null, {
+      skipLint: true,
+      skipBuild: true,
+      maxCycles: 1,
+    });
+    spy.mockRestore();
+
+    const calls = mockExecSync.mock.calls.map((c) => String(c[0]));
+    expect(calls.some((cmd) => cmd === "npm test")).toBe(true);
+  });
+
+  it("uses npx vitest run when vitest.config.ts exists (no test script)", async () => {
+    await fs.writeJson(path.join(tmpDir, "package.json"), { scripts: {} });
+    await fs.writeFile(path.join(tmpDir, "vitest.config.ts"), "export default {}");
+
+    const spy = vi.spyOn(console, "log").mockImplementation(() => {});
+    await runErrorFeedback(NOOP_PROVIDER, tmpDir, null, {
+      skipLint: true,
+      skipBuild: true,
+      maxCycles: 1,
+    });
+    spy.mockRestore();
+
+    const calls = mockExecSync.mock.calls.map((c) => String(c[0]));
+    expect(calls.some((cmd) => cmd.includes("vitest run"))).toBe(true);
+  });
+
+  it("uses npm run lint when package.json has a lint script", async () => {
+    await fs.writeJson(path.join(tmpDir, "package.json"), {
+      scripts: { lint: "eslint ." },
+    });
+
+    const spy = vi.spyOn(console, "log").mockImplementation(() => {});
+    await runErrorFeedback(NOOP_PROVIDER, tmpDir, null, {
+      skipTests: true,
+      skipBuild: true,
+      maxCycles: 1,
+    });
+    spy.mockRestore();
+
+    const calls = mockExecSync.mock.calls.map((c) => String(c[0]));
+    expect(calls.some((cmd) => cmd === "npm run lint")).toBe(true);
+  });
+
+  it("uses npx tsc --noEmit when tsconfig.json exists", async () => {
+    await fs.writeJson(path.join(tmpDir, "package.json"), { scripts: {} });
+    await fs.writeFile(path.join(tmpDir, "tsconfig.json"), "{}");
+
+    const spy = vi.spyOn(console, "log").mockImplementation(() => {});
+    await runErrorFeedback(NOOP_PROVIDER, tmpDir, null, {
+      skipTests: true,
+      skipLint: true,
+      maxCycles: 1,
+    });
+    spy.mockRestore();
+
+    const calls = mockExecSync.mock.calls.map((c) => String(c[0]));
+    expect(calls.some((cmd) => cmd.includes("tsc --noEmit"))).toBe(true);
+  });
+
+  it("prefers vue-tsc when vue-tsc is in devDependencies", async () => {
+    await fs.writeJson(path.join(tmpDir, "package.json"), {
+      scripts: {},
+      devDependencies: { "vue-tsc": "^1.0.0" },
+    });
+    await fs.writeFile(path.join(tmpDir, "tsconfig.json"), "{}");
+
+    const spy = vi.spyOn(console, "log").mockImplementation(() => {});
+    await runErrorFeedback(NOOP_PROVIDER, tmpDir, null, {
+      skipTests: true,
+      skipLint: true,
+      maxCycles: 1,
+    });
+    spy.mockRestore();
+
+    const calls = mockExecSync.mock.calls.map((c) => String(c[0]));
+    expect(calls.some((cmd) => cmd.includes("vue-tsc"))).toBe(true);
+  });
+});
+
+// ─── Pass/fail outcomes ───────────────────────────────────────────────────────
+
+describe("runErrorFeedback — pass/fail outcomes", () => {
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = await makeTempDir();
+    await fs.writeJson(path.join(tmpDir, "package.json"), {
+      scripts: { test: "vitest run" },
+    });
+    mockExecSync.mockClear();
+  });
+
+  afterEach(async () => {
+    await fs.remove(tmpDir);
+  });
+
+  it("returns true when all checks pass", async () => {
+    mockExecSync.mockReturnValue(Buffer.from("All tests passed"));
+
+    const spy = vi.spyOn(console, "log").mockImplementation(() => {});
+    const result = await runErrorFeedback(NOOP_PROVIDER, tmpDir, null, {
+      skipLint: true,
+      skipBuild: true,
+      maxCycles: 1,
+    });
+    spy.mockRestore();
+
+    expect(result).toBe(true);
+  });
+
+  it("returns false when checks fail after all fix cycles", async () => {
+    // Always throw to simulate test failures with a file reference in the output
+    mockExecSync.mockImplementation(() => {
+      const err: Error & { stdout?: string; stderr?: string } = new Error("Tests failed");
+      err.stdout = "src/auth.ts:10:5 - error TS2345: Type mismatch\n";
+      throw err;
+    });
+
+    // Create the file so attemptFix can read it
+    await fs.mkdirp(path.join(tmpDir, "src"));
+    await fs.writeFile(path.join(tmpDir, "src/auth.ts"), "// broken code");
+
+    const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {});
+    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+    const result = await runErrorFeedback(NOOP_PROVIDER, tmpDir, null, {
+      skipLint: true,
+      skipBuild: true,
+      maxCycles: 2,
+    });
+    consoleSpy.mockRestore();
+    warnSpy.mockRestore();
+
+    expect(result).toBe(false);
+  });
+});