@codexa/cli 9.0.7 → 9.0.9

package/gates/standards-validator.test.ts

@@ -0,0 +1,447 @@
+import { describe, it, expect, beforeEach, mock, spyOn } from "bun:test";
+import { getDb } from "../db/connection";
+import { initSchema, runMigrations } from "../db/schema";
+import { writeFileSync, mkdirSync, rmSync, existsSync } from "fs";
+import { join } from "path";
+
+// Mock grepai functions before importing validator
+import * as patterns from "../commands/patterns";
+
+import { validateAgainstStandards, printValidationResult } from "./standards-validator";
+
+// ═══════════════════════════════════════════════════════════════
+// HELPERS
+// ═══════════════════════════════════════════════════════════════
+
+const TMP_DIR = join(process.cwd(), ".codexa", "test-tmp-standards");
+
+function ensureTmpDir() {
+  if (!existsSync(TMP_DIR)) {
+    mkdirSync(TMP_DIR, { recursive: true });
+  }
+}
+
+function createTmpFile(name: string, content: string): string {
+  ensureTmpDir();
+  const path = join(TMP_DIR, name);
+  writeFileSync(path, content);
+  return path;
+}
+
+function insertStandard(opts: {
+  category: string;
+  scope: string;
+  rule: string;
+  enforcement?: string;
+  anti_examples?: string;
+  semantic_query?: string | null;
+  expect?: string;
+}) {
+  const db = getDb();
+  db.run(
+    `INSERT INTO standards (category, scope, rule, anti_examples, enforcement, semantic_query, expect, source, created_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, 'test', datetime('now'))`,
+    [
+      opts.category,
+      opts.scope,
+      opts.rule,
+      opts.anti_examples || null,
+      opts.enforcement || "required",
+      opts.semantic_query || null,
+      opts.expect || "no_match",
+    ]
+  );
+}
+
+// ═══════════════════════════════════════════════════════════════
+// TESTS
+// ═══════════════════════════════════════════════════════════════
+
+describe("standards-validator", () => {
+  beforeEach(() => {
+    initSchema();
+    runMigrations();
+    const db = getDb();
+    db.run("DELETE FROM standards");
+
+    // Clean tmp files
+    if (existsSync(TMP_DIR)) {
+      rmSync(TMP_DIR, { recursive: true });
+    }
+  });
+
+  // ─────────────────────────────────────────────────────────
+  // BASE CASES
+  // ─────────────────────────────────────────────────────────
+
+  it("returns passed when no standards exist", () => {
+    const file = createTmpFile("test.ts", "export const x = 1;");
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(true);
+    expect(result.violations).toHaveLength(0);
+    expect(result.warnings).toHaveLength(0);
+  });
+
+  it("returns passed when files list is empty", () => {
+    insertStandard({ category: "code", scope: "all", rule: "No console.log" });
+    const result = validateAgainstStandards([], "all");
+    expect(result.passed).toBe(true);
+  });
+
+  // ─────────────────────────────────────────────────────────
+  // TEXTUAL VALIDATION (anti_examples)
+  // ─────────────────────────────────────────────────────────
+
+  it("detects anti_examples violation in file content", () => {
+    const file = createTmpFile("api.ts", 'console.log("debug"); export function handle() {}');
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "Nao usar console.log",
+      anti_examples: JSON.stringify(["console.log"]),
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(false);
+    expect(result.violations).toHaveLength(1);
+    expect(result.violations[0].detail).toContain("console.log");
+  });
+
+  it("passes when anti_examples not found in file", () => {
+    const file = createTmpFile("api.ts", "export function handle() { return 'ok'; }");
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "Nao usar console.log",
+      anti_examples: JSON.stringify(["console.log"]),
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(true);
+  });
+
+  it("treats recommended enforcement as warning not violation", () => {
+    const file = createTmpFile("api.ts", 'console.log("test");');
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "Evitar console.log",
+      anti_examples: JSON.stringify(["console.log"]),
+      enforcement: "recommended",
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(true); // recommended does NOT block
+    expect(result.warnings).toHaveLength(1);
+    expect(result.violations).toHaveLength(0);
+  });
+
+  it("filters standards by scope", () => {
+    const file = createTmpFile("api.ts", 'console.log("test");');
+
+    // Standard for frontend scope only
+    insertStandard({
+      category: "code",
+      scope: "frontend",
+      rule: "Frontend: no console.log",
+      anti_examples: JSON.stringify(["console.log"]),
+    });
+
+    // Validate with backend domain — should NOT match frontend standard
+    const result = validateAgainstStandards([file], "backend");
+    expect(result.passed).toBe(true);
+  });
+
+  it("matches scope 'all' for any domain", () => {
+    const file = createTmpFile("api.ts", 'console.log("test");');
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "No console.log",
+      anti_examples: JSON.stringify(["console.log"]),
+    });
+
+    const result = validateAgainstStandards([file], "backend");
+    expect(result.passed).toBe(false);
+  });
+
+  it("skips nonexistent files gracefully", () => {
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "No eval",
+      anti_examples: JSON.stringify(["eval("]),
+    });
+
+    const result = validateAgainstStandards(["/nonexistent/file.ts"], "all");
+    expect(result.passed).toBe(true);
+  });
+
+  it("handles invalid anti_examples JSON gracefully", () => {
+    const file = createTmpFile("api.ts", "export const x = 1;");
+    const db = getDb();
+    db.run(
+      `INSERT INTO standards (category, scope, rule, anti_examples, enforcement, source, created_at)
+       VALUES ('code', 'all', 'Bad JSON', 'not-valid-json', 'required', 'test', datetime('now'))`
+    );
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(true); // Invalid JSON should not crash
+  });
+
+  // ─────────────────────────────────────────────────────────
+  // SEMANTIC VALIDATION (grepai)
+  // ─────────────────────────────────────────────────────────
+
+  it("detects no_match violation when grepai finds matching file", () => {
+    const file = createTmpFile("handler.ts", "export function handle() {}");
+    const resolvedPath = join(TMP_DIR, "handler.ts").replace(/\\/g, "/");
+
+    const availSpy = spyOn(patterns, "isGrepaiAvailable").mockReturnValue(true);
+    const searchSpy = spyOn(patterns, "searchWithGrepai").mockReturnValue([
+      { path: resolvedPath, score: 0.85 },
+    ]);
+
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "No .then() chains",
+      semantic_query: "promise .then() chain instead of async await",
+      expect: "no_match",
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(false);
+    expect(result.violations).toHaveLength(1);
+    expect(result.violations[0].detail).toContain("Violacao semantica");
+    expect(result.violations[0].detail).toContain("0.85");
+
+    availSpy.mockRestore();
+    searchSpy.mockRestore();
+  });
+
+  it("passes no_match when grepai finds no matches", () => {
+    const file = createTmpFile("clean.ts", "export const x = 1;");
+
+    const availSpy = spyOn(patterns, "isGrepaiAvailable").mockReturnValue(true);
+    const searchSpy = spyOn(patterns, "searchWithGrepai").mockReturnValue([]);
+
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "No .then() chains",
+      semantic_query: "promise .then() chain",
+      expect: "no_match",
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(true);
+
+    availSpy.mockRestore();
+    searchSpy.mockRestore();
+  });
+
+  it("detects must_match violation when grepai finds no match", () => {
+    const file = createTmpFile("service.ts", "export function process() {}");
+
+    const availSpy = spyOn(patterns, "isGrepaiAvailable").mockReturnValue(true);
+    const searchSpy = spyOn(patterns, "searchWithGrepai").mockReturnValue([]);
+
+    insertStandard({
+      category: "practice",
+      scope: "all",
+      rule: "Must have error handling",
+      semantic_query: "try catch error handling",
+      expect: "must_match",
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(false);
+    expect(result.violations).toHaveLength(1);
+    expect(result.violations[0].detail).toContain("Padrao obrigatorio nao encontrado");
+
+    availSpy.mockRestore();
+    searchSpy.mockRestore();
+  });
+
+  it("passes must_match when grepai finds matching file", () => {
+    const file = createTmpFile("service.ts", "try { } catch (e) { }");
+    const resolvedPath = join(TMP_DIR, "service.ts").replace(/\\/g, "/");
+
+    const availSpy = spyOn(patterns, "isGrepaiAvailable").mockReturnValue(true);
+    const searchSpy = spyOn(patterns, "searchWithGrepai").mockReturnValue([
+      { path: resolvedPath, score: 0.9 },
+    ]);
+
+    insertStandard({
+      category: "practice",
+      scope: "all",
+      rule: "Must have error handling",
+      semantic_query: "try catch error handling",
+      expect: "must_match",
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(true);
+
+    availSpy.mockRestore();
+    searchSpy.mockRestore();
+  });
+
+  it("ignores grepai results below score threshold", () => {
+    const file = createTmpFile("maybe.ts", "export const x = 1;");
+    const resolvedPath = join(TMP_DIR, "maybe.ts").replace(/\\/g, "/");
+
+    const availSpy = spyOn(patterns, "isGrepaiAvailable").mockReturnValue(true);
+    const searchSpy = spyOn(patterns, "searchWithGrepai").mockReturnValue([
+      { path: resolvedPath, score: 0.5 }, // Below 0.7 threshold
+    ]);
+
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "No console.log",
+      semantic_query: "console.log statement",
+      expect: "no_match",
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(true); // Score too low = no match
+
+    availSpy.mockRestore();
+    searchSpy.mockRestore();
+  });
+
+  it("ignores grepai results for files not in validation list", () => {
+    const file = createTmpFile("target.ts", "export const x = 1;");
+
+    const availSpy = spyOn(patterns, "isGrepaiAvailable").mockReturnValue(true);
+    const searchSpy = spyOn(patterns, "searchWithGrepai").mockReturnValue([
+      { path: "/some/other/file.ts", score: 0.95 }, // High score but wrong file
+    ]);
+
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "No eval",
+      semantic_query: "eval() usage",
+      expect: "no_match",
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(true); // Match is for a different file
+
+    availSpy.mockRestore();
+    searchSpy.mockRestore();
+  });
+
+  // ─────────────────────────────────────────────────────────
+  // FALLBACK (grepai unavailable)
+  // ─────────────────────────────────────────────────────────
+
+  it("falls back to anti_examples when grepai not available", () => {
+    const file = createTmpFile("api.ts", 'eval("alert(1)");');
+
+    const availSpy = spyOn(patterns, "isGrepaiAvailable").mockReturnValue(false);
+
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "No eval",
+      anti_examples: JSON.stringify(["eval("]),
+      semantic_query: "eval usage for dynamic code execution",
+      expect: "no_match",
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(false); // Falls back to anti_examples check
+    expect(result.violations).toHaveLength(1);
+
+    availSpy.mockRestore();
+  });
+
+  it("skips must_match standards when grepai not available", () => {
+    const file = createTmpFile("service.ts", "export function process() {}");
+
+    const availSpy = spyOn(patterns, "isGrepaiAvailable").mockReturnValue(false);
+
+    insertStandard({
+      category: "practice",
+      scope: "all",
+      rule: "Must have error handling",
+      semantic_query: "try catch error handling",
+      expect: "must_match",
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(true); // Cannot validate must_match without grepai
+
+    availSpy.mockRestore();
+  });
+
+  // ─────────────────────────────────────────────────────────
+  // CAP & MIXED
+  // ─────────────────────────────────────────────────────────
+
+  it("caps semantic standards at MAX_SEMANTIC_QUERIES (10)", () => {
+    const file = createTmpFile("test.ts", "export const x = 1;");
+
+    let callCount = 0;
+    const availSpy = spyOn(patterns, "isGrepaiAvailable").mockReturnValue(true);
+    const searchSpy = spyOn(patterns, "searchWithGrepai").mockImplementation(() => {
+      callCount++;
+      return [];
+    });
+
+    // Insert 15 semantic standards
+    for (let i = 0; i < 15; i++) {
+      insertStandard({
+        category: "code",
+        scope: "all",
+        rule: `Rule ${i}`,
+        semantic_query: `query ${i}`,
+        expect: "no_match",
+      });
+    }
+
+    validateAgainstStandards([file], "all");
+    expect(callCount).toBe(10); // Capped at 10
+
+    availSpy.mockRestore();
+    searchSpy.mockRestore();
+  });
+
+  it("validates both semantic and textual standards in same run", () => {
+    const file = createTmpFile("mixed.ts", 'eval("code"); export const x = 1;');
+    const resolvedPath = join(TMP_DIR, "mixed.ts").replace(/\\/g, "/");
+
+    const availSpy = spyOn(patterns, "isGrepaiAvailable").mockReturnValue(true);
+    const searchSpy = spyOn(patterns, "searchWithGrepai").mockReturnValue([
+      { path: resolvedPath, score: 0.9 },
+    ]);
+
+    // Semantic standard
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "No .then() chains",
+      semantic_query: "promise .then() chain",
+      expect: "no_match",
+    });
+
+    // Textual standard (no semantic_query)
+    insertStandard({
+      category: "code",
+      scope: "all",
+      rule: "No eval",
+      anti_examples: JSON.stringify(["eval("]),
+    });
+
+    const result = validateAgainstStandards([file], "all");
+    expect(result.passed).toBe(false);
+    expect(result.violations).toHaveLength(2); // One semantic + one textual
+
+    availSpy.mockRestore();
+    searchSpy.mockRestore();
+  });
+});