cue-ai 0.5.0 → 0.7.0

package/src/commands/eval-behavior.test.ts

@@ -0,0 +1,56 @@
+/**
+ * Tests for `cue eval-behavior` — purely structural, no LLM or live commands.
+ * Uses the real profile tree because the loader is happy to find existing
+ * scenarios under resources/evals/.
+ */
+
+import { describe, expect, test, beforeEach } from "bun:test";
+import { run as evalRun } from "./eval-behavior";
+
+beforeEach(() => {
+  // Some sibling tests redirect CUE_PROFILES_DIR — restore so loadProfile finds real profiles.
+  delete process.env.CUE_PROFILES_DIR;
+  delete process.env.SOUL_PROFILES_DIR;
+});
+
+async function capture<T>(fn: () => Promise<T>): Promise<{ stdout: string; value: T }> {
+  const orig = process.stdout.write.bind(process.stdout);
+  let buf = "";
+  (process.stdout as any).write = (c: string | Uint8Array) => { buf += String(c); return true; };
+  try { const value = await fn(); return { stdout: buf, value }; }
+  finally { (process.stdout as any).write = orig; }
+}
+
+describe("cue eval-behavior", () => {
+  test("core passes both seeded scenarios", async () => {
+    const { stdout, value } = await capture(() => evalRun(["core", "--json"]));
+    expect(value).toBe(0);
+    const reports = JSON.parse(stdout) as Array<{ profile: string; results: Array<{ scenario: string; passed: boolean; score: number; max: number }> }>;
+    expect(reports).toHaveLength(1);
+    expect(reports[0]!.profile).toBe("core");
+    expect(reports[0]!.results.length).toBeGreaterThan(0);
+    for (const r of reports[0]!.results) {
+      expect(r.passed).toBe(true);
+      expect(r.score).toBeGreaterThan(0);
+    }
+  });
+
+  test("explicit profile arg evaluates that profile and reports its inherited scenarios", async () => {
+    // Every profile inheriting core now picks up the seeded evals.
+    const { stdout, value } = await capture(() => evalRun(["frontend", "--json"]));
+    expect(value === 0 || value === 1).toBe(true);
+    const reports = JSON.parse(stdout) as Array<{ profile: string; results: Array<{ scenario: string }> }>;
+    expect(reports[0]!.profile).toBe("frontend");
+    // frontend inherits core, so the core-declared scenarios show up here too.
+    const scenarioNames = reports[0]!.results.map((r) => r.scenario);
+    expect(scenarioNames).toContain("feature-shipping");
+  });
+
+  test("--all runs across every profile + JSON output is well-formed", async () => {
+    const { stdout, value } = await capture(() => evalRun(["--all", "--json"]));
+    expect(value === 0 || value === 1).toBe(true);  // 1 if any profile fails
+    const reports = JSON.parse(stdout);
+    expect(Array.isArray(reports)).toBe(true);
+    expect(reports.length).toBeGreaterThan(5);
+  });
+});

package/src/commands/eval-behavior.ts

@@ -0,0 +1,189 @@
+/**
+ * `cue eval-behavior [profile]` — structural eval harness.
+ *
+ * Reads `resources/evals/<scenario>.md` files referenced from each profile's
+ * `evals:` field, parses each scenario's required + recommended capabilities,
+ * and scores the profile by whether it actually has those skills/commands/
+ * playbooks/gates loaded.
+ *
+ * This is a STRUCTURAL eval — it doesn't run an LLM. It answers "is this
+ * profile equipped to handle scenario X?", not "did it do well at X?". That's
+ * Phase 5 territory and needs LLM-in-the-loop.
+ *
+ * Output: per-scenario pass/fail + a single summary score per profile.
+ */
+
+import { readFileSync, existsSync } from "node:fs";
+import { join, resolve, dirname, isAbsolute } from "node:path";
+import { fileURLToPath } from "node:url";
+import { homedir } from "node:os";
+
+import { loadProfile, listProfiles } from "../lib/profile-loader";
+import { resolveProfileForCwd } from "../lib/cwd-resolver";
+import type { ResolvedProfile } from "../../profiles/_types";
+
+const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..");
+const EVALS_ROOT = join(REPO_ROOT, "resources", "evals");
+
+const bold = (s: string) => `\x1b[1m${s}\x1b[0m`;
+const green = (s: string) => `\x1b[32m${s}\x1b[0m`;
+const red = (s: string) => `\x1b[31m${s}\x1b[0m`;
+const yellow = (s: string) => `\x1b[33m${s}\x1b[0m`;
+const dim = (s: string) => `\x1b[2m${s}\x1b[0m`;
+
+interface EvalScenario {
+  name: string;
+  description: string;
+  requiredSkills: string[];
+  oneOfCommands: string[];        // any one satisfies
+  recommendedPlaybooks: string[];
+  recommendedGates: string[];
+  triggerPhrases: string[];
+}
+
+function parseScenario(path: string): EvalScenario | null {
+  let raw: string;
+  try { raw = readFileSync(path, "utf8"); } catch { return null; }
+  const fm = raw.match(/^---\n([\s\S]*?)\n---/);
+  const name = fm?.[1]?.match(/^name:\s*(.+)$/m)?.[1]?.trim() ?? path;
+  const description = fm?.[1]?.match(/^description:\s*(.+)$/m)?.[1]?.trim() ?? "";
+
+  const section = (heading: string): string[] => {
+    const re = new RegExp(`^##\\s+${heading}[^\\n]*\\n([\\s\\S]*?)(?=\\n##\\s+|$)`, "m");
+    const m = raw.match(re);
+    if (!m) return [];
+    return m[1]!.split("\n")
+      .filter((l) => l.startsWith("- "))
+      .map((l) => l.slice(2).split("#")[0]!.trim())
+      .filter(Boolean);
+  };
+
+  return {
+    name,
+    description,
+    requiredSkills: section("Skills"),
+    oneOfCommands: section("Commands"),
+    recommendedPlaybooks: section("Playbooks"),
+    recommendedGates: section("Quality gates"),
+    triggerPhrases: section("Trigger phrases"),
+  };
+}
+
+interface ScenarioResult {
+  scenario: string;
+  score: number;
+  max: number;
+  passed: boolean;
+  missing: string[];      // human-readable list of what's lacking
+}
+
+function scoreScenario(profile: ResolvedProfile, scenario: EvalScenario): ScenarioResult {
+  const skillSlugs = new Set(profile.skills.local.map((s) => s.id));
+  const commandSet = new Set((profile.commands ?? []).map((c) => c.replace(/\.md$/, "")));
+  const playbookSet = new Set(((profile as any).playbooks ?? []).map((p: string) => p.replace(/\.md$/, "")));
+  const gateSet = new Set(((profile as any).qualityGates ?? []));
+
+  let score = 0;
+  let max = 0;
+  const missing: string[] = [];
+
+  for (const req of scenario.requiredSkills) {
+    max += 1;
+    if (skillSlugs.has(req)) score += 1;
+    else missing.push(`skill: ${req}`);
+  }
+
+  if (scenario.oneOfCommands.length > 0) {
+    max += 1;
+    if (scenario.oneOfCommands.some((c) => commandSet.has(c))) score += 1;
+    else missing.push(`commands: need any of [${scenario.oneOfCommands.join(", ")}]`);
+  }
+
+  for (const pb of scenario.recommendedPlaybooks) {
+    max += 1;
+    if (playbookSet.has(pb)) score += 1;
+    else missing.push(`playbook: ${pb}`);
+  }
+
+  for (const gate of scenario.recommendedGates) {
+    max += 1;
+    if (gateSet.has(gate)) score += 1;
+    else missing.push(`quality-gate: ${gate}`);
+  }
+
+  // Pass at >= 50%. If max is 0 (scenario declared nothing), trivially passes.
+  const passed = max === 0 || score >= Math.ceil(max / 2);
+  return { scenario: scenario.name, score, max, passed, missing };
+}
+
+async function evalProfile(name: string): Promise<{ profile: string; results: ScenarioResult[] }> {
+  const profile = await loadProfile(name);
+  const evalRefs = ((profile as any).evals ?? []) as string[];
+  const results: ScenarioResult[] = [];
+  for (const ref of evalRefs) {
+    const path = isAbsolute(ref) ? ref : join(EVALS_ROOT, ref.endsWith(".md") ? ref : `${ref}.md`);
+    const scenario = parseScenario(path);
+    if (!scenario) {
+      results.push({ scenario: ref, score: 0, max: 0, passed: false, missing: [`scenario file not found: ${path}`] });
+      continue;
+    }
+    results.push(scoreScenario(profile, scenario));
+  }
+  return { profile: name, results };
+}
+
+export async function run(args: string[]): Promise<number> {
+  const asJson = args.includes("--json");
+  const all = args.includes("--all");
+
+  let profiles: string[];
+  if (all) {
+    profiles = await listProfiles();
+  } else {
+    const explicit = args.find((a) => !a.startsWith("-"));
+    if (explicit) profiles = [explicit];
+    else {
+      try {
+        const r = await resolveProfileForCwd({ cwd: process.cwd(), homeDir: homedir(), configDir: join(homedir(), ".config", "cue") });
+        if (r.source === "none") {
+          process.stderr.write("Usage: cue eval-behavior [profile] | --all\n");
+          return 1;
+        }
+        profiles = [(r as any).profile];
+      } catch {
+        process.stderr.write("Usage: cue eval-behavior [profile] | --all\n");
+        return 1;
+      }
+    }
+  }
+
+  const reports = await Promise.all(profiles.map(evalProfile));
+
+  if (asJson) {
+    process.stdout.write(JSON.stringify(reports, null, 2) + "\n");
+    return 0;
+  }
+
+  let anyFail = 0;
+  for (const r of reports) {
+    if (r.results.length === 0) {
+      process.stdout.write(`\n  ${dim("·")} ${r.profile} ${dim("(no evals declared)")}\n`);
+      continue;
+    }
+    const passed = r.results.filter((s) => s.passed).length;
+    const total = r.results.length;
+    const tag = passed === total ? green(`PASS ${passed}/${total}`) : red(`FAIL ${passed}/${total}`);
+    process.stdout.write(`\n  ${bold(r.profile)}  ${tag}\n`);
+    for (const sc of r.results) {
+      const mark = sc.passed ? green("✓") : red("✗");
+      const pct = sc.max > 0 ? `(${sc.score}/${sc.max})` : "(no checks)";
+      process.stdout.write(`    ${mark} ${sc.scenario.padEnd(20)} ${dim(pct)}\n`);
+      if (!sc.passed && sc.missing.length > 0) {
+        for (const m of sc.missing.slice(0, 5)) process.stdout.write(`        ${yellow("→")} ${m}\n`);
+      }
+    }
+    if (passed < total) anyFail++;
+  }
+  process.stdout.write("\n");
+  return anyFail > 0 ? 1 : 0;
+}

package/src/commands/eval.test.ts

@@ -0,0 +1,102 @@
+/**
+ * Tests for `cue eval`.
+ *
+ * Runs the command's JSON mode against the real profiles tree (the repo's
+ * actual `profiles/`) and asserts on the structure + math. Avoids parsing
+ * ANSI-formatted text mode, which is unstable.
+ */
+
+import { describe, expect, test, beforeEach } from "bun:test";
+import { run as evalRun } from "./eval";
+
+// Some sibling test files set CUE_PROFILES_DIR to a temp dir and don't reset.
+// Force the repo's real profiles/ tree before each test so loadProfile finds core/ecc/etc.
+beforeEach(() => {
+  delete process.env.CUE_PROFILES_DIR;
+  delete process.env.SOUL_PROFILES_DIR;
+});
+
+interface JsonReport {
+  profile: string;
+  counts: { skills: number; rules: number; commands: number; hooks: number; mcps: number; plugins: number };
+  tokens: {
+    perMessage: number;
+    onDemand: number;
+    bySource: Record<string, { perMessage: number; onDemand: number }>;
+  };
+  fullPerMessage: number;
+  savingsPct: number;
+  costPerMessage: string;
+  sessions: number;
+  score: number;
+  grade: string;
+}
+
+async function captureStdout<T>(fn: () => Promise<T>): Promise<{ stdout: string; value: T }> {
+  const orig = process.stdout.write.bind(process.stdout);
+  let buf = "";
+  (process.stdout as any).write = (chunk: string | Uint8Array) => { buf += String(chunk); return true; };
+  try {
+    const value = await fn();
+    return { stdout: buf, value };
+  } finally {
+    (process.stdout as any).write = orig;
+  }
+}
+
+describe("cue eval", () => {
+  test("--json on a real profile returns the expected shape and positive totals", async () => {
+    const { stdout, value } = await captureStdout(() => evalRun(["core", "--json"]));
+    expect(value).toBe(0);
+    const report = JSON.parse(stdout) as JsonReport;
+    expect(report.profile).toBe("core");
+    expect(report.counts.skills).toBeGreaterThan(0);
+    expect(report.tokens.perMessage).toBeGreaterThan(0);
+    expect(report.tokens.perMessage).toBeLessThan(report.tokens.onDemand);
+    // sum of bySource.perMessage should equal tokens.perMessage
+    const sum = Object.values(report.tokens.bySource).reduce((s, b) => s + b.perMessage, 0);
+    expect(sum).toBe(report.tokens.perMessage);
+    expect(report.grade).toMatch(/^[A-F]$/);
+  });
+
+  test("--compare emits delta and is symmetric in absolute value", async () => {
+    const { stdout, value } = await captureStdout(() => evalRun(["--compare", "core", "ecc", "--json"]));
+    expect(value).toBe(0);
+    const cmp = JSON.parse(stdout) as { a: any; b: any; delta: { perMessage: number; score: number } };
+    expect(cmp.a.profile).toBe("core");
+    expect(cmp.b.profile).toBe("ecc");
+    expect(cmp.delta.perMessage).toBe(cmp.b.tokens.perMessage - cmp.a.tokens.perMessage);
+  });
+
+  test("--all returns an array sorted by perMessage ascending", async () => {
+    const { stdout, value } = await captureStdout(() => evalRun(["--all", "--json"]));
+    expect(value).toBe(0);
+    const rows = JSON.parse(stdout) as Array<{ name: string; perMessage: number; ok: boolean }>;
+    expect(rows.length).toBeGreaterThan(5);
+    const okRows = rows.filter((r) => r.ok);
+    for (let i = 1; i < okRows.length; i++) {
+      expect(okRows[i].perMessage).toBeGreaterThanOrEqual(okRows[i - 1].perMessage);
+    }
+  });
+
+  test("--compare with missing second arg returns usage error (exit 1)", async () => {
+    const orig = process.stderr.write.bind(process.stderr);
+    let err = "";
+    (process.stderr as any).write = (chunk: string | Uint8Array) => { err += String(chunk); return true; };
+    try {
+      const exit = await evalRun(["--compare", "core"]);
+      expect(exit).toBe(1);
+      expect(err).toContain("--compare");
+    } finally {
+      (process.stderr as any).write = orig;
+    }
+  });
+
+  test("on-demand bodies exceed per-message tokens — verifies the lazy/eager split", async () => {
+    const { stdout } = await captureStdout(() => evalRun(["ecc", "--json"]));
+    const report = JSON.parse(stdout) as JsonReport;
+    // ecc declares rules + commands; both should have measurable on-demand cost
+    expect(report.tokens.bySource.rules.onDemand).toBeGreaterThan(report.tokens.bySource.rules.perMessage);
+    expect(report.tokens.bySource.commands.onDemand).toBeGreaterThan(report.tokens.bySource.commands.perMessage);
+  });
+});