@hasna/evals 0.1.5 → 0.1.7

package/dist/cli/commands/ci.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/ci.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,SAAS,IAAI,OAAO,CA0DnC"}
+{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/ci.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,SAAS,IAAI,OAAO,CAiEnC"}

package/dist/cli/commands/doctor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,wBAAgB,aAAa,IAAI,OAAO,CAoDvC"}
+{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,wBAAgB,aAAa,IAAI,OAAO,CAkEvC"}

package/dist/cli/commands/mcp.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/mcp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,UAAU,IAAI,OAAO,CASpC"}
+{"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/mcp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,UAAU,IAAI,OAAO,CAgCpC"}

package/dist/cli/commands/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,UAAU,IAAI,OAAO,CAoDpC"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,UAAU,IAAI,OAAO,CAyDpC"}

package/dist/cli/index.js

@@ -30145,7 +30145,7 @@ function parseAdapterConfig(opts) {
 
 // src/cli/commands/run.ts
 function runCommand() {
-  return new Command("run").description("Run an eval dataset against your app").argument("<dataset>", "Path to JSONL/JSON dataset or glob pattern").option("--adapter <type>", "Adapter type: http|anthropic|openai|mcp|function|cli", "http").option("--url <url>", "App URL (for http adapter)").option("--model <model>", "Model name (for anthropic/openai adapter)").option("--system <prompt>", "System prompt (for anthropic/openai adapter)").option("--concurrency <n>", "Parallel execution limit", "5").option("--repeat <n>", "Run each case N times (Pass^k metric)", "1").option("--tags <tags>", "Comma-separated tags to filter cases").option("--no-judge", "Skip LLM judge, run assertions only").option("--output <format>", "Output format: terminal|json|markdown", "terminal").option("--save", "Save run to database").option("--json", "Alias for --output json").action(async (dataset, opts) => {
+  return new Command("run").description("Run an eval dataset against your app").argument("<dataset>", "Path to JSONL/JSON dataset or glob pattern").option("--adapter <type>", "Adapter type: http|anthropic|openai|mcp|function|cli", "http").option("--url <url>", "App URL (for http adapter)").option("--model <model>", "Model name (for anthropic/openai adapter)").option("--system <prompt>", "System prompt (for anthropic/openai adapter)").option("--module <path>", "Module path (for function adapter)").option("--export <name>", "Export name (for function adapter, default: default)").option("--command <cmd>", "Shell command (for cli adapter, use {{input}} placeholder)").option("--mcp-command <cmd>", "MCP server command (for mcp adapter)").option("--tool <name>", "MCP tool name (for mcp adapter)").option("--concurrency <n>", "Parallel execution limit", "5").option("--repeat <n>", "Run each case N times (Pass^k metric)", "1").option("--tags <tags>", "Comma-separated tags to filter cases").option("--no-judge", "Skip LLM judge, run assertions only").option("--output <format>", "Output format: terminal|json|markdown", "terminal").option("--save", "Save run to database").option("--json", "Alias for --output json").action(async (dataset, opts) => {
     const { cases, warnings } = await loadDataset(dataset, {
       tags: opts["tags"] ? opts["tags"].split(",") : undefined
     });
@@ -30164,7 +30164,7 @@ function runCommand() {
       concurrency: parseInt(opts["concurrency"] ?? "5"),
       repeat: parseInt(opts["repeat"] ?? "1"),
       tags: opts["tags"] ? opts["tags"].split(",") : undefined,
-      skipJudge: opts["noJudge"] === "true"
+      skipJudge: opts["judge"] === false || opts["noJudge"] === true
     });
     if (opts["save"])
       saveRun(run);
@@ -30185,10 +30185,10 @@ init_loader();
 init_store();
 function ciCommand() {
   const cmd = new Command("ci").description("Run evals in CI mode \u2014 exit non-zero on regression");
-  cmd.command("run <dataset>").description("Run and compare to baseline").option("--adapter <type>", "Adapter type", "http").option("--url <url>", "App URL").option("--model <model>", "Model name").option("--baseline <name>", "Baseline name to compare against", "main").option("--fail-if-regression <pct>", "Fail if score drops by more than N%", "0").option("--output <format>", "Output format: terminal|markdown", "terminal").option("--json", "Output JSON").action(async (dataset, opts) => {
+  cmd.command("run <dataset>").description("Run and compare to baseline").option("--adapter <type>", "Adapter type: http|anthropic|openai|mcp|function|cli", "http").option("--url <url>", "App URL (for http adapter)").option("--model <model>", "Model name (for anthropic/openai adapter)").option("--system <prompt>", "System prompt (for anthropic/openai adapter)").option("--module <path>", "Module path (for function adapter)").option("--export <name>", "Export name (for function adapter)").option("--command <cmd>", "Shell command (for cli adapter)").option("--mcp-command <cmd>", "MCP server command (for mcp adapter)").option("--tool <name>", "MCP tool name (for mcp adapter)").option("--no-judge", "Skip LLM judge, assertions only").option("--baseline <name>", "Baseline name to compare against", "main").option("--fail-if-regression <pct>", "Fail if score drops by more than N%", "0").option("--output <format>", "Output format: terminal|markdown", "terminal").option("--json", "Output JSON").action(async (dataset, opts) => {
     const { cases } = await loadDataset(dataset);
     const adapter = parseAdapterConfig(opts);
-    const run = await runEvals(cases, { dataset, adapter });
+    const run = await runEvals(cases, { dataset, adapter, skipJudge: opts["judge"] === false || opts["noJudge"] === true });
     saveRun(run);
     const baselineName = opts["baseline"] ?? "main";
     const baseline = getBaseline(baselineName);
@@ -30480,11 +30480,23 @@ function doctorCommand() {
     }
     try {
       const { loadDataset: loadDataset2 } = await Promise.resolve().then(() => (init_loader(), exports_loader));
-      const examplesPath = new URL("../../../datasets/examples/smoke.jsonl", import.meta.url).pathname;
-      const { cases } = await loadDataset2(examplesPath);
-      checks.push({ name: `Example dataset (${cases.length} cases)`, ok: cases.length > 0, hint: "datasets/examples/smoke.jsonl missing" });
+      const { existsSync } = await import("fs");
+      const { join: join2 } = await import("path");
+      const { homedir: homedir2 } = await import("os");
+      const candidates = [
+        new URL("../../../datasets/examples/smoke.jsonl", import.meta.url).pathname,
+        join2(import.meta.dir, "../../../datasets/examples/smoke.jsonl"),
+        join2(import.meta.dir, "../../datasets/examples/smoke.jsonl"),
+        join2(import.meta.dir, "../datasets/examples/smoke.jsonl"),
+        join2(homedir2(), ".hasna", "evals", "examples", "smoke.jsonl")
+      ];
+      const found = candidates.find((p) => existsSync(p));
+      if (!found)
+        throw new Error("not found");
+      const { cases } = await loadDataset2(found);
+      checks.push({ name: `Example dataset (${cases.length} cases)`, ok: cases.length > 0 });
     } catch {
-      checks.push({ name: "Example dataset", ok: false, hint: "Run from the open-evals project directory" });
+      checks.push({ name: "Example dataset (optional)", ok: false, hint: "datasets/examples/smoke.jsonl not found \u2014 install @hasna/evals globally to include examples" });
     }
     console.log(`
 \x1B[1mevals doctor\x1B[0m
@@ -30494,7 +30506,7 @@ function doctorCommand() {
       console.log(`  ${icon} ${c.name}${!c.ok && c.hint ? `
       hint: ${c.hint}` : ""}`);
     }
-    const allOk = checks.every((c) => c.ok || c.name.includes("optional"));
+    const allOk = checks.every((c) => c.ok || c.name.toLowerCase().includes("optional"));
     console.log(allOk ? `
 \x1B[32m  All checks passed.\x1B[0m
 ` : `
@@ -30509,25 +30521,59 @@ import { readFileSync, writeFileSync as writeFileSync2, existsSync } from "fs";
 import { homedir as homedir2 } from "os";
 import { join as join2 } from "path";
 function mcpCommand() {
-  return new Command("mcp").description("MCP server management").addCommand(new Command("--claude").description("Register evals-mcp with Claude Code").action(registerClaude)).addCommand(new Command("start").description("Start MCP server (stdio)").action(() => {
+  const cmd = new Command("mcp").description("MCP server management");
+  cmd.addCommand(new Command("register").description("Register evals-mcp with an agent (Claude Code, Codex, Gemini)").option("--claude", "Register with Claude Code (~/.claude/mcp.json)").option("--codex", "Register with Codex (~/.codex/config.json)").option("--gemini", "Register with Gemini (~/.gemini/settings.json)").option("--all", "Register with all agents").action((opts) => {
+    if (opts.claude || opts.all)
+      registerClaude();
+    if (opts.codex || opts.all)
+      registerCodex();
+    if (opts.gemini || opts.all)
+      registerGemini();
+    if (!opts.claude && !opts.codex && !opts.gemini && !opts.all) {
+      registerClaude();
+    }
+  }));
+  cmd.addCommand(new Command("start").description("Start MCP server (stdio)").action(() => {
     const { spawnSync } = __require("child_process");
     spawnSync(process.execPath, [join2(import.meta.dir, "../../mcp/index.js")], { stdio: "inherit" });
   }));
+  return cmd;
 }
+var ENTRY = { command: "/home/hasna/.bun/bin/evals-mcp", args: [] };
 function registerClaude() {
-  const settingsPath = join2(homedir2(), ".claude", "settings.json");
-  let settings = {};
-  if (existsSync(settingsPath)) {
-    settings = JSON.parse(readFileSync(settingsPath, "utf8"));
-  }
-  const mcpServers = settings["mcpServers"] ?? {};
-  mcpServers["evals"] = { command: "evals-mcp", args: [] };
-  settings["mcpServers"] = mcpServers;
-  writeFileSync2(settingsPath, JSON.stringify(settings, null, 2) + `
+  const mcpPath = join2(homedir2(), ".claude", "mcp.json");
+  let config = {};
+  if (existsSync(mcpPath)) {
+    config = JSON.parse(readFileSync(mcpPath, "utf8"));
+  }
+  config.mcpServers = { ...config.mcpServers ?? {}, evals: ENTRY };
+  writeFileSync2(mcpPath, JSON.stringify(config, null, 2) + `
 `);
-  console.log("\x1B[32m\u2713 Registered evals-mcp in ~/.claude/settings.json\x1B[0m");
+  console.log("\x1B[32m\u2713 Registered evals-mcp in ~/.claude/mcp.json\x1B[0m");
   console.log("  Restart Claude Code to load the new MCP server.");
 }
+function registerCodex() {
+  const cfgPath = join2(homedir2(), ".codex", "config.json");
+  let config = {};
+  if (existsSync(cfgPath)) {
+    config = JSON.parse(readFileSync(cfgPath, "utf8"));
+  }
+  config.mcpServers = { ...config.mcpServers ?? {}, evals: { type: "stdio", ...ENTRY, env: {} } };
+  writeFileSync2(cfgPath, JSON.stringify(config, null, 2) + `
+`);
+  console.log("\x1B[32m\u2713 Registered evals-mcp in ~/.codex/config.json\x1B[0m");
+}
+function registerGemini() {
+  const cfgPath = join2(homedir2(), ".gemini", "settings.json");
+  let config = {};
+  if (existsSync(cfgPath)) {
+    config = JSON.parse(readFileSync(cfgPath, "utf8"));
+  }
+  config.mcpServers = { ...config.mcpServers ?? {}, evals: ENTRY };
+  writeFileSync2(cfgPath, JSON.stringify(config, null, 2) + `
+`);
+  console.log("\x1B[32m\u2713 Registered evals-mcp in ~/.gemini/settings.json\x1B[0m");
+}
 
 // src/cli/commands/capture.ts
 import { createServer } from "http";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hasna/evals",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "Open source AI evaluation framework — LLM-as-judge + assertion-based evals for any AI app. CLI + MCP server.",
   "type": "module",
   "main": "dist/index.js",