@radaros/eval 0.3.20 → 0.3.22

package/dist/index.cjs

@@ -0,0 +1,371 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  ConsoleReporter: () => ConsoleReporter,
+  EvalSuite: () => EvalSuite,
+  JsonReporter: () => JsonReporter,
+  contains: () => contains,
+  custom: () => custom,
+  jsonMatch: () => jsonMatch,
+  llmJudge: () => llmJudge,
+  regexMatch: () => regexMatch,
+  semanticSimilarity: () => semanticSimilarity
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/reporters/console.ts
+var ConsoleReporter = class {
+  report(result) {
+    const line = "\u2500".repeat(70);
+    console.log(`
+${line}`);
+    console.log(`  Eval Suite: ${result.name}`);
+    console.log(`${line}`);
+    for (const r of result.results) {
+      const status = r.pass ? "PASS" : "FAIL";
+      const icon = r.pass ? "+" : "-";
+      console.log(`
+  [${icon}] ${status} | ${r.caseName} (${r.durationMs}ms)`);
+      console.log(`      Input: ${r.input.slice(0, 80)}${r.input.length > 80 ? "..." : ""}`);
+      for (const [scorerName, score] of Object.entries(r.scores)) {
+        const mark = score.pass ? "+" : "-";
+        console.log(
+          `      [${mark}] ${scorerName}: ${score.score.toFixed(3)}${score.reason ? ` \u2014 ${score.reason}` : ""}`
+        );
+      }
+    }
+    console.log(`
+${line}`);
+    console.log(
+      `  Results: ${result.passed}/${result.total} passed | Avg score: ${result.averageScore.toFixed(3)} | ${result.durationMs}ms`
+    );
+    console.log(`${line}
+`);
+  }
+};
+
+// src/reporters/json.ts
+var import_promises = require("fs/promises");
+var JsonReporter = class {
+  outputPath;
+  constructor(outputPath) {
+    this.outputPath = outputPath ?? `eval-results-${Date.now()}.json`;
+  }
+  async report(result) {
+    const serializable = {
+      ...result,
+      results: result.results.map((r) => ({
+        caseName: r.caseName,
+        input: r.input,
+        outputText: r.output?.text,
+        scores: r.scores,
+        durationMs: r.durationMs,
+        pass: r.pass,
+        usage: r.output?.usage
+      }))
+    };
+    try {
+      await (0, import_promises.writeFile)(this.outputPath, JSON.stringify(serializable, null, 2));
+      console.log(`Eval results written to ${this.outputPath}`);
+    } catch (err) {
+      console.warn(`Failed to write eval report to ${this.outputPath}:`, err instanceof Error ? err.message : err);
+    }
+  }
+};
+
+// src/scorers/contains.ts
+function contains(expected, options) {
+  const caseSensitive = options?.caseSensitive ?? false;
+  return {
+    name: "contains",
+    async score(_input, output, _expected) {
+      const text = caseSensitive ? output.text ?? "" : (output.text ?? "").toLowerCase();
+      const target = caseSensitive ? expected : expected.toLowerCase();
+      const pass = text.includes(target);
+      return {
+        score: pass ? 1 : 0,
+        pass,
+        reason: pass ? void 0 : `Output does not contain "${expected}"`
+      };
+    }
+  };
+}
+
+// src/scorers/custom.ts
+function custom(name, fn) {
+  return {
+    name,
+    async score(input, output, expected) {
+      return fn(input, output, expected);
+    }
+  };
+}
+
+// src/scorers/json-match.ts
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (typeof a !== typeof b) return false;
+  if (a === null || b === null) return false;
+  if (typeof a !== "object") return false;
+  if (Array.isArray(a) !== Array.isArray(b)) return false;
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    return a.every((v, i) => deepEqual(v, b[i]));
+  }
+  const aObj = a;
+  const bObj = b;
+  const aKeys = Object.keys(aObj);
+  const bKeys = Object.keys(bObj);
+  if (aKeys.length !== bKeys.length) return false;
+  return aKeys.every((k) => deepEqual(aObj[k], bObj[k]));
+}
+function jsonMatch(expectedFields) {
+  return {
+    name: "json-match",
+    async score(_input, output) {
+      if (!output.structured) {
+        return { score: 0, pass: false, reason: "No structured output" };
+      }
+      const obj = output.structured;
+      const totalFields = Object.keys(expectedFields).length;
+      let matched = 0;
+      const mismatches = [];
+      for (const [key, expected] of Object.entries(expectedFields)) {
+        if (deepEqual(obj[key], expected)) {
+          matched++;
+        } else {
+          mismatches.push(`${key}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(obj[key])}`);
+        }
+      }
+      const score = totalFields > 0 ? matched / totalFields : 1;
+      const pass = score >= 1;
+      return {
+        score,
+        pass,
+        reason: pass ? void 0 : `Mismatches: ${mismatches.join("; ")}`
+      };
+    }
+  };
+}
+
+// src/scorers/llm-judge.ts
+function llmJudge(config) {
+  const criteria = config.criteria ?? ["relevance", "helpfulness"];
+  return {
+    name: "llm-judge",
+    async score(input, output, expected) {
+      const criteriaList = criteria.map((c) => `- ${c}`).join("\n");
+      const prompt = config.customPrompt ?? `You are an expert evaluator. Rate the following AI response on a scale of 0.0 to 1.0.
+
+Criteria:
+${criteriaList}
+
+User input: ${input}
+${expected ? `Expected output: ${expected}
+` : ""}
+AI response: ${output.text}
+
+Respond with ONLY a JSON object: {"score": <0.0-1.0>, "reason": "<brief explanation>"}`;
+      try {
+        const response = await config.model.generate([{ role: "user", content: prompt }]);
+        const text = typeof response.message.content === "string" ? response.message.content : "";
+        const jsonMatch2 = text.match(/\{[\s\S]*\}/);
+        if (!jsonMatch2) {
+          return { score: 0, pass: false, reason: "Judge failed to return valid JSON" };
+        }
+        const parsed = JSON.parse(jsonMatch2[0]);
+        const score = Math.max(0, Math.min(1, Number(parsed.score) || 0));
+        return {
+          score,
+          pass: score >= (config.threshold ?? 0.7),
+          reason: parsed.reason ?? void 0
+        };
+      } catch (err) {
+        return {
+          score: 0,
+          pass: false,
+          reason: `Judge error: ${err.message}`
+        };
+      }
+    }
+  };
+}
+
+// src/scorers/regex.ts
+function regexMatch(pattern) {
+  const regex = typeof pattern === "string" ? new RegExp(pattern) : pattern;
+  return {
+    name: "regex",
+    async score(_input, output) {
+      regex.lastIndex = 0;
+      const pass = regex.test(output.text);
+      return {
+        score: pass ? 1 : 0,
+        pass,
+        reason: pass ? void 0 : `Output does not match pattern ${regex}`
+      };
+    }
+  };
+}
+
+// src/scorers/similarity.ts
+function semanticSimilarity(config) {
+  const threshold = config.threshold ?? 0.8;
+  return {
+    name: "similarity",
+    async score(_input, output) {
+      const vecs = await config.embedding.embedBatch([output.text, config.expected]);
+      if (!vecs || vecs.length < 2 || !vecs[0] || !vecs[1]) {
+        return { score: 0, pass: false, reason: "Embedding failed: insufficient vectors returned" };
+      }
+      const [outputVec, expectedVec] = vecs;
+      if (outputVec.length !== expectedVec.length) {
+        return {
+          score: 0,
+          pass: false,
+          reason: `Vector dimension mismatch: ${outputVec.length} vs ${expectedVec.length}`
+        };
+      }
+      const sim = cosineSimilarity(outputVec, expectedVec);
+      const pass = sim >= threshold;
+      return {
+        score: sim,
+        pass,
+        reason: pass ? void 0 : `Similarity ${sim.toFixed(3)} < threshold ${threshold}`
+      };
+    }
+  };
+}
+function cosineSimilarity(a, b) {
+  let dot = 0;
+  let magA = 0;
+  let magB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(magA) * Math.sqrt(magB);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+// src/suite.ts
+var import_core = require("@radaros/core");
+var EvalSuite = class {
+  config;
+  constructor(config) {
+    this.config = config;
+  }
+  async run(reporters) {
+    const startTime = Date.now();
+    const threshold = this.config.threshold ?? 0.7;
+    const concurrency = this.config.concurrency ?? 1;
+    const results = [];
+    const chunks = this.chunk(this.config.cases, concurrency);
+    for (const batch of chunks) {
+      const batchResults = await Promise.allSettled(batch.map((c) => this.runCase(c, threshold)));
+      for (const result of batchResults) {
+        if (result.status === "fulfilled") {
+          results.push(result.value);
+        } else {
+          results.push({
+            input: "unknown",
+            scores: {},
+            pass: false,
+            durationMs: 0,
+            error: result.reason instanceof Error ? result.reason.message : String(result.reason)
+          });
+        }
+      }
+    }
+    const passed = results.filter((r) => r.pass).length;
+    const failed = results.length - passed;
+    const allScores = results.flatMap((r) => Object.values(r.scores).map((s) => s.score));
+    const averageScore = allScores.length > 0 ? allScores.reduce((a, b) => a + b, 0) / allScores.length : 0;
+    const suiteResult = {
+      name: this.config.name,
+      results,
+      passed,
+      failed,
+      total: results.length,
+      averageScore,
+      durationMs: Date.now() - startTime
+    };
+    if (reporters) {
+      for (const reporter of reporters) {
+        await reporter.report(suiteResult);
+      }
+    }
+    return suiteResult;
+  }
+  async runCase(evalCase, threshold) {
+    const startTime = Date.now();
+    const inputText = typeof evalCase.input === "string" ? evalCase.input : (0, import_core.getTextContent)(evalCase.input);
+    const timeoutMs = this.config.timeoutMs ?? 3e4;
+    const output = await Promise.race([
+      this.config.agent.run(evalCase.input, evalCase.runOpts),
+      new Promise(
+        (_, reject) => setTimeout(() => reject(new Error(`Eval case timed out after ${timeoutMs}ms`)), timeoutMs)
+      )
+    ]);
+    const scores = {};
+    for (const scorer of this.config.scorers) {
+      try {
+        scores[scorer.name] = await scorer.score(inputText, output, evalCase.expected);
+      } catch (err) {
+        scores[scorer.name] = {
+          score: 0,
+          pass: false,
+          reason: `Scorer error: ${err instanceof Error ? err.message : String(err)}`
+        };
+      }
+    }
+    const allPass = Object.values(scores).every((s) => s.score >= threshold);
+    return {
+      caseName: evalCase.name,
+      input: inputText,
+      output,
+      scores,
+      durationMs: Date.now() - startTime,
+      pass: allPass
+    };
+  }
+  chunk(arr, size) {
+    const chunks = [];
+    for (let i = 0; i < arr.length; i += size) {
+      chunks.push(arr.slice(i, i + size));
+    }
+    return chunks;
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ConsoleReporter,
+  EvalSuite,
+  JsonReporter,
+  contains,
+  custom,
+  jsonMatch,
+  llmJudge,
+  regexMatch,
+  semanticSimilarity
+});

package/dist/index.d.cts

@@ -0,0 +1,92 @@
+import { RunOutput, MessageContent, RunOpts, Agent, ModelProvider, EmbeddingProvider } from '@radaros/core';
+
+interface EvalCase {
+    name: string;
+    input: string | MessageContent;
+    expected?: string;
+    metadata?: Record<string, unknown>;
+    runOpts?: RunOpts;
+}
+interface ScorerResult {
+    score: number;
+    pass: boolean;
+    reason?: string;
+}
+interface Scorer {
+    name: string;
+    score(input: string, output: RunOutput, expected?: string): Promise<ScorerResult>;
+}
+interface EvalResult {
+    caseName?: string;
+    input: string;
+    output?: RunOutput;
+    scores: Record<string, ScorerResult>;
+    durationMs: number;
+    pass: boolean;
+    error?: string;
+}
+interface EvalSuiteResult {
+    name: string;
+    results: EvalResult[];
+    passed: number;
+    failed: number;
+    total: number;
+    averageScore: number;
+    durationMs: number;
+}
+interface EvalSuiteConfig {
+    name: string;
+    agent: Agent;
+    cases: EvalCase[];
+    scorers: Scorer[];
+    threshold?: number;
+    concurrency?: number;
+    timeoutMs?: number;
+}
+interface Reporter {
+    report(result: EvalSuiteResult): void | Promise<void>;
+}
+
+declare class ConsoleReporter implements Reporter {
+    report(result: EvalSuiteResult): void;
+}
+
+declare class JsonReporter implements Reporter {
+    private outputPath;
+    constructor(outputPath?: string);
+    report(result: EvalSuiteResult): Promise<void>;
+}
+
+declare function contains(expected: string, options?: {
+    caseSensitive?: boolean;
+}): Scorer;
+
+declare function custom(name: string, fn: (input: string, output: RunOutput, expected?: string) => Promise<ScorerResult> | ScorerResult): Scorer;
+
+declare function jsonMatch(expectedFields: Record<string, unknown>): Scorer;
+
+type JudgeCriteria = "faithfulness" | "relevance" | "helpfulness" | "safety" | "conciseness";
+declare function llmJudge(config: {
+    model: ModelProvider;
+    criteria?: JudgeCriteria[];
+    customPrompt?: string;
+    threshold?: number;
+}): Scorer;
+
+declare function regexMatch(pattern: string | RegExp): Scorer;
+
+declare function semanticSimilarity(config: {
+    expected: string;
+    embedding: EmbeddingProvider;
+    threshold?: number;
+}): Scorer;
+
+declare class EvalSuite {
+    private config;
+    constructor(config: EvalSuiteConfig);
+    run(reporters?: Reporter[]): Promise<EvalSuiteResult>;
+    private runCase;
+    private chunk;
+}
+
+export { ConsoleReporter, type EvalCase, type EvalResult, EvalSuite, type EvalSuiteConfig, type EvalSuiteResult, JsonReporter, type JudgeCriteria, type Reporter, type Scorer, type ScorerResult, contains, custom, jsonMatch, llmJudge, regexMatch, semanticSimilarity };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@radaros/eval",
-  "version": "0.3.20",
+  "version": "0.3.22",
   "description": "Evaluation framework for testing and scoring RadarOS agents",
   "license": "MIT",
   "repository": {
@@ -17,24 +17,27 @@
     "benchmarks"
   ],
   "type": "module",
-  "main": "./dist/index.js",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "exports": {
     ".": {
+      "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
-      "types": "./dist/index.d.ts"
+      "require": "./dist/index.cjs",
+      "default": "./dist/index.js"
     }
   },
   "files": [
     "dist"
   ],
   "scripts": {
-    "build": "tsup src/index.ts --format esm --dts --clean",
-    "dev": "tsup src/index.ts --format esm --dts --watch",
+    "build": "tsup src/index.ts --format esm,cjs --dts --clean",
+    "dev": "tsup src/index.ts --format esm,cjs --dts --watch",
     "prepublishOnly": "npm run build"
   },
   "peerDependencies": {
-    "@radaros/core": "^0.3.20"
+    "@radaros/core": "^0.3.22"
   },
   "devDependencies": {
     "@types/node": "^25.3.1",