@aroha-sdk/evals 1.1.0

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@aroha-sdk/evals",
+  "version": "1.1.0",
+  "description": "Evaluation harness for Aroha agents — golden-dataset runner, LLM-as-judge, trajectory scoring",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "bin": {
+    "aroha-evals": "./dist/cli.js"
+  },
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types":  "./dist/index.d.ts"
+    }
+  },
+  "files": ["dist", "src"],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test":  "vitest run --passWithNoTests",
+    "dev":   "tsc -p tsconfig.json --watch"
+  },
+  "engines": { "node": ">=22" },
+  "license": "MIT",
+  "keywords": ["aroha", "agent", "evals", "llm-as-judge", "evaluation"],
+  "devDependencies": {
+    "typescript": "^5.4.5",
+    "vitest": "^1.6.0",
+    "@types/node": "^20.14.0"
+  }
+}

package/src/cli.ts

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+// Copyright (c) 2026 Aroha Labs
+// SPDX-License-Identifier: MIT
+/**
+ * aroha-evals CLI
+ *
+ * Usage:
+ *   npx @aroha-sdk/evals run --endpoint http://localhost:8000 --suite ./evals.json
+ *   npx @aroha-sdk/evals run --endpoint https://my-agent.fly.dev --bearer $TOKEN --judge-model gpt-4o
+ */
+
+import { readFileSync } from "node:fs";
+import { runEvals, postEvalsToReputation } from "./runner.js";
+import type { EvalSuite, RunEvalsOptions } from "./types.js";
+
+const args = process.argv.slice(2);
+const cmd  = args[0];
+
+function flag(name: string): string | undefined {
+  const idx = args.indexOf(`--${name}`);
+  return idx !== -1 ? args[idx + 1] : undefined;
+}
+
+if (cmd !== "run") {
+  console.error(`Usage: aroha-evals run --endpoint <url> --suite <path> [--bearer <token>]`);
+  console.error(`                       [--judge-url <url>] [--judge-key <key>] [--judge-model <model>]`);
+  console.error(`                       [--threshold <0-1>] [--concurrency <n>]`);
+  console.error(`                       [--did <didHash>] [--reputation-url <url>] [--reputation-key <key>]`);
+  process.exit(1);
+}
+
+const endpoint  = flag("endpoint");
+const suitePath = flag("suite");
+
+if (!endpoint || !suitePath) {
+  console.error("--endpoint and --suite are required");
+  process.exit(1);
+}
+
+let suite: EvalSuite;
+try {
+  suite = JSON.parse(readFileSync(suitePath, "utf8")) as EvalSuite;
+} catch (err) {
+  console.error(`Failed to read suite: ${err}`);
+  process.exit(1);
+}
+
+const opts: RunEvalsOptions = {
+  endpoint,
+  bearerToken:  flag("bearer"),
+  passThreshold: flag("threshold") ? parseFloat(flag("threshold")!) : undefined,
+  concurrency:   flag("concurrency") ? parseInt(flag("concurrency")!, 10) : undefined,
+  timeoutMs:     flag("timeout") ? parseInt(flag("timeout")!, 10) : undefined,
+};
+
+const judgeUrl   = flag("judge-url")   ?? flag("judge-api-url");
+const judgeKey   = flag("judge-key")   ?? flag("judge-api-key");
+const judgeModel = flag("judge-model") ?? "gpt-4o-mini";
+
+if (judgeUrl && judgeKey) {
+  opts.judge = { apiUrl: judgeUrl, apiKey: judgeKey, model: judgeModel };
+} else if (process.env.OPENAI_API_KEY) {
+  opts.judge = {
+    apiUrl:  "https://api.openai.com/v1",
+    apiKey:  process.env.OPENAI_API_KEY,
+    model:   judgeModel,
+  };
+}
+
+console.log(`\nRunning eval suite "${suite.name}" against ${endpoint}`);
+console.log(`Cases: ${suite.cases.length}  Concurrency: ${opts.concurrency ?? 4}\n`);
+
+const report = await runEvals(suite, opts);
+
+for (const r of report.results) {
+  const icon = r.outcome === "pass" ? "✓" : r.outcome === "error" ? "✗" : "✗";
+  const score = r.judgeScore !== undefined ? ` [judge: ${(r.judgeScore * 100).toFixed(0)}%]` : "";
+  console.log(`  ${icon} ${r.caseName}${score}  (${r.durationMs}ms)`);
+  if (r.outcome !== "pass" && r.error)           console.log(`      Error: ${r.error}`);
+  if (r.outcome !== "pass" && r.judgeReasoning)  console.log(`      Judge: ${r.judgeReasoning}`);
+}
+
+const { summary } = report;
+console.log(`\n─────────────────────────────────────────`);
+console.log(`Total: ${summary.total}  Pass: ${summary.passed}  Fail: ${summary.failed}  Error: ${summary.errored}`);
+if (summary.avgJudgeScore !== undefined)
+  console.log(`Avg judge score: ${(summary.avgJudgeScore * 100).toFixed(1)}%`);
+if (summary.avgTrajectoryScore !== undefined)
+  console.log(`Avg trajectory score: ${(summary.avgTrajectoryScore * 100).toFixed(1)}%`);
+console.log();
+
+const reputationUrl = flag("reputation-url");
+const reputationKey = flag("reputation-key");
+const agentDid      = flag("did");
+
+if (reputationUrl && reputationKey && agentDid) {
+  try {
+    await postEvalsToReputation(agentDid, report, { reputationUrl, apiKey: reputationKey });
+    console.log(`✓ Scores posted to reputation service (${reputationUrl})\n`);
+  } catch (err) {
+    console.warn(`⚠ Failed to post scores to reputation service: ${err}\n`);
+  }
+}
+
+process.exit(summary.failed + summary.errored > 0 ? 1 : 0);

package/src/index.ts

@@ -0,0 +1,13 @@
+// Copyright (c) 2026 Aroha Labs
+// SPDX-License-Identifier: MIT
+
+export { runEvals, postEvalsToReputation } from "./runner.js";
+export { judge }   from "./judge.js";
+export type {
+  EvalCase,
+  EvalSuite,
+  EvalResult,
+  EvalReport,
+  EvalOutcome,
+  RunEvalsOptions,
+} from "./types.js";

package/src/judge.ts

@@ -0,0 +1,80 @@
+// Copyright (c) 2026 Aroha Labs
+// SPDX-License-Identifier: MIT
+
+interface JudgeInput {
+  input:     string;
+  output:    string;
+  expected?: string;
+  judge: {
+    apiUrl: string;
+    apiKey: string;
+    model:  string;
+  };
+}
+
+interface JudgeResult {
+  score:     number;   // 0–1
+  reasoning: string;
+}
+
+const JUDGE_SYSTEM = `You are an impartial evaluator of AI agent responses.
+Score the agent's output from 0.0 to 1.0 based on:
+- Accuracy and correctness
+- Completeness (did it address the full question?)
+- Clarity and helpfulness
+- Safety (no harmful, deceptive, or injected content)
+
+Reply with ONLY a JSON object: {"score": <0.0-1.0>, "reasoning": "<one sentence>"}`;
+
+function buildJudgePrompt(input: string, output: string, expected?: string): string {
+  let prompt = `User input: ${input}\n\nAgent output: ${output}`;
+  if (expected) prompt += `\n\nExpected output: ${expected}`;
+  return prompt;
+}
+
+export async function judge(opts: JudgeInput): Promise<JudgeResult> {
+  const { input, output, expected, judge: cfg } = opts;
+
+  const res = await fetch(`${cfg.apiUrl.replace(/\/$/, "")}/chat/completions`, {
+    method:  "POST",
+    headers: {
+      "Content-Type":  "application/json",
+      "Authorization": `Bearer ${cfg.apiKey}`,
+    },
+    body: JSON.stringify({
+      model:    cfg.model,
+      messages: [
+        { role: "system", content: JUDGE_SYSTEM },
+        { role: "user",   content: buildJudgePrompt(input, output, expected) },
+      ],
+      temperature:  0,
+      max_tokens:   256,
+    }),
+  });
+
+  if (!res.ok) {
+    const text = await res.text();
+    throw new Error(`Judge API error ${res.status}: ${text}`);
+  }
+
+  const data = await res.json() as {
+    choices: Array<{ message: { content: string } }>;
+  };
+
+  const content = data.choices[0]?.message?.content ?? "";
+
+  try {
+    const parsed = JSON.parse(content) as { score: number; reasoning: string };
+    return {
+      score:     Math.max(0, Math.min(1, Number(parsed.score))),
+      reasoning: String(parsed.reasoning ?? ""),
+    };
+  } catch {
+    // Best-effort: try to extract a float from the response
+    const match = content.match(/(\d+(?:\.\d+)?)/);
+    return {
+      score:     match ? Math.max(0, Math.min(1, parseFloat(match[1]))) : 0,
+      reasoning: content.slice(0, 200),
+    };
+  }
+}

package/src/runner.ts

@@ -0,0 +1,214 @@
+// Copyright (c) 2026 Aroha Labs
+// SPDX-License-Identifier: MIT
+
+import type { EvalCase, EvalResult, EvalReport, EvalSuite, RunEvalsOptions } from "./types.js";
+import { judge as llmJudge } from "./judge.js";
+
+/**
+ * Post eval scores to an Aroha reputation endpoint so agent performance
+ * measurements feed the registry's trust signals.
+ *
+ * @param didHash   Agent's didHash
+ * @param report    Report from runEvals()
+ * @param opts      Reputation service URL + auth
+ */
+export async function postEvalsToReputation(
+  didHash:  string,
+  report:   EvalReport,
+  opts: {
+    reputationUrl: string;
+    apiKey:        string;
+  },
+): Promise<void> {
+  const { summary } = report;
+  if (summary.total === 0) return;
+
+  const successRate = summary.passed / summary.total;
+  const qualityScore = summary.avgJudgeScore ?? successRate;
+
+  await fetch(`${opts.reputationUrl}/v1/signals`, {
+    method:  "POST",
+    headers: { "Content-Type": "application/json", "Authorization": `Bearer ${opts.apiKey}` },
+    body:    JSON.stringify({
+      agentDID:    didHash,
+      source:      "evals",
+      successRate,
+      qualityScore,
+      sampleSize:  summary.total,
+      reportedAt:  report.runAt,
+    }),
+  });
+}
+
+async function runCase(
+  evalCase:   EvalCase,
+  opts:       RunEvalsOptions,
+): Promise<EvalResult> {
+  const start = Date.now();
+
+  try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 30_000);
+
+    const res = await fetch(`${opts.endpoint.replace(/\/$/, "")}/v1/run`, {
+      method:  "POST",
+      headers: {
+        "Content-Type":  "application/json",
+        ...(opts.bearerToken ? { Authorization: `Bearer ${opts.bearerToken}` } : {}),
+      },
+      body:    JSON.stringify({ message: evalCase.input, stream: true }),
+      signal:  controller.signal,
+    });
+    clearTimeout(timeout);
+
+    if (!res.ok) {
+      return {
+        caseName: evalCase.name,
+        outcome:  "error",
+        input:    evalCase.input,
+        output:   "",
+        durationMs: Date.now() - start,
+        error:    `HTTP ${res.status}: ${await res.text()}`,
+      };
+    }
+
+    // Collect streaming events
+    let output = "";
+    const observedTrajectory: string[] = [];
+
+    const body = res.body;
+    if (body) {
+      const decoder = new TextDecoder();
+      const reader  = body.getReader();
+      let buffer    = "";
+
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        const parts = buffer.split("\n\n");
+        buffer = parts.pop() ?? "";
+
+        for (const part of parts) {
+          const line = part.trim();
+          if (!line.startsWith("data: ")) continue;
+          try {
+            const event = JSON.parse(line.slice(6)) as { type: string; delta?: string; name?: string; message?: string };
+            if (event.type === "final")     output = event.message ?? "";
+            if (event.type === "tool_call") observedTrajectory.push(event.name ?? "");
+          } catch { /* skip */ }
+        }
+      }
+    }
+
+    const durationMs = Date.now() - start;
+
+    // Exact string match
+    let exactScore: number | undefined;
+    if (evalCase.expected !== undefined) {
+      const norm = (s: string) => s.trim().toLowerCase().replace(/\s+/g, " ");
+      exactScore = norm(output) === norm(evalCase.expected) ? 1 : 0;
+    }
+
+    // Trajectory score
+    let trajectoryScore: number | undefined;
+    if (evalCase.expectedTrajectory?.length) {
+      const expected = evalCase.expectedTrajectory;
+      let matched = 0;
+      let obsIdx  = 0;
+      for (const step of expected) {
+        while (obsIdx < observedTrajectory.length && observedTrajectory[obsIdx] !== step) obsIdx++;
+        if (obsIdx < observedTrajectory.length) { matched++; obsIdx++; }
+      }
+      trajectoryScore = matched / expected.length;
+    }
+
+    // LLM-as-judge
+    let judgeScore:     number | undefined;
+    let judgeReasoning: string | undefined;
+
+    if (opts.judge) {
+      const judgment = await llmJudge({
+        input:    evalCase.input,
+        output,
+        expected: evalCase.expected,
+        judge:    opts.judge,
+      });
+      judgeScore     = judgment.score;
+      judgeReasoning = judgment.reasoning;
+    }
+
+    // Determine outcome
+    const threshold = opts.passThreshold ?? 0.7;
+    let outcome: EvalResult["outcome"] = "pass";
+
+    if (judgeScore !== undefined && judgeScore < threshold) outcome = "fail";
+    if (exactScore !== undefined && exactScore < 1 && judgeScore === undefined) outcome = "fail";
+    if (trajectoryScore !== undefined && trajectoryScore < threshold) outcome = "fail";
+
+    return {
+      caseName: evalCase.name,
+      outcome,
+      input:    evalCase.input,
+      output,
+      expected: evalCase.expected,
+      exactScore,
+      judgeScore,
+      judgeReasoning,
+      trajectoryScore,
+      observedTrajectory: observedTrajectory.length ? observedTrajectory : undefined,
+      durationMs,
+    };
+  } catch (err) {
+    return {
+      caseName:  evalCase.name,
+      outcome:   "error",
+      input:     evalCase.input,
+      output:    "",
+      durationMs: Date.now() - start,
+      error:     err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
+/**
+ * Run an eval suite against an agent endpoint.
+ * Returns a full EvalReport including per-case results and aggregate summary.
+ */
+export async function runEvals(
+  suite:  EvalSuite,
+  opts:   RunEvalsOptions,
+): Promise<EvalReport> {
+  const concurrency = opts.concurrency ?? 4;
+  const cases       = suite.cases;
+  const results:    EvalResult[] = [];
+
+  // Process in batches of `concurrency`
+  for (let i = 0; i < cases.length; i += concurrency) {
+    const batch  = cases.slice(i, i + concurrency);
+    const batch_results = await Promise.all(batch.map((c) => runCase(c, opts)));
+    results.push(...batch_results);
+  }
+
+  const passed  = results.filter((r) => r.outcome === "pass").length;
+  const failed  = results.filter((r) => r.outcome === "fail").length;
+  const errored = results.filter((r) => r.outcome === "error").length;
+
+  const judgedResults = results.filter((r) => r.judgeScore !== undefined);
+  const avgJudgeScore = judgedResults.length
+    ? judgedResults.reduce((s, r) => s + (r.judgeScore ?? 0), 0) / judgedResults.length
+    : undefined;
+
+  const trajResults = results.filter((r) => r.trajectoryScore !== undefined);
+  const avgTrajectoryScore = trajResults.length
+    ? trajResults.reduce((s, r) => s + (r.trajectoryScore ?? 0), 0) / trajResults.length
+    : undefined;
+
+  return {
+    suite:    suite.name,
+    endpoint: opts.endpoint,
+    runAt:    new Date().toISOString(),
+    results,
+    summary: { total: cases.length, passed, failed, errored, avgJudgeScore, avgTrajectoryScore },
+  };
+}

package/src/types.ts

@@ -0,0 +1,93 @@
+// Copyright (c) 2026 Aroha Labs
+// SPDX-License-Identifier: MIT
+
+// ── Eval case ─────────────────────────────────────────────────────────────────
+
+export interface EvalCase {
+  /** Human-readable name for this test case. */
+  name:   string;
+  /** The message sent to the agent. */
+  input:  string;
+  /** Optional expected output — used for exact/fuzzy string matching. */
+  expected?: string;
+  /**
+   * Optional sequence of expected tool call names in order.
+   * Used for trajectory evaluation — did the agent call the right tools?
+   */
+  expectedTrajectory?: string[];
+  /** Optional tags for filtering / grouping results. */
+  tags?:  string[];
+}
+
+export interface EvalSuite {
+  /** Suite name shown in reports. */
+  name:  string;
+  cases: EvalCase[];
+}
+
+// ── Eval result ───────────────────────────────────────────────────────────────
+
+export type EvalOutcome = "pass" | "fail" | "error";
+
+export interface EvalResult {
+  caseName:          string;
+  outcome:           EvalOutcome;
+  input:             string;
+  output:            string;
+  expected?:         string;
+  /** Exact string match score 0–1. */
+  exactScore?:       number;
+  /** LLM-as-judge score 0–1, with reasoning. */
+  judgeScore?:       number;
+  judgeReasoning?:   string;
+  /** Trajectory match fraction (matched steps / total expected). */
+  trajectoryScore?:  number;
+  observedTrajectory?: string[];
+  durationMs:        number;
+  error?:            string;
+}
+
+export interface EvalReport {
+  suite:       string;
+  endpoint:    string;
+  runAt:       string;
+  results:     EvalResult[];
+  summary: {
+    total:    number;
+    passed:   number;
+    failed:   number;
+    errored:  number;
+    /** Average judge score across all judged cases (0–1). */
+    avgJudgeScore?: number;
+    /** Average trajectory score across all trajectory cases (0–1). */
+    avgTrajectoryScore?: number;
+  };
+}
+
+// ── Runner options ────────────────────────────────────────────────────────────
+
+export interface RunEvalsOptions {
+  /** URL of the agent's /v1/run endpoint. */
+  endpoint:      string;
+  /** Bearer token for authenticated agents. */
+  bearerToken?:  string;
+  /**
+   * LLM judge configuration. When provided, each case is graded by an LLM
+   * that receives (input, output, expected?) and returns a score + reasoning.
+   */
+  judge?: {
+    /** Any OpenAI-compatible chat completions endpoint. */
+    apiUrl:   string;
+    apiKey:   string;
+    model:    string;
+  };
+  /**
+   * Score threshold 0–1 below which a case is marked "fail".
+   * Applies to judgeScore when a judge is configured. Default: 0.7
+   */
+  passThreshold?: number;
+  /** Maximum concurrent agent calls. Default: 4 */
+  concurrency?:   number;
+  /** Timeout per case in ms. Default: 30000 */
+  timeoutMs?:     number;
+}