@hasna/evals 0.1.24 → 0.1.26

package/dist/mcp/mcp.test.d.ts

@@ -0,0 +1,2 @@
+export declare function echoFn(input: string): Promise<string>;
+//# sourceMappingURL=mcp.test.d.ts.map

package/dist/mcp/mcp.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp.test.d.ts","sourceRoot":"","sources":["../../src/mcp/mcp.test.ts"],"names":[],"mappings":"AAsDA,wBAAsB,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAE3D"}

package/dist/mcp/server.d.ts

@@ -0,0 +1,5 @@
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+export declare const MCP_NAME = "evals";
+export declare const DEFAULT_MCP_HTTP_PORT = 8862;
+export declare function buildServer(): Server;
+//# sourceMappingURL=server.d.ts.map

package/dist/mcp/server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AAanE,eAAO,MAAM,QAAQ,UAAU,CAAC;AAChC,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAE1C,wBAAgB,WAAW,IAAI,MAAM,CAmSpC"}

package/dist/server/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env bun
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/server/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":""}

package/dist/server/index.js

@@ -11666,7 +11666,7 @@ var safeJSON2 = (text) => {
 var sleep2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
 // node_modules/openai/version.mjs
-var VERSION2 = "6.39.0";
+var VERSION2 = "6.42.0";
 
 // node_modules/openai/internal/detect-platform.mjs
 var isRunningInBrowser2 = () => {
@@ -18252,7 +18252,10 @@ https://help.openai.com/en/articles/5112595-best-practices-for-api-key-safety
       if (isTimeout) {
         throw new APIConnectionTimeoutError2;
       }
-      throw new APIConnectionError2({ cause: response });
+      throw new APIConnectionError2({
+        message: getConnectionErrorMessage(response),
+        cause: response
+      });
     }
     const specialHeaders = [...response.headers.entries()].filter(([name]) => name === "x-request-id").map(([name, value]) => ", " + name + ": " + JSON.stringify(value)).join("");
     const responseInfo = `[${requestLogID}${retryLogStr}${specialHeaders}] ${req.method} ${url} ${response.ok ? "succeeded" : "failed"} with status ${response.status} in ${headersTime - startTime}ms`;
@@ -18525,6 +18528,23 @@ OpenAI.Evals = Evals;
 OpenAI.Containers = Containers;
 OpenAI.Skills = Skills2;
 OpenAI.Videos = Videos;
+function getConnectionErrorMessage(error3) {
+  if (isUndiciDispatcherVersionMismatchError(error3)) {
+    return `Connection error. This may be caused by passing an undici dispatcher, such as ProxyAgent, that is incompatible with the fetch implementation. If you are using undici's ProxyAgent, pass the fetch implementation from the same undici package: import { fetch, ProxyAgent } from 'undici'; new OpenAI({ fetch, fetchOptions: { dispatcher: new ProxyAgent(...) } });`;
+  }
+  return;
+}
+function isUndiciDispatcherVersionMismatchError(error3) {
+  let current = error3;
+  for (let i = 0;i < 8 && current && typeof current === "object"; i++) {
+    const err = current;
+    if (err.code === "UND_ERR_INVALID_ARG" && typeof err.message === "string" && err.message.includes("invalid onRequestStart method")) {
+      return true;
+    }
+    current = err.cause;
+  }
+  return false;
+}
 // node_modules/openai/azure.mjs
 var _deployments_endpoints = new Set([
   "/completions",
@@ -22595,7 +22615,7 @@ function finalize(ctx, schema) {
     result.$schema = "http://json-schema.org/draft-07/schema#";
   } else if (ctx.target === "draft-04") {
     result.$schema = "http://json-schema.org/draft-04/schema#";
-  } else if (ctx.target === "openapi-3.0") {} else {}
+  } else if (ctx.target === "openapi-3.0") {}
   if (ctx.external?.uri) {
     const id = ctx.external.registry.get(schema)?.id;
     if (!id)
@@ -22813,7 +22833,7 @@ var literalProcessor = (schema, ctx, json, _params) => {
     if (val === undefined) {
       if (ctx.unrepresentable === "throw") {
         throw new Error("Literal `undefined` cannot be represented in JSON Schema");
-      } else {}
+      }
     } else if (typeof val === "bigint") {
       if (ctx.unrepresentable === "throw") {
         throw new Error("BigInt literals cannot be represented in JSON Schema");

package/dist/server/server.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=server.test.d.ts.map

package/dist/server/server.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.test.d.ts","sourceRoot":"","sources":["../../src/server/server.test.ts"],"names":[],"mappings":""}

package/dist/types/index.d.ts

@@ -0,0 +1,171 @@
+export type Verdict = "PASS" | "FAIL" | "UNKNOWN";
+export interface HttpAdapterConfig {
+    type: "http";
+    url: string;
+    method?: "GET" | "POST" | "PUT" | "PATCH";
+    headers?: Record<string, string>;
+    /** Path into request body where the input message goes, e.g. "messages[-1].content" */
+    inputPath?: string;
+    /** Path into response body where the output text lives, e.g. "choices[0].message.content" */
+    outputPath?: string;
+    timeoutMs?: number;
+}
+export interface AnthropicAdapterConfig {
+    type: "anthropic";
+    model: string;
+    systemPrompt?: string;
+    maxTokens?: number;
+    apiKey?: string;
+}
+export interface OpenAIAdapterConfig {
+    type: "openai";
+    model: string;
+    systemPrompt?: string;
+    maxTokens?: number;
+    baseURL?: string;
+    apiKey?: string;
+}
+export interface McpAdapterConfig {
+    type: "mcp";
+    /** Command to start the MCP server, e.g. ["node", "dist/mcp/index.js"] */
+    command: string[];
+    /** Tool name to call */
+    tool: string;
+    /** How to map the EvalCase input into tool arguments */
+    inputMapping?: Record<string, string>;
+    timeoutMs?: number;
+}
+export interface FunctionAdapterConfig {
+    type: "function";
+    /** Absolute path to module */
+    modulePath: string;
+    /** Named export to call */
+    exportName?: string;
+}
+export interface CliAdapterConfig {
+    type: "cli";
+    /** Command template — use {{input}} as placeholder */
+    command: string;
+    timeoutMs?: number;
+    env?: Record<string, string>;
+}
+export type AdapterConfig = HttpAdapterConfig | AnthropicAdapterConfig | OpenAIAdapterConfig | McpAdapterConfig | FunctionAdapterConfig | CliAdapterConfig;
+export type AssertionType = "contains" | "not_contains" | "starts_with" | "ends_with" | "equals" | "regex" | "not_regex" | "max_length" | "min_length" | "json_valid" | "json_schema" | "tool_called" | "tool_not_called" | "tool_call_count" | "tool_args_match" | "response_time_ms" | "token_count" | "cost_usd" | "semantic_similarity";
+export interface Assertion {
+    type: AssertionType;
+    /** The value to check against — type depends on assertion type */
+    value?: string | number | boolean | Record<string, unknown>;
+    /** For range-based assertions */
+    min?: number;
+    max?: number;
+    /** For semantic_similarity — 0.0 to 1.0, default 0.8 */
+    threshold?: number;
+    /** Human-readable label for reports */
+    label?: string;
+}
+export interface AssertionResult {
+    type: AssertionType;
+    passed: boolean;
+    reason: string;
+    label?: string;
+    durationMs?: number;
+}
+export interface JudgeConfig {
+    /** Plain-English grading criteria. Required. */
+    rubric: string;
+    /** Judge model. Default: claude-sonnet-4-6 */
+    model?: string;
+    /** Judge provider. Default: anthropic */
+    provider?: "anthropic" | "openai";
+    /** API key override — falls back to env */
+    apiKey?: string;
+}
+export interface JudgeResult {
+    verdict: Verdict;
+    /** Chain-of-thought reasoning — always present before verdict */
+    reasoning: string;
+    durationMs: number;
+    inputTokens?: number;
+    outputTokens?: number;
+    costUsd?: number;
+}
+export interface ConversationTurn {
+    role: "user" | "assistant";
+    content: string;
+    /** For assistant turns: what the expected behavior should be (natural language) */
+    expected?: string;
+}
+export interface EvalCase {
+    id: string;
+    /** Single-turn: plain string input */
+    input?: string;
+    /** Multi-turn: conversation turns. If present, input is ignored. */
+    turns?: ConversationTurn[];
+    /** Natural language description of expected output (for judge) */
+    expected?: string;
+    /** Adapter config override — falls back to run-level config */
+    adapter?: AdapterConfig;
+    assertions?: Assertion[];
+    judge?: JudgeConfig;
+    /** Run this case N times and report pass_rate (Pass^k metric) */
+    repeat?: number;
+    /** Minimum pass rate for Pass^k to be considered passing (0.0–1.0, default 1.0) */
+    passThreshold?: number;
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+}
+export interface EvalResult {
+    caseId: string;
+    verdict: Verdict;
+    /** Raw output from the app under test */
+    output: string;
+    /** For multi-turn: all turn outputs */
+    turnOutputs?: string[];
+    assertionResults: AssertionResult[];
+    judgeResult?: JudgeResult;
+    /** For Pass^k: individual verdicts per repeat */
+    repeatVerdicts?: Verdict[];
+    passRate?: number;
+    durationMs: number;
+    inputTokens?: number;
+    outputTokens?: number;
+    costUsd?: number;
+    error?: string;
+}
+export interface EvalRunStats {
+    total: number;
+    passed: number;
+    failed: number;
+    unknown: number;
+    errors: number;
+    passRate: number;
+    totalDurationMs: number;
+    totalCostUsd: number;
+    totalTokens: number;
+}
+export interface EvalRun {
+    id: string;
+    createdAt: string;
+    dataset: string;
+    adapterConfig?: AdapterConfig;
+    results: EvalResult[];
+    stats: EvalRunStats;
+    /** Named baseline tag if set */
+    baselineName?: string;
+}
+export interface RunOptions {
+    dataset: string;
+    adapter?: AdapterConfig;
+    concurrency?: number;
+    tags?: string[];
+    skipJudge?: boolean;
+    repeat?: number;
+    outputFormat?: "terminal" | "json" | "markdown";
+    verbose?: boolean;
+}
+export interface CiOptions extends RunOptions {
+    baselineName?: string;
+    baselineRunId?: string;
+    failIfRegressionPct?: number;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;AAIlD,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uFAAuF;IACvF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6FAA6F;IAC7F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,KAAK,CAAC;IACZ,0EAA0E;IAC1E,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,UAAU,CAAC;IACjB,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,KAAK,CAAC;IACZ,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAED,MAAM,MAAM,aAAa,GACrB,iBAAiB,GACjB,sBAAsB,GACtB,mBAAmB,GACnB,gBAAgB,GAChB,qBAAqB,GACrB,gBAAgB,CAAC;AAIrB,MAAM,MAAM,aAAa,GACrB,UAAU,GACV,cAAc,GACd,aAAa,GACb,WAAW,GACX,QAAQ,GACR,OAAO,GACP,WAAW,GACX,YAAY,GACZ,YAAY,GACZ,YAAY,GACZ,aAAa,GACb,aAAa,GACb,iBAAiB,GACjB,iBAAiB,GACjB,iBAAiB,GACjB,kBAAkB,GAClB,aAAa,GACb,UAAU,GACV,qBAAqB,CAAC;AAE1B,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5D,iCAAiC;IACjC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,aAAa,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAID,MAAM,WAAW,WAAW;IAC1B,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yCAAyC;IACzC,QAAQ,CAAC,EAAE,WAAW,GAAG,QAAQ,CAAC;IAClC,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAID,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,mFAAmF;IACnF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,sCAAsC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oEAAoE;IACpE,KAAK,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC3B,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mFAAmF;IACnF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAID,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,uCAAuC;IACvC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,EAAE,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,gCAAgC;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAID,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC;IAChD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,SAAU,SAAQ,UAAU;IAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hasna/evals",
-  "version": "0.1.24",
+  "version": "0.1.26",
   "description": "Open source AI evaluation framework — LLM-as-judge + assertion-based evals for any AI app. CLI + MCP server.",
   "type": "module",
   "main": "dist/index.js",
@@ -65,7 +65,8 @@
   "license": "Apache-2.0",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.82.0",
-    "@hasna/cloud": "^0.1.30",
+    "@hasna/cloud": "0.1.24",
+    "@hasna/events": "^0.1.6",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "ajv": "^8.18.0",
     "chalk": "^5.4.1",