@hasna/evals 0.1.0

package/dist/types/index.d.ts

@@ -0,0 +1,171 @@
+export type Verdict = "PASS" | "FAIL" | "UNKNOWN";
+export interface HttpAdapterConfig {
+    type: "http";
+    url: string;
+    method?: "GET" | "POST" | "PUT" | "PATCH";
+    headers?: Record<string, string>;
+    /** Path into request body where the input message goes, e.g. "messages[-1].content" */
+    inputPath?: string;
+    /** Path into response body where the output text lives, e.g. "choices[0].message.content" */
+    outputPath?: string;
+    timeoutMs?: number;
+}
+export interface AnthropicAdapterConfig {
+    type: "anthropic";
+    model: string;
+    systemPrompt?: string;
+    maxTokens?: number;
+    apiKey?: string;
+}
+export interface OpenAIAdapterConfig {
+    type: "openai";
+    model: string;
+    systemPrompt?: string;
+    maxTokens?: number;
+    baseURL?: string;
+    apiKey?: string;
+}
+export interface McpAdapterConfig {
+    type: "mcp";
+    /** Command to start the MCP server, e.g. ["node", "dist/mcp/index.js"] */
+    command: string[];
+    /** Tool name to call */
+    tool: string;
+    /** How to map the EvalCase input into tool arguments */
+    inputMapping?: Record<string, string>;
+    timeoutMs?: number;
+}
+export interface FunctionAdapterConfig {
+    type: "function";
+    /** Absolute path to module */
+    modulePath: string;
+    /** Named export to call */
+    exportName?: string;
+}
+export interface CliAdapterConfig {
+    type: "cli";
+    /** Command template — use {{input}} as placeholder */
+    command: string;
+    timeoutMs?: number;
+    env?: Record<string, string>;
+}
+export type AdapterConfig = HttpAdapterConfig | AnthropicAdapterConfig | OpenAIAdapterConfig | McpAdapterConfig | FunctionAdapterConfig | CliAdapterConfig;
+export type AssertionType = "contains" | "not_contains" | "starts_with" | "ends_with" | "equals" | "regex" | "not_regex" | "max_length" | "min_length" | "json_valid" | "json_schema" | "tool_called" | "tool_not_called" | "tool_call_count" | "tool_args_match" | "response_time_ms" | "token_count" | "cost_usd" | "semantic_similarity";
+export interface Assertion {
+    type: AssertionType;
+    /** The value to check against — type depends on assertion type */
+    value?: string | number | boolean | Record<string, unknown>;
+    /** For range-based assertions */
+    min?: number;
+    max?: number;
+    /** For semantic_similarity — 0.0 to 1.0, default 0.8 */
+    threshold?: number;
+    /** Human-readable label for reports */
+    label?: string;
+}
+export interface AssertionResult {
+    type: AssertionType;
+    passed: boolean;
+    reason: string;
+    label?: string;
+    durationMs?: number;
+}
+export interface JudgeConfig {
+    /** Plain-English grading criteria. Required. */
+    rubric: string;
+    /** Judge model. Default: claude-sonnet-4-6 */
+    model?: string;
+    /** Judge provider. Default: anthropic */
+    provider?: "anthropic" | "openai";
+    /** API key override — falls back to env */
+    apiKey?: string;
+}
+export interface JudgeResult {
+    verdict: Verdict;
+    /** Chain-of-thought reasoning — always present before verdict */
+    reasoning: string;
+    durationMs: number;
+    inputTokens?: number;
+    outputTokens?: number;
+    costUsd?: number;
+}
+export interface ConversationTurn {
+    role: "user" | "assistant";
+    content: string;
+    /** For assistant turns: what the expected behavior should be (natural language) */
+    expected?: string;
+}
+export interface EvalCase {
+    id: string;
+    /** Single-turn: plain string input */
+    input?: string;
+    /** Multi-turn: conversation turns. If present, input is ignored. */
+    turns?: ConversationTurn[];
+    /** Natural language description of expected output (for judge) */
+    expected?: string;
+    /** Adapter config override — falls back to run-level config */
+    adapter?: AdapterConfig;
+    assertions?: Assertion[];
+    judge?: JudgeConfig;
+    /** Run this case N times and report pass_rate (Pass^k metric) */
+    repeat?: number;
+    /** Minimum pass rate for Pass^k to be considered passing (0.0–1.0, default 1.0) */
+    passThreshold?: number;
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+}
+export interface EvalResult {
+    caseId: string;
+    verdict: Verdict;
+    /** Raw output from the app under test */
+    output: string;
+    /** For multi-turn: all turn outputs */
+    turnOutputs?: string[];
+    assertionResults: AssertionResult[];
+    judgeResult?: JudgeResult;
+    /** For Pass^k: individual verdicts per repeat */
+    repeatVerdicts?: Verdict[];
+    passRate?: number;
+    durationMs: number;
+    inputTokens?: number;
+    outputTokens?: number;
+    costUsd?: number;
+    error?: string;
+}
+export interface EvalRunStats {
+    total: number;
+    passed: number;
+    failed: number;
+    unknown: number;
+    errors: number;
+    passRate: number;
+    totalDurationMs: number;
+    totalCostUsd: number;
+    totalTokens: number;
+}
+export interface EvalRun {
+    id: string;
+    createdAt: string;
+    dataset: string;
+    adapterConfig?: AdapterConfig;
+    results: EvalResult[];
+    stats: EvalRunStats;
+    /** Named baseline tag if set */
+    baselineName?: string;
+}
+export interface RunOptions {
+    dataset: string;
+    adapter?: AdapterConfig;
+    concurrency?: number;
+    tags?: string[];
+    skipJudge?: boolean;
+    repeat?: number;
+    outputFormat?: "terminal" | "json" | "markdown";
+    verbose?: boolean;
+}
+export interface CiOptions extends RunOptions {
+    baselineName?: string;
+    baselineRunId?: string;
+    failIfRegressionPct?: number;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;AAIlD,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uFAAuF;IACvF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6FAA6F;IAC7F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,KAAK,CAAC;IACZ,0EAA0E;IAC1E,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,UAAU,CAAC;IACjB,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,KAAK,CAAC;IACZ,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAED,MAAM,MAAM,aAAa,GACrB,iBAAiB,GACjB,sBAAsB,GACtB,mBAAmB,GACnB,gBAAgB,GAChB,qBAAqB,GACrB,gBAAgB,CAAC;AAIrB,MAAM,MAAM,aAAa,GACrB,UAAU,GACV,cAAc,GACd,aAAa,GACb,WAAW,GACX,QAAQ,GACR,OAAO,GACP,WAAW,GACX,YAAY,GACZ,YAAY,GACZ,YAAY,GACZ,aAAa,GACb,aAAa,GACb,iBAAiB,GACjB,iBAAiB,GACjB,iBAAiB,GACjB,kBAAkB,GAClB,aAAa,GACb,UAAU,GACV,qBAAqB,CAAC;AAE1B,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5D,iCAAiC;IACjC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,aAAa,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAID,MAAM,WAAW,WAAW;IAC1B,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yCAAyC;IACzC,QAAQ,CAAC,EAAE,WAAW,GAAG,QAAQ,CAAC;IAClC,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAID,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,mFAAmF;IACnF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,sCAAsC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oEAAoE;IACpE,KAAK,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC3B,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mFAAmF;IACnF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAID,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,uCAAuC;IACvC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,EAAE,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,gCAAgC;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAID,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC;IAChD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,SAAU,SAAQ,UAAU;IAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B"}

package/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "@hasna/evals",
+  "version": "0.1.0",
+  "description": "Open source AI evaluation framework — LLM-as-judge + assertion-based evals for any AI app. CLI + MCP server.",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "evals": "dist/cli/index.js",
+    "evals-mcp": "dist/mcp/index.js",
+    "evals-serve": "dist/server/index.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "datasets/examples",
+    "LICENSE",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "bun build src/cli/index.ts --outdir dist/cli --target bun --external ink --external react --external chalk --external @modelcontextprotocol/sdk && bun build src/mcp/index.ts --outdir dist/mcp --target bun --external @modelcontextprotocol/sdk && bun build src/server/index.ts --outdir dist/server --target bun && bun build src/index.ts --outdir dist --target bun && tsc --emitDeclarationOnly --outDir dist",
+    "typecheck": "tsc --noEmit",
+    "test": "bun test",
+    "dev:cli": "bun run src/cli/index.ts",
+    "dev:mcp": "bun run src/mcp/index.ts",
+    "dev:serve": "bun run src/server/index.ts",
+    "prepublishOnly": "bun run typecheck && bun test && bun run build",
+    "postinstall": "mkdir -p $HOME/.hasna/evals 2>/dev/null || true"
+  },
+  "keywords": [
+    "evals",
+    "llm",
+    "ai",
+    "testing",
+    "evaluation",
+    "mcp",
+    "claude",
+    "llm-as-judge",
+    "typescript",
+    "cli"
+  ],
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org",
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/hasna/open-evals.git"
+  },
+  "homepage": "https://github.com/hasna/open-evals",
+  "bugs": {
+    "url": "https://github.com/hasna/open-evals/issues"
+  },
+  "engines": {
+    "bun": ">=1.0.0"
+  },
+  "author": "Andrei Hasna <andrei@hasna.com>",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.39.0",
+    "@modelcontextprotocol/sdk": "^1.12.1",
+    "ajv": "^8.18.0",
+    "chalk": "^5.4.1",
+    "commander": "^13.1.0",
+    "openai": "^4.98.0",
+    "zod": "^3.24.2"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.2.4",
+    "typescript": "^5.7.3"
+  }
+}