@hasna/evals 0.1.24 → 0.1.26

package/dist/core/assertions-coverage.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=assertions-coverage.test.d.ts.map

package/dist/core/assertions-coverage.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions-coverage.test.d.ts","sourceRoot":"","sources":["../../src/core/assertions-coverage.test.ts"],"names":[],"mappings":""}

package/dist/core/assertions.d.ts

@@ -0,0 +1,18 @@
+import type { Assertion, AssertionResult } from "../types/index.js";
+export declare function sortAssertionsCheapestFirst(assertions: Assertion[]): Assertion[];
+export interface AssertionContext {
+    output: string;
+    durationMs?: number;
+    inputTokens?: number;
+    outputTokens?: number;
+    costUsd?: number;
+    toolCalls?: Array<{
+        name: string;
+        arguments?: Record<string, unknown>;
+    }>;
+}
+export declare function runAssertion(assertion: Assertion, ctx: AssertionContext): Promise<AssertionResult>;
+export declare function runAssertions(assertions: Assertion[], ctx: AssertionContext): Promise<AssertionResult[]>;
+export declare function assertionsPassed(results: AssertionResult[]): boolean;
+export declare function allAssertionsPassed(results: AssertionResult[]): boolean;
+//# sourceMappingURL=assertions.d.ts.map

package/dist/core/assertions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../../src/core/assertions.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AA6BpE,wBAAgB,2BAA2B,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,SAAS,EAAE,CAMhF;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,CAAC;CAC1E;AAED,wBAAsB,YAAY,CAChC,SAAS,EAAE,SAAS,EACpB,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,eAAe,CAAC,CAqB1B;AAED,wBAAsB,aAAa,CACjC,UAAU,EAAE,SAAS,EAAE,EACvB,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,eAAe,EAAE,CAAC,CAyB5B;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,OAAO,CAKpE;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,OAAO,CAEvE"}

package/dist/core/assertions.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=assertions.test.d.ts.map

package/dist/core/assertions.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.test.d.ts","sourceRoot":"","sources":["../../src/core/assertions.test.ts"],"names":[],"mappings":""}

package/dist/core/e2e.test.d.ts

@@ -0,0 +1,2 @@
+export declare function echoInput(input: string): Promise<string>;
+//# sourceMappingURL=e2e.test.d.ts.map

package/dist/core/e2e.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"e2e.test.d.ts","sourceRoot":"","sources":["../../src/core/e2e.test.ts"],"names":[],"mappings":"AAyBA,wBAAsB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAE9D"}

package/dist/core/judge.d.ts

@@ -0,0 +1,13 @@
+import type { JudgeConfig, JudgeResult } from "../types/index.js";
+export declare function runJudge(input: string, output: string, config: JudgeConfig, expected?: string): Promise<JudgeResult>;
+/** One-shot judge: no full eval case, just input/output/rubric */
+export declare function judgeOnce(params: {
+    input: string;
+    output: string;
+    rubric: string;
+    expected?: string;
+    model?: string;
+    provider?: "anthropic" | "openai";
+    apiKey?: string;
+}): Promise<JudgeResult>;
+//# sourceMappingURL=judge.d.ts.map

package/dist/core/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/core/judge.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAW,MAAM,mBAAmB,CAAC;AA4E3E,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,WAAW,EACnB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,WAAW,CAAC,CAmBtB;AA4ED,kEAAkE;AAClE,wBAAsB,SAAS,CAAC,MAAM,EAAE;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,WAAW,GAAG,QAAQ,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,GAAG,OAAO,CAAC,WAAW,CAAC,CAOvB"}

package/dist/core/judge.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=judge.test.d.ts.map

package/dist/core/judge.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.test.d.ts","sourceRoot":"","sources":["../../src/core/judge.test.ts"],"names":[],"mappings":""}

package/dist/core/reporter.d.ts

@@ -0,0 +1,21 @@
+import type { EvalRun, Verdict } from "../types/index.js";
+export declare function printTerminalReport(run: EvalRun): void;
+export declare function toJson(run: EvalRun): string;
+export declare function toMarkdown(run: EvalRun): string;
+export interface RunDiff {
+    regressions: Array<{
+        caseId: string;
+        before: Verdict;
+        after: Verdict;
+    }>;
+    improvements: Array<{
+        caseId: string;
+        before: Verdict;
+        after: Verdict;
+    }>;
+    scoreDelta: number;
+    passRateDelta: number;
+}
+export declare function compareRuns(before: EvalRun, after: EvalRun): RunDiff;
+export declare function printDiffReport(diff: RunDiff): void;
+//# sourceMappingURL=reporter.d.ts.map

package/dist/core/reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/core/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAgB1D,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAoCtD;AAID,wBAAgB,MAAM,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAE3C;AAID,wBAAgB,UAAU,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAkD/C;AAID,MAAM,WAAW,OAAO;IACtB,WAAW,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACxE,YAAY,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAuBpE;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CAcnD"}

package/dist/core/reporter.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=reporter.test.d.ts.map

package/dist/core/reporter.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.test.d.ts","sourceRoot":"","sources":["../../src/core/reporter.test.ts"],"names":[],"mappings":""}

package/dist/core/runner.d.ts

@@ -0,0 +1,4 @@
+import type { AdapterConfig, EvalCase, EvalResult, EvalRun, RunOptions } from "../types/index.js";
+export declare function runEvals(cases: EvalCase[], options: RunOptions): Promise<EvalRun>;
+export declare function runSingleCase(evalCase: EvalCase, adapterConfig: AdapterConfig, skipJudge?: boolean): Promise<EvalResult>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/core/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/core/runner.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EACV,aAAa,EACb,QAAQ,EACR,UAAU,EACV,OAAO,EAEP,UAAU,EAGX,MAAM,mBAAmB,CAAC;AAqI3B,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,QAAQ,EAAE,EACjB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,OAAO,CAAC,CAqClB;AAID,wBAAsB,aAAa,CACjC,QAAQ,EAAE,QAAQ,EAClB,aAAa,EAAE,aAAa,EAC5B,SAAS,UAAQ,GAChB,OAAO,CAAC,UAAU,CAAC,CAKrB"}

package/dist/core/runner.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=runner.test.d.ts.map

package/dist/core/runner.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.test.d.ts","sourceRoot":"","sources":["../../src/core/runner.test.ts"],"names":[],"mappings":""}

package/dist/datasets/loader.d.ts

@@ -0,0 +1,18 @@
+import type { EvalCase } from "../types/index.js";
+export interface LoadOptions {
+    /** Fail hard on malformed lines instead of warning and skipping */
+    strict?: boolean;
+    /** Filter by tags */
+    tags?: string[];
+}
+export interface LoadResult {
+    cases: EvalCase[];
+    warnings: string[];
+    totalLines: number;
+    skipped: number;
+}
+/** Load eval cases from a JSONL or JSON file (or glob pattern) */
+export declare function loadDataset(pathOrGlob: string, opts?: LoadOptions): Promise<LoadResult>;
+/** Stream large JSONL files case-by-case (for very large datasets) */
+export declare function streamDataset(path: string, opts?: LoadOptions): AsyncGenerator<EvalCase>;
+//# sourceMappingURL=loader.d.ts.map

package/dist/datasets/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/datasets/loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAElD,MAAM,WAAW,WAAW;IAC1B,mEAAmE;IACnE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,qBAAqB;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,kEAAkE;AAClE,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAiCjG;AAoFD,sEAAsE;AACtE,wBAAuB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,WAAgB,GAAG,cAAc,CAAC,QAAQ,CAAC,CAyBnG"}

package/dist/datasets/loader.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=loader.test.d.ts.map

package/dist/datasets/loader.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.test.d.ts","sourceRoot":"","sources":["../../src/datasets/loader.test.ts"],"names":[],"mappings":""}

package/dist/db/store.d.ts

@@ -0,0 +1,17 @@
+import { Database } from "bun:sqlite";
+import type { EvalRun } from "../types/index.js";
+export declare function getDatabase(): Database;
+export declare function closeDatabase(): void;
+export declare function saveRun(run: EvalRun): void;
+export declare function getRun(id: string): EvalRun | null;
+export declare function listRuns(limit?: number, dataset?: string): EvalRun[];
+export declare function deleteRun(id: string): void;
+export declare function setBaseline(name: string, runId: string): void;
+export declare function getBaseline(name: string): EvalRun | null;
+export declare function listBaselines(): Array<{
+    name: string;
+    runId: string;
+    createdAt: string;
+}>;
+export declare function clearBaseline(name: string): void;
+//# sourceMappingURL=store.d.ts.map

package/dist/db/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/db/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAItC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAQjD,wBAAgB,WAAW,IAAI,QAAQ,CAUtC;AAED,wBAAgB,aAAa,IAAI,IAAI,CAGpC;AA6BD,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAa1C;AAED,wBAAgB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAIjD;AAED,wBAAgB,QAAQ,CAAC,KAAK,SAAK,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,EAAE,CAMhE;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAE1C;AAID,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAI7D;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAKxD;AAED,wBAAgB,aAAa,IAAI,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAKzF;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAEhD"}

package/dist/db/store.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=store.test.d.ts.map

package/dist/db/store.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.test.d.ts","sourceRoot":"","sources":["../../src/db/store.test.ts"],"names":[],"mappings":""}

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export * from "./types/index.js";
+export * from "./core/runner.js";
+export * from "./core/assertions.js";
+export * from "./core/judge.js";
+export * from "./core/reporter.js";
+export * from "./datasets/loader.js";
+export * from "./db/store.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,sBAAsB,CAAC;AACrC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,eAAe,CAAC"}

package/dist/index.js

@@ -11667,7 +11667,7 @@ var safeJSON2 = (text) => {
 var sleep2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
 // node_modules/openai/version.mjs
-var VERSION2 = "6.39.0";
+var VERSION2 = "6.42.0";
 
 // node_modules/openai/internal/detect-platform.mjs
 var isRunningInBrowser2 = () => {
@@ -18253,7 +18253,10 @@ https://help.openai.com/en/articles/5112595-best-practices-for-api-key-safety
       if (isTimeout) {
         throw new APIConnectionTimeoutError2;
       }
-      throw new APIConnectionError2({ cause: response });
+      throw new APIConnectionError2({
+        message: getConnectionErrorMessage(response),
+        cause: response
+      });
     }
     const specialHeaders = [...response.headers.entries()].filter(([name]) => name === "x-request-id").map(([name, value]) => ", " + name + ": " + JSON.stringify(value)).join("");
     const responseInfo = `[${requestLogID}${retryLogStr}${specialHeaders}] ${req.method} ${url} ${response.ok ? "succeeded" : "failed"} with status ${response.status} in ${headersTime - startTime}ms`;
@@ -18526,6 +18529,23 @@ OpenAI.Evals = Evals;
 OpenAI.Containers = Containers;
 OpenAI.Skills = Skills2;
 OpenAI.Videos = Videos;
+function getConnectionErrorMessage(error3) {
+  if (isUndiciDispatcherVersionMismatchError(error3)) {
+    return `Connection error. This may be caused by passing an undici dispatcher, such as ProxyAgent, that is incompatible with the fetch implementation. If you are using undici's ProxyAgent, pass the fetch implementation from the same undici package: import { fetch, ProxyAgent } from 'undici'; new OpenAI({ fetch, fetchOptions: { dispatcher: new ProxyAgent(...) } });`;
+  }
+  return;
+}
+function isUndiciDispatcherVersionMismatchError(error3) {
+  let current = error3;
+  for (let i = 0;i < 8 && current && typeof current === "object"; i++) {
+    const err = current;
+    if (err.code === "UND_ERR_INVALID_ARG" && typeof err.message === "string" && err.message.includes("invalid onRequestStart method")) {
+      return true;
+    }
+    current = err.cause;
+  }
+  return false;
+}
 // node_modules/openai/azure.mjs
 var _deployments_endpoints = new Set([
   "/completions",
@@ -22596,7 +22616,7 @@ function finalize(ctx, schema) {
     result.$schema = "http://json-schema.org/draft-07/schema#";
   } else if (ctx.target === "draft-04") {
     result.$schema = "http://json-schema.org/draft-04/schema#";
-  } else if (ctx.target === "openapi-3.0") {} else {}
+  } else if (ctx.target === "openapi-3.0") {}
   if (ctx.external?.uri) {
     const id = ctx.external.registry.get(schema)?.id;
     if (!id)
@@ -22814,7 +22834,7 @@ var literalProcessor = (schema, ctx, json, _params) => {
     if (val === undefined) {
       if (ctx.unrepresentable === "throw") {
         throw new Error("Literal `undefined` cannot be represented in JSON Schema");
-      } else {}
+      }
     } else if (typeof val === "bigint") {
       if (ctx.unrepresentable === "throw") {
         throw new Error("BigInt literals cannot be represented in JSON Schema");

package/dist/mcp/http.d.ts

@@ -0,0 +1,13 @@
+import { DEFAULT_MCP_HTTP_PORT, MCP_NAME } from "./server.js";
+export { DEFAULT_MCP_HTTP_PORT, MCP_NAME };
+export declare function isHttpMode(argv?: string[]): boolean;
+export declare function isStdioMode(argv?: string[]): boolean;
+export declare function resolveHttpPort(argv?: string[]): number;
+export declare function handleMcpHttpRequest(req: Request): Promise<Response>;
+export interface StartHttpServerOptions {
+    port?: number;
+    hostname?: string;
+    log?: (message: string) => void;
+}
+export declare function startHttpServer(options?: StartHttpServerOptions): ReturnType<typeof Bun.serve>;
+//# sourceMappingURL=http.d.ts.map

package/dist/mcp/http.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/mcp/http.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,qBAAqB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE3E,OAAO,EAAE,qBAAqB,EAAE,QAAQ,EAAE,CAAC;AAE3C,wBAAgB,UAAU,CAAC,IAAI,GAAE,MAAM,EAA0B,GAAG,OAAO,CAE1E;AAED,wBAAgB,WAAW,CAAC,IAAI,GAAE,MAAM,EAA0B,GAAG,OAAO,CAE3E;AAED,wBAAgB,eAAe,CAAC,IAAI,GAAE,MAAM,EAA0B,GAAG,MAAM,CAa9E;AAUD,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAiB1E;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACjC;AAED,wBAAgB,eAAe,CAAC,OAAO,GAAE,sBAA2B,GAAG,UAAU,CAAC,OAAO,GAAG,CAAC,KAAK,CAAC,CAclG"}

package/dist/mcp/http.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=http.test.d.ts.map

package/dist/mcp/http.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.test.d.ts","sourceRoot":"","sources":["../../src/mcp/http.test.ts"],"names":[],"mappings":""}

package/dist/mcp/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env bun
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/mcp/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/mcp/index.ts"],"names":[],"mappings":""}