@hasna/evals 0.1.23 → 0.1.25

package/dist/core/assertions-coverage.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=assertions-coverage.test.d.ts.map

package/dist/core/assertions-coverage.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions-coverage.test.d.ts","sourceRoot":"","sources":["../../src/core/assertions-coverage.test.ts"],"names":[],"mappings":""}

package/dist/core/assertions.d.ts

@@ -0,0 +1,18 @@
+import type { Assertion, AssertionResult } from "../types/index.js";
+export declare function sortAssertionsCheapestFirst(assertions: Assertion[]): Assertion[];
+export interface AssertionContext {
+    output: string;
+    durationMs?: number;
+    inputTokens?: number;
+    outputTokens?: number;
+    costUsd?: number;
+    toolCalls?: Array<{
+        name: string;
+        arguments?: Record<string, unknown>;
+    }>;
+}
+export declare function runAssertion(assertion: Assertion, ctx: AssertionContext): Promise<AssertionResult>;
+export declare function runAssertions(assertions: Assertion[], ctx: AssertionContext): Promise<AssertionResult[]>;
+export declare function assertionsPassed(results: AssertionResult[]): boolean;
+export declare function allAssertionsPassed(results: AssertionResult[]): boolean;
+//# sourceMappingURL=assertions.d.ts.map

package/dist/core/assertions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../../src/core/assertions.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AA6BpE,wBAAgB,2BAA2B,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,SAAS,EAAE,CAMhF;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,CAAC;CAC1E;AAED,wBAAsB,YAAY,CAChC,SAAS,EAAE,SAAS,EACpB,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,eAAe,CAAC,CAqB1B;AAED,wBAAsB,aAAa,CACjC,UAAU,EAAE,SAAS,EAAE,EACvB,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,eAAe,EAAE,CAAC,CAyB5B;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,OAAO,CAKpE;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,OAAO,CAEvE"}

package/dist/core/assertions.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=assertions.test.d.ts.map

package/dist/core/assertions.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.test.d.ts","sourceRoot":"","sources":["../../src/core/assertions.test.ts"],"names":[],"mappings":""}

package/dist/core/e2e.test.d.ts

@@ -0,0 +1,2 @@
+export declare function echoInput(input: string): Promise<string>;
+//# sourceMappingURL=e2e.test.d.ts.map

package/dist/core/e2e.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"e2e.test.d.ts","sourceRoot":"","sources":["../../src/core/e2e.test.ts"],"names":[],"mappings":"AAyBA,wBAAsB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAE9D"}

package/dist/core/judge.d.ts

@@ -0,0 +1,13 @@
+import type { JudgeConfig, JudgeResult } from "../types/index.js";
+export declare function runJudge(input: string, output: string, config: JudgeConfig, expected?: string): Promise<JudgeResult>;
+/** One-shot judge: no full eval case, just input/output/rubric */
+export declare function judgeOnce(params: {
+    input: string;
+    output: string;
+    rubric: string;
+    expected?: string;
+    model?: string;
+    provider?: "anthropic" | "openai";
+    apiKey?: string;
+}): Promise<JudgeResult>;
+//# sourceMappingURL=judge.d.ts.map

package/dist/core/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/core/judge.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAW,MAAM,mBAAmB,CAAC;AAsD3E,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,WAAW,EACnB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,WAAW,CAAC,CAmBtB;AA4ED,kEAAkE;AAClE,wBAAsB,SAAS,CAAC,MAAM,EAAE;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,WAAW,GAAG,QAAQ,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,GAAG,OAAO,CAAC,WAAW,CAAC,CAOvB"}

package/dist/core/judge.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=judge.test.d.ts.map

package/dist/core/judge.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.test.d.ts","sourceRoot":"","sources":["../../src/core/judge.test.ts"],"names":[],"mappings":""}

package/dist/core/reporter.d.ts

@@ -0,0 +1,21 @@
+import type { EvalRun, Verdict } from "../types/index.js";
+export declare function printTerminalReport(run: EvalRun): void;
+export declare function toJson(run: EvalRun): string;
+export declare function toMarkdown(run: EvalRun): string;
+export interface RunDiff {
+    regressions: Array<{
+        caseId: string;
+        before: Verdict;
+        after: Verdict;
+    }>;
+    improvements: Array<{
+        caseId: string;
+        before: Verdict;
+        after: Verdict;
+    }>;
+    scoreDelta: number;
+    passRateDelta: number;
+}
+export declare function compareRuns(before: EvalRun, after: EvalRun): RunDiff;
+export declare function printDiffReport(diff: RunDiff): void;
+//# sourceMappingURL=reporter.d.ts.map

package/dist/core/reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/core/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAgB1D,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAoCtD;AAID,wBAAgB,MAAM,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAE3C;AAID,wBAAgB,UAAU,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAkD/C;AAID,MAAM,WAAW,OAAO;IACtB,WAAW,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACxE,YAAY,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAuBpE;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CAcnD"}

package/dist/core/reporter.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=reporter.test.d.ts.map

package/dist/core/reporter.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.test.d.ts","sourceRoot":"","sources":["../../src/core/reporter.test.ts"],"names":[],"mappings":""}

package/dist/core/runner.d.ts

@@ -0,0 +1,4 @@
+import type { AdapterConfig, EvalCase, EvalResult, EvalRun, RunOptions } from "../types/index.js";
+export declare function runEvals(cases: EvalCase[], options: RunOptions): Promise<EvalRun>;
+export declare function runSingleCase(evalCase: EvalCase, adapterConfig: AdapterConfig, skipJudge?: boolean): Promise<EvalResult>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/core/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/core/runner.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EACV,aAAa,EACb,QAAQ,EACR,UAAU,EACV,OAAO,EAEP,UAAU,EAGX,MAAM,mBAAmB,CAAC;AAqI3B,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,QAAQ,EAAE,EACjB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,OAAO,CAAC,CAqClB;AAID,wBAAsB,aAAa,CACjC,QAAQ,EAAE,QAAQ,EAClB,aAAa,EAAE,aAAa,EAC5B,SAAS,UAAQ,GAChB,OAAO,CAAC,UAAU,CAAC,CAKrB"}

package/dist/core/runner.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=runner.test.d.ts.map

package/dist/core/runner.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.test.d.ts","sourceRoot":"","sources":["../../src/core/runner.test.ts"],"names":[],"mappings":""}

package/dist/datasets/loader.d.ts

@@ -0,0 +1,18 @@
+import type { EvalCase } from "../types/index.js";
+export interface LoadOptions {
+    /** Fail hard on malformed lines instead of warning and skipping */
+    strict?: boolean;
+    /** Filter by tags */
+    tags?: string[];
+}
+export interface LoadResult {
+    cases: EvalCase[];
+    warnings: string[];
+    totalLines: number;
+    skipped: number;
+}
+/** Load eval cases from a JSONL or JSON file (or glob pattern) */
+export declare function loadDataset(pathOrGlob: string, opts?: LoadOptions): Promise<LoadResult>;
+/** Stream large JSONL files case-by-case (for very large datasets) */
+export declare function streamDataset(path: string, opts?: LoadOptions): AsyncGenerator<EvalCase>;
+//# sourceMappingURL=loader.d.ts.map

package/dist/datasets/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/datasets/loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAElD,MAAM,WAAW,WAAW;IAC1B,mEAAmE;IACnE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,qBAAqB;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,kEAAkE;AAClE,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAiCjG;AAoFD,sEAAsE;AACtE,wBAAuB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,WAAgB,GAAG,cAAc,CAAC,QAAQ,CAAC,CAyBnG"}

package/dist/datasets/loader.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=loader.test.d.ts.map

package/dist/datasets/loader.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.test.d.ts","sourceRoot":"","sources":["../../src/datasets/loader.test.ts"],"names":[],"mappings":""}

package/dist/db/cloud-sync.d.ts

@@ -0,0 +1,33 @@
+import { PgAdapterAsync } from "./remote-storage.js";
+export declare const CLOUD_TABLES: readonly ["runs", "baselines"];
+type CloudTable = (typeof CLOUD_TABLES)[number];
+export interface SyncResult {
+    table: string;
+    rowsRead: number;
+    rowsWritten: number;
+    errors: string[];
+}
+export interface SyncMeta {
+    table_name: string;
+    last_synced_at: string | null;
+    direction: "push" | "pull";
+}
+export declare function getCloudDatabaseUrl(): string | null;
+export declare function getCloudPg(): Promise<PgAdapterAsync>;
+export declare function runCloudMigrations(remote: PgAdapterAsync): Promise<void>;
+export declare function cloudPush(options?: {
+    tables?: string[];
+}): Promise<SyncResult[]>;
+export declare function cloudPull(options?: {
+    tables?: string[];
+}): Promise<SyncResult[]>;
+export declare function cloudSync(options?: {
+    tables?: string[];
+}): Promise<{
+    pull: SyncResult[];
+    push: SyncResult[];
+}>;
+export declare function getSyncMetaAll(): SyncMeta[];
+export declare function resolveTables(tables?: string[]): CloudTable[];
+export {};
+//# sourceMappingURL=cloud-sync.d.ts.map

package/dist/db/cloud-sync.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloud-sync.d.ts","sourceRoot":"","sources":["../../src/db/cloud-sync.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,eAAO,MAAM,YAAY,gCAAiC,CAAC;AAE3D,KAAK,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAQhD,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,QAAQ;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;CAC5B;AAED,wBAAgB,mBAAmB,IAAI,MAAM,GAAG,IAAI,CAKnD;AAED,wBAAsB,UAAU,IAAI,OAAO,CAAC,cAAc,CAAC,CAM1D;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAE9E;AAED,wBAAsB,SAAS,CAAC,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAYtF;AAED,wBAAsB,SAAS,CAAC,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAYtF;AAED,wBAAsB,SAAS,CAAC,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAAC,IAAI,EAAE,UAAU,EAAE,CAAA;CAAE,CAAC,CAIpH;AAED,wBAAgB,cAAc,IAAI,QAAQ,EAAE,CAI3C;AAED,wBAAgB,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAO7D"}

package/dist/db/pg-migrations.d.ts

@@ -0,0 +1,5 @@
+/**
+ * PostgreSQL migrations for open-evals cloud sync.
+ */
+export declare const PG_MIGRATIONS: string[];
+//# sourceMappingURL=pg-migrations.d.ts.map

package/dist/db/pg-migrations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pg-migrations.d.ts","sourceRoot":"","sources":["../../src/db/pg-migrations.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,eAAO,MAAM,aAAa,EAAE,MAAM,EAgBjC,CAAC"}

package/dist/db/remote-storage.d.ts

@@ -0,0 +1,10 @@
+export declare class PgAdapterAsync {
+    private readonly pool;
+    constructor(connectionString: string);
+    run(sql: string, ...params: unknown[]): Promise<{
+        changes: number;
+    }>;
+    all(sql: string, ...params: unknown[]): Promise<unknown[]>;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=remote-storage.d.ts.map

package/dist/db/remote-storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-storage.d.ts","sourceRoot":"","sources":["../../src/db/remote-storage.ts"],"names":[],"mappings":"AAmBA,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAO;gBAEhB,gBAAgB,EAAE,MAAM;IAI9B,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAKpE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAK1D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}

package/dist/db/store.d.ts

@@ -0,0 +1,17 @@
+import { Database } from "bun:sqlite";
+import type { EvalRun } from "../types/index.js";
+export declare function getDatabase(): Database;
+export declare function closeDatabase(): void;
+export declare function saveRun(run: EvalRun): void;
+export declare function getRun(id: string): EvalRun | null;
+export declare function listRuns(limit?: number, dataset?: string): EvalRun[];
+export declare function deleteRun(id: string): void;
+export declare function setBaseline(name: string, runId: string): void;
+export declare function getBaseline(name: string): EvalRun | null;
+export declare function listBaselines(): Array<{
+    name: string;
+    runId: string;
+    createdAt: string;
+}>;
+export declare function clearBaseline(name: string): void;
+//# sourceMappingURL=store.d.ts.map

package/dist/db/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/db/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAItC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAQjD,wBAAgB,WAAW,IAAI,QAAQ,CAUtC;AAED,wBAAgB,aAAa,IAAI,IAAI,CAGpC;AA6BD,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAa1C;AAED,wBAAgB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAIjD;AAED,wBAAgB,QAAQ,CAAC,KAAK,SAAK,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,EAAE,CAMhE;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAE1C;AAID,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAI7D;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAKxD;AAED,wBAAgB,aAAa,IAAI,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAKzF;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAEhD"}

package/dist/db/store.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=store.test.d.ts.map

package/dist/db/store.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.test.d.ts","sourceRoot":"","sources":["../../src/db/store.test.ts"],"names":[],"mappings":""}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+export * from "./types/index.js";
+export * from "./core/runner.js";
+export * from "./core/assertions.js";
+export * from "./core/judge.js";
+export * from "./core/reporter.js";
+export * from "./datasets/loader.js";
+export * from "./db/store.js";
+export * from "./db/pg-migrations.js";
+export * from "./db/remote-storage.js";
+export * from "./db/cloud-sync.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,sBAAsB,CAAC;AACrC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC"}