@cogitator-ai/evals 0.1.0

package/dist/metrics/statistical.js

@@ -0,0 +1,85 @@
+import { aggregate, mean } from '../stats';
+function createStatisticalFn(name, fn) {
+    const statFn = fn;
+    statFn.metricName = name;
+    return statFn;
+}
+export function latency() {
+    return createStatisticalFn('latency', (results) => {
+        const durations = results.map((r) => r.duration);
+        const stats = aggregate(durations);
+        return {
+            name: 'latency',
+            score: 0,
+            metadata: {
+                p50: stats.p50,
+                p95: stats.p95,
+                p99: stats.p99,
+                mean: stats.mean,
+                median: stats.median,
+                min: stats.min,
+                max: stats.max,
+            },
+        };
+    });
+}
+export function cost() {
+    return createStatisticalFn('cost', (results) => {
+        const costs = results.filter((r) => r.usage).map((r) => r.usage.cost);
+        if (costs.length === 0) {
+            return {
+                name: 'cost',
+                score: 0,
+                metadata: { total: 0, mean: 0, median: 0, min: 0, max: 0 },
+            };
+        }
+        const stats = aggregate(costs);
+        let total = 0;
+        for (let i = 0; i < costs.length; i++) {
+            total += costs[i];
+        }
+        return {
+            name: 'cost',
+            score: 0,
+            metadata: {
+                total,
+                mean: stats.mean,
+                median: stats.median,
+                min: stats.min,
+                max: stats.max,
+            },
+        };
+    });
+}
+export function tokenUsage() {
+    return createStatisticalFn('tokenUsage', (results) => {
+        const withUsage = results.filter((r) => r.usage);
+        if (withUsage.length === 0) {
+            return {
+                name: 'tokenUsage',
+                score: 0,
+                metadata: { totalInput: 0, totalOutput: 0, totalTokens: 0, meanInput: 0, meanOutput: 0 },
+            };
+        }
+        const inputTokens = withUsage.map((r) => r.usage.inputTokens);
+        const outputTokens = withUsage.map((r) => r.usage.outputTokens);
+        let totalInput = 0;
+        let totalOutput = 0;
+        for (let i = 0; i < withUsage.length; i++) {
+            totalInput += inputTokens[i];
+            totalOutput += outputTokens[i];
+        }
+        return {
+            name: 'tokenUsage',
+            score: 0,
+            metadata: {
+                totalInput,
+                totalOutput,
+                totalTokens: totalInput + totalOutput,
+                meanInput: mean(inputTokens),
+                meanOutput: mean(outputTokens),
+            },
+        };
+    });
+}
+//# sourceMappingURL=statistical.js.map

package/dist/metrics/statistical.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,EAA8C;IAE9C,MAAM,MAAM,GAAG,EAAyB,CAAC;IACzC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,OAAO;IACrB,OAAO,mBAAmB,CAAC,SAAS,EAAE,CAAC,OAAyB,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,IAAI;IAClB,OAAO,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAyB,EAAE,EAAE;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,IAAI,CAAC,CAAC;QAEvE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;aAC3D,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE;gBACR,KAAK;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,OAAO,mBAAmB,CAAC,YAAY,EAAE,CAAC,OAAyB,EAAE,EAAE;QACrE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAEjD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;aACzF,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QAEjE,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,UAAU,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;YAC7B,WAAW,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC;QAED,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE;gBACR,UAAU;gBACV,WAAW;gBACX,WAAW,EAAE,UAAU,GAAG,WAAW;gBACrC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;aAC/B;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/metrics/types.d.ts

@@ -0,0 +1,31 @@
+import type { EvalCase } from '../schema';
+export interface MetricScore {
+    name: string;
+    score: number;
+    details?: string;
+    metadata?: Record<string, unknown>;
+}
+export interface EvalCaseResult {
+    case: EvalCase;
+    output: string;
+    duration: number;
+    usage?: {
+        inputTokens: number;
+        outputTokens: number;
+        totalTokens: number;
+        cost: number;
+        duration: number;
+    };
+    toolCalls?: readonly {
+        id: string;
+        name: string;
+        arguments: Record<string, unknown>;
+    }[];
+}
+export type MetricFn = ((result: EvalCaseResult) => Promise<MetricScore>) & {
+    metricName: string;
+};
+export type StatisticalMetricFn = ((results: EvalCaseResult[]) => MetricScore) & {
+    metricName: string;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/metrics/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,CAAC,EAAE,SAAS;QACnB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,EAAE,CAAC;CACL;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,WAAW,CAAC,GAAG;IAC/E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC"}

package/dist/metrics/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/metrics/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":""}

package/dist/reporters/ci.d.ts

@@ -0,0 +1,3 @@
+import type { EvalSuiteResult } from './index';
+export declare function ciReport(result: EvalSuiteResult): void;
+//# sourceMappingURL=ci.d.ts.map

package/dist/reporters/ci.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAM/C,wBAAgB,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAqBtD"}

package/dist/reporters/ci.js

@@ -0,0 +1,21 @@
+const GREEN = '\x1b[32m';
+const RED = '\x1b[31m';
+const RESET = '\x1b[0m';
+export function ciReport(result) {
+    const passed = result.assertions.filter((a) => a.passed).length;
+    const failed = result.assertions.filter((a) => !a.passed).length;
+    console.log(`Eval: ${result.stats.total} cases | ${result.stats.duration}ms | $${result.stats.cost}`);
+    for (const a of result.assertions) {
+        if (a.passed) {
+            console.log(`  ${GREEN}PASS${RESET} ${a.name}`);
+        }
+        else {
+            console.log(`  ${RED}FAIL${RESET} ${a.name}: ${a.message}`);
+        }
+    }
+    console.log(`Result: ${passed} passed, ${failed} failed`);
+    if (failed > 0) {
+        process.exit(1);
+    }
+}
+//# sourceMappingURL=ci.js.map

package/dist/reporters/ci.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ci.js","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,MAAM,UAAU,QAAQ,CAAC,MAAuB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,OAAO,CAAC,GAAG,CACT,SAAS,MAAM,CAAC,KAAK,CAAC,KAAK,YAAY,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CACzF,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,YAAY,MAAM,SAAS,CAAC,CAAC;IAE1D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}

package/dist/reporters/console.d.ts

@@ -0,0 +1,3 @@
+import type { EvalSuiteResult } from './index';
+export declare function consoleReport(result: EvalSuiteResult): void;
+//# sourceMappingURL=console.d.ts.map

package/dist/reporters/console.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"console.d.ts","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAgB/C,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CA2C3D"}

package/dist/reporters/console.js

@@ -0,0 +1,46 @@
+const GREEN = '\x1b[32m';
+const RED = '\x1b[31m';
+const BOLD = '\x1b[1m';
+const DIM = '\x1b[2m';
+const RESET = '\x1b[0m';
+function pad(str, len) {
+    return str.length >= len ? str : str + ' '.repeat(len - str.length);
+}
+function fmt(n) {
+    return Number.isInteger(n) ? String(n) : n.toFixed(4);
+}
+export function consoleReport(result) {
+    const metrics = Object.values(result.aggregated);
+    if (metrics.length > 0) {
+        const cols = { metric: 14, mean: 10, median: 10, p95: 10, min: 10, max: 10 };
+        const header = pad('Metric', cols.metric) +
+            pad('Mean', cols.mean) +
+            pad('Median', cols.median) +
+            pad('P95', cols.p95) +
+            pad('Min', cols.min) +
+            pad('Max', cols.max);
+        console.log(`\n${BOLD}${header}${RESET}`);
+        console.log(DIM + '─'.repeat(header.length) + RESET);
+        for (const m of metrics) {
+            const row = pad(m.name, cols.metric) +
+                pad(fmt(m.mean), cols.mean) +
+                pad(fmt(m.median), cols.median) +
+                pad(fmt(m.p95), cols.p95) +
+                pad(fmt(m.min), cols.min) +
+                pad(fmt(m.max), cols.max);
+            console.log(row);
+        }
+    }
+    if (result.assertions.length > 0) {
+        console.log(`\n${BOLD}Assertions${RESET}`);
+        for (const a of result.assertions) {
+            const icon = a.passed ? `${GREEN}✓${RESET}` : `${RED}✗${RESET}`;
+            const color = a.passed ? GREEN : RED;
+            console.log(`  ${icon} ${color}${a.name}${RESET} ${DIM}${a.message}${RESET}`);
+        }
+    }
+    const passed = result.assertions.filter((a) => a.passed).length;
+    const failed = result.assertions.filter((a) => !a.passed).length;
+    console.log(`\n${BOLD}Summary${RESET}: ${result.stats.total} cases | ${result.stats.duration}ms | $${result.stats.cost} | ${GREEN}${passed} passed${RESET} ${failed > 0 ? `${RED}${failed} failed${RESET}` : ''}`);
+}
+//# sourceMappingURL=console.js.map

package/dist/reporters/console.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"console.js","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,SAAS,GAAG,CAAC,GAAW,EAAE,GAAW;IACnC,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;QAC7E,MAAM,MAAM,GACV,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC;YACtB,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAEvB,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QAErD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,GAAG,GACP,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;gBACxB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,aAAa,KAAK,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,UAAU,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,KAAK,YAAY,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,GAAG,MAAM,UAAU,KAAK,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACtM,CAAC;AACJ,CAAC"}

package/dist/reporters/csv.d.ts

@@ -0,0 +1,5 @@
+import type { EvalSuiteResult } from './index';
+export declare function csvReport(result: EvalSuiteResult, options: {
+    path: string;
+}): void;
+//# sourceMappingURL=csv.d.ts.map

package/dist/reporters/csv.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAS/C,wBAAgB,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAyBlF"}

package/dist/reporters/csv.js

@@ -0,0 +1,31 @@
+import { writeFileSync } from 'node:fs';
+function escapeField(value) {
+    if (value.includes(',') || value.includes('"') || value.includes('\n')) {
+        return `"${value.replace(/"/g, '""')}"`;
+    }
+    return value;
+}
+export function csvReport(result, options) {
+    const metricNames = new Set();
+    for (const r of result.results) {
+        for (const s of r.scores) {
+            metricNames.add(s.name);
+        }
+    }
+    const metrics = [...metricNames];
+    const headers = ['input', 'expected', 'output', 'duration', ...metrics];
+    const lines = [headers.join(',')];
+    for (const r of result.results) {
+        const scoreMap = new Map(r.scores.map((s) => [s.name, s.score]));
+        const row = [
+            escapeField(r.case.input),
+            escapeField(r.case.expected ?? ''),
+            escapeField(r.output),
+            String(r.duration),
+            ...metrics.map((m) => String(scoreMap.get(m) ?? '')),
+        ];
+        lines.push(row.join(','));
+    }
+    writeFileSync(options.path, lines.join('\n') + '\n', 'utf-8');
+}
+//# sourceMappingURL=csv.js.map

package/dist/reporters/csv.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACvE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAuB,EAAE,OAAyB;IAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;IAEjC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,CAAC;IACxE,MAAM,KAAK,GAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG;YACV,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAClC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;YACrB,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SACrD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC"}

package/dist/reporters/index.d.ts

@@ -0,0 +1,50 @@
+export interface AggregatedMetric {
+    name: string;
+    mean: number;
+    median: number;
+    min: number;
+    max: number;
+    stdDev: number;
+    p50: number;
+    p95: number;
+    p99: number;
+}
+export interface AssertionResult {
+    name: string;
+    passed: boolean;
+    message: string;
+    actual?: number;
+    expected?: number;
+}
+export interface EvalSuiteResult {
+    results: Array<{
+        case: {
+            input: string;
+            expected?: string;
+        };
+        output: string;
+        duration: number;
+        scores: Array<{
+            name: string;
+            score: number;
+            details?: string;
+        }>;
+    }>;
+    aggregated: Record<string, AggregatedMetric>;
+    assertions: AssertionResult[];
+    stats: {
+        total: number;
+        duration: number;
+        cost: number;
+    };
+}
+export type ReporterType = 'console' | 'json' | 'csv' | 'ci';
+export type ReporterOptions = {
+    path?: string;
+};
+export declare function report(result: EvalSuiteResult, type: ReporterType | ReporterType[], options?: ReporterOptions): void;
+export { consoleReport } from './console';
+export { jsonReport } from './json';
+export { csvReport } from './csv';
+export { ciReport } from './ci';
+//# sourceMappingURL=index.d.ts.map

package/dist/reporters/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3C,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClE,CAAC,CAAC;IACH,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC;AAC7D,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEhD,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAmBN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}

package/dist/reporters/index.js

@@ -0,0 +1,28 @@
+import { consoleReport } from './console';
+import { jsonReport } from './json';
+import { csvReport } from './csv';
+import { ciReport } from './ci';
+export function report(result, type, options) {
+    const types = Array.isArray(type) ? type : [type];
+    for (const t of types) {
+        switch (t) {
+            case 'console':
+                consoleReport(result);
+                break;
+            case 'json':
+                jsonReport(result, { path: options?.path ?? 'eval-report.json' });
+                break;
+            case 'csv':
+                csvReport(result, { path: options?.path ?? 'eval-report.csv' });
+                break;
+            case 'ci':
+                ciReport(result);
+                break;
+        }
+    }
+}
+export { consoleReport } from './console';
+export { jsonReport } from './json';
+export { csvReport } from './csv';
+export { ciReport } from './ci';
+//# sourceMappingURL=index.js.map

package/dist/reporters/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAqChC,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAElD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,kBAAkB,EAAE,CAAC,CAAC;gBAClE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,iBAAiB,EAAE,CAAC,CAAC;gBAChE,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}

package/dist/reporters/json.d.ts

@@ -0,0 +1,5 @@
+import type { EvalSuiteResult } from './index';
+export declare function jsonReport(result: EvalSuiteResult, options: {
+    path: string;
+}): void;
+//# sourceMappingURL=json.d.ts.map

package/dist/reporters/json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/reporters/json.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C,wBAAgB,UAAU,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAEnF"}

package/dist/reporters/json.js

@@ -0,0 +1,5 @@
+import { writeFileSync } from 'node:fs';
+export function jsonReport(result, options) {
+    writeFileSync(options.path, JSON.stringify(result, null, 2), 'utf-8');
+}
+//# sourceMappingURL=json.js.map

package/dist/reporters/json.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/reporters/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,MAAM,UAAU,UAAU,CAAC,MAAuB,EAAE,OAAyB;IAC3E,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACxE,CAAC"}

package/dist/schema.d.ts

@@ -0,0 +1,29 @@
+import { z } from 'zod';
+export declare const EvalCaseSchema: z.ZodObject<{
+    input: z.ZodString;
+    expected: z.ZodOptional<z.ZodString>;
+    context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strip>;
+export declare const EvalSuiteConfigSchema: z.ZodObject<{
+    concurrency: z.ZodDefault<z.ZodNumber>;
+    timeout: z.ZodDefault<z.ZodNumber>;
+    retries: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+export declare const JudgeConfigSchema: z.ZodObject<{
+    model: z.ZodString;
+    temperature: z.ZodDefault<z.ZodNumber>;
+    maxTokens: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+export declare const EvalComparisonConfigSchema: z.ZodObject<{
+    concurrency: z.ZodDefault<z.ZodNumber>;
+    timeout: z.ZodDefault<z.ZodNumber>;
+    retries: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+export type EvalCase = z.output<typeof EvalCaseSchema>;
+export type EvalCaseInput = z.input<typeof EvalCaseSchema>;
+export type EvalSuiteConfig = z.output<typeof EvalSuiteConfigSchema>;
+export type EvalSuiteConfigInput = z.input<typeof EvalSuiteConfigSchema>;
+export type JudgeConfig = z.output<typeof JudgeConfigSchema>;
+export type EvalComparisonConfig = z.output<typeof EvalComparisonConfigSchema>;
+//# sourceMappingURL=schema.d.ts.map

package/dist/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;iBAI5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;iBAIrC,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}

package/dist/schema.js

@@ -0,0 +1,23 @@
+import { z } from 'zod';
+export const EvalCaseSchema = z.object({
+    input: z.string(),
+    expected: z.string().optional(),
+    context: z.record(z.string(), z.unknown()).optional(),
+    metadata: z.record(z.string(), z.unknown()).optional(),
+});
+export const EvalSuiteConfigSchema = z.object({
+    concurrency: z.number().int().min(1).default(5),
+    timeout: z.number().int().min(1000).default(30000),
+    retries: z.number().int().min(0).max(10).default(0),
+});
+export const JudgeConfigSchema = z.object({
+    model: z.string(),
+    temperature: z.number().default(0),
+    maxTokens: z.number().int().positive().optional(),
+});
+export const EvalComparisonConfigSchema = z.object({
+    concurrency: z.number().int().min(1).default(5),
+    timeout: z.number().int().min(1000).default(30000),
+    retries: z.number().int().min(0).max(10).default(0),
+});
+//# sourceMappingURL=schema.js.map

package/dist/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAClD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC"}

package/dist/stats/index.d.ts

@@ -0,0 +1,6 @@
+export { percentile, mean, median, stdDev, aggregate } from './percentiles';
+export { pairedTTest } from './t-test';
+export type { TTestResult } from './t-test';
+export { mcnemarsTest } from './mcnemar';
+export type { McNemarResult } from './mcnemar';
+//# sourceMappingURL=index.d.ts.map

package/dist/stats/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stats/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,YAAY,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC"}

package/dist/stats/index.js

@@ -0,0 +1,4 @@
+export { percentile, mean, median, stdDev, aggregate } from './percentiles';
+export { pairedTTest } from './t-test';
+export { mcnemarsTest } from './mcnemar';
+//# sourceMappingURL=index.js.map

package/dist/stats/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/stats/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEvC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC"}

package/dist/stats/mcnemar.d.ts

@@ -0,0 +1,7 @@
+export interface McNemarResult {
+    chiSquare: number;
+    pValue: number;
+    significant: boolean;
+}
+export declare function mcnemarsTest(pairsACorrect_BIncorrect: number, pairsAIncorrect_BCorrect: number): McNemarResult;
+//# sourceMappingURL=mcnemar.d.ts.map

package/dist/stats/mcnemar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcnemar.d.ts","sourceRoot":"","sources":["../../src/stats/mcnemar.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;CACtB;AAuBD,wBAAgB,YAAY,CAC1B,wBAAwB,EAAE,MAAM,EAChC,wBAAwB,EAAE,MAAM,GAC/B,aAAa,CAiBf"}

package/dist/stats/mcnemar.js

@@ -0,0 +1,34 @@
+function erfc(x) {
+    const a1 = 0.254829592;
+    const a2 = -0.284496736;
+    const a3 = 1.421413741;
+    const a4 = -1.453152027;
+    const a5 = 1.061405429;
+    const p = 0.3275911;
+    const sign = x < 0 ? -1 : 1;
+    const absX = Math.abs(x);
+    const t = 1.0 / (1.0 + p * absX);
+    const y = 1.0 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-absX * absX);
+    return 1.0 - sign * y;
+}
+function chiSquareSurvival(x) {
+    if (x <= 0)
+        return 1;
+    return erfc(Math.sqrt(x / 2));
+}
+export function mcnemarsTest(pairsACorrect_BIncorrect, pairsAIncorrect_BCorrect) {
+    const b = pairsACorrect_BIncorrect;
+    const c = pairsAIncorrect_BCorrect;
+    if (b + c === 0) {
+        return { chiSquare: 0, pValue: 1, significant: false };
+    }
+    const diff = Math.abs(b - c) - 1;
+    const chiSquare = (diff * diff) / (b + c);
+    const pValue = chiSquareSurvival(chiSquare);
+    return {
+        chiSquare,
+        pValue,
+        significant: pValue < 0.05,
+    };
+}
+//# sourceMappingURL=mcnemar.js.map

package/dist/stats/mcnemar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcnemar.js","sourceRoot":"","sources":["../../src/stats/mcnemar.ts"],"names":[],"mappings":"AAMA,SAAS,IAAI,CAAC,CAAS;IACrB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,CAAC,GAAG,SAAS,CAAC;IAEpB,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IAE5F,OAAO,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,iBAAiB,CAAC,CAAS;IAClC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,wBAAgC,EAChC,wBAAgC;IAEhC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IACnC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IAEnC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,SAAS,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE5C,OAAO;QACL,SAAS;QACT,MAAM;QACN,WAAW,EAAE,MAAM,GAAG,IAAI;KAC3B,CAAC;AACJ,CAAC"}

package/dist/stats/percentiles.d.ts

@@ -0,0 +1,15 @@
+export declare function mean(values: number[]): number;
+export declare function stdDev(values: number[]): number;
+export declare function percentile(values: number[], p: number): number;
+export declare function median(values: number[]): number;
+export declare function aggregate(values: number[]): {
+    mean: number;
+    median: number;
+    min: number;
+    max: number;
+    stdDev: number;
+    p50: number;
+    p95: number;
+    p99: number;
+};
+//# sourceMappingURL=percentiles.d.ts.map

package/dist/stats/percentiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"percentiles.d.ts","sourceRoot":"","sources":["../../src/stats/percentiles.ts"],"names":[],"mappings":"AAAA,wBAAgB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAO7C;AAED,wBAAgB,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAS/C;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAa9D;AAED,wBAAgB,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAE/C;AAED,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;CACb,CAiBA"}

package/dist/stats/percentiles.js

@@ -0,0 +1,54 @@
+export function mean(values) {
+    if (values.length === 0)
+        return 0;
+    let sum = 0;
+    for (let i = 0; i < values.length; i++) {
+        sum += values[i];
+    }
+    return sum / values.length;
+}
+export function stdDev(values) {
+    if (values.length < 2)
+        return 0;
+    const m = mean(values);
+    let sumSq = 0;
+    for (let i = 0; i < values.length; i++) {
+        const d = values[i] - m;
+        sumSq += d * d;
+    }
+    return Math.sqrt(sumSq / (values.length - 1));
+}
+export function percentile(values, p) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    if (sorted.length === 1)
+        return sorted[0];
+    const rank = p * (sorted.length - 1);
+    const lower = Math.floor(rank);
+    const upper = Math.ceil(rank);
+    if (lower === upper)
+        return sorted[lower];
+    const fraction = rank - lower;
+    return sorted[lower] + fraction * (sorted[upper] - sorted[lower]);
+}
+export function median(values) {
+    return percentile(values, 0.5);
+}
+export function aggregate(values) {
+    if (values.length === 0) {
+        return { mean: 0, median: 0, min: 0, max: 0, stdDev: 0, p50: 0, p95: 0, p99: 0 };
+    }
+    const sorted = [...values].sort((a, b) => a - b);
+    return {
+        mean: mean(values),
+        median: percentile(values, 0.5),
+        min: sorted[0],
+        max: sorted[sorted.length - 1],
+        stdDev: stdDev(values),
+        p50: percentile(values, 0.5),
+        p95: percentile(values, 0.95),
+        p99: percentile(values, 0.99),
+    };
+}
+//# sourceMappingURL=percentiles.js.map

package/dist/stats/percentiles.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"percentiles.js","sourceRoot":"","sources":["../../src/stats/percentiles.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,IAAI,CAAC,MAAgB;IACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,MAAgB;IACrC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAgB,EAAE,CAAS;IACpD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IAE1C,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE9B,IAAI,KAAK,KAAK,KAAK;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IAE1C,MAAM,QAAQ,GAAG,IAAI,GAAG,KAAK,CAAC;IAC9B,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,MAAgB;IACrC,OAAO,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAgB;IAUxC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IACnF,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjD,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;QAClB,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;QAC/B,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;QACd,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QAC9B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;QACtB,GAAG,EAAE,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;QAC5B,GAAG,EAAE,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC;QAC7B,GAAG,EAAE,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC;KAC9B,CAAC;AACJ,CAAC"}

package/dist/stats/t-test.d.ts

@@ -0,0 +1,9 @@
+export interface TTestResult {
+    tStatistic: number;
+    degreesOfFreedom: number;
+    pValue: number;
+    significant: boolean;
+    confidenceInterval: [number, number];
+}
+export declare function pairedTTest(samplesA: number[], samplesB: number[]): TTestResult;
+//# sourceMappingURL=t-test.d.ts.map

package/dist/stats/t-test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"t-test.d.ts","sourceRoot":"","sources":["../../src/stats/t-test.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,kBAAkB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC;AAkGD,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,WAAW,CA4C/E"}