@cogitator-ai/evals 0.1.0

package/dist/stats/t-test.js

@@ -0,0 +1,129 @@
+import { mean, stdDev } from './percentiles';
+function lnGamma(z) {
+    const g = 7;
+    const coefficients = [
+        0.99999999999980993, 676.5203681218851, -1259.1392167224028, 771.32342877765313,
+        -176.61502916214059, 12.507343278686905, -0.13857109526572012, 9.9843695780195716e-6,
+        1.5056327351493116e-7,
+    ];
+    if (z < 0.5) {
+        return Math.log(Math.PI / Math.sin(Math.PI * z)) - lnGamma(1 - z);
+    }
+    z -= 1;
+    let x = coefficients[0];
+    for (let i = 1; i < g + 2; i++) {
+        x += coefficients[i] / (z + i);
+    }
+    const t = z + g + 0.5;
+    return 0.5 * Math.log(2 * Math.PI) + (z + 0.5) * Math.log(t) - t + Math.log(x);
+}
+function betaCf(x, a, b) {
+    const qab = a + b;
+    const qap = a + 1;
+    const qam = a - 1;
+    let c = 1.0;
+    let d = 1.0 - (qab * x) / qap;
+    if (Math.abs(d) < 1e-30)
+        d = 1e-30;
+    d = 1.0 / d;
+    let h = d;
+    for (let m = 1; m <= 200; m++) {
+        const m2 = 2 * m;
+        let aa = (m * (b - m) * x) / ((qam + m2) * (a + m2));
+        d = 1.0 + aa * d;
+        if (Math.abs(d) < 1e-30)
+            d = 1e-30;
+        c = 1.0 + aa / c;
+        if (Math.abs(c) < 1e-30)
+            c = 1e-30;
+        d = 1.0 / d;
+        h *= d * c;
+        aa = (-(a + m) * (qab + m) * x) / ((a + m2) * (qap + m2));
+        d = 1.0 + aa * d;
+        if (Math.abs(d) < 1e-30)
+            d = 1e-30;
+        c = 1.0 + aa / c;
+        if (Math.abs(c) < 1e-30)
+            c = 1e-30;
+        d = 1.0 / d;
+        const del = d * c;
+        h *= del;
+        if (Math.abs(del - 1.0) < 1e-14)
+            break;
+    }
+    return h;
+}
+function regularizedIncompleteBeta(x, a, b) {
+    if (x <= 0)
+        return 0;
+    if (x >= 1)
+        return 1;
+    if (x > (a + 1) / (a + b + 2)) {
+        return 1.0 - regularizedIncompleteBeta(1.0 - x, b, a);
+    }
+    const lnBeta = lnGamma(a) + lnGamma(b) - lnGamma(a + b);
+    const front = Math.exp(a * Math.log(x) + b * Math.log(1 - x) - lnBeta);
+    return (front * betaCf(x, a, b)) / a;
+}
+function tDistCdf(t, df) {
+    const x = df / (df + t * t);
+    const ibeta = regularizedIncompleteBeta(x, df / 2, 0.5);
+    return 0.5 * (1 + Math.sign(t) * (1 - ibeta));
+}
+function tDistQuantile(p, df) {
+    if (p <= 0)
+        return -Infinity;
+    if (p >= 1)
+        return Infinity;
+    let lo = -100;
+    let hi = 100;
+    for (let i = 0; i < 100; i++) {
+        const mid = (lo + hi) / 2;
+        if (tDistCdf(mid, df) < p) {
+            lo = mid;
+        }
+        else {
+            hi = mid;
+        }
+    }
+    return (lo + hi) / 2;
+}
+export function pairedTTest(samplesA, samplesB) {
+    if (samplesA.length !== samplesB.length) {
+        throw new Error('Sample arrays must have equal length');
+    }
+    if (samplesA.length < 2) {
+        throw new Error('Need at least 2 paired samples');
+    }
+    const n = samplesA.length;
+    const diffs = new Array(n);
+    for (let i = 0; i < n; i++) {
+        diffs[i] = samplesA[i] - samplesB[i];
+    }
+    const meanD = mean(diffs);
+    const sdD = stdDev(diffs);
+    const df = n - 1;
+    if (sdD === 0) {
+        return {
+            tStatistic: meanD === 0 ? 0 : Infinity,
+            degreesOfFreedom: df,
+            pValue: meanD === 0 ? 1 : 0,
+            significant: meanD !== 0,
+            confidenceInterval: [meanD, meanD],
+        };
+    }
+    const se = sdD / Math.sqrt(n);
+    const tStat = meanD / se;
+    const cdfVal = tDistCdf(Math.abs(tStat), df);
+    const pValue = 2 * (1 - cdfVal);
+    const tCritical = tDistQuantile(0.975, df);
+    const ci = [meanD - tCritical * se, meanD + tCritical * se];
+    return {
+        tStatistic: tStat,
+        degreesOfFreedom: df,
+        pValue,
+        significant: pValue < 0.05,
+        confidenceInterval: ci,
+    };
+}
+//# sourceMappingURL=t-test.js.map

package/dist/stats/t-test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"t-test.js","sourceRoot":"","sources":["../../src/stats/t-test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAU7C,SAAS,OAAO,CAAC,CAAS;IACxB,MAAM,CAAC,GAAG,CAAC,CAAC;IACZ,MAAM,YAAY,GAAG;QACnB,mBAAmB,EAAE,iBAAiB,EAAE,CAAC,kBAAkB,EAAE,kBAAkB;QAC/E,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,mBAAmB,EAAE,qBAAqB;QACpF,qBAAqB;KACtB,CAAC;IAEF,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACZ,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,CAAC,IAAI,CAAC,CAAC;IACP,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACtB,OAAO,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,SAAS,MAAM,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS;IAC7C,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAClB,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAClB,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAElB,IAAI,CAAC,GAAG,GAAG,CAAC;IACZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;IAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK;QAAE,CAAC,GAAG,KAAK,CAAC;IACnC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,CAAC,GAAG,CAAC,CAAC;IAEV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjB,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrD,CAAC,GAAG,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;QACjB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK;YAAE,CAAC,GAAG,KAAK,CAAC;QACnC,CAAC,GAAG,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;QACjB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK;YAAE,CAAC,GAAG,KAAK,CAAC;QACnC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;QACZ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEX,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC,GAAG,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;QACjB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK;YAAE,CAAC,GAAG,KAAK,CAAC;QACnC,CAAC,GAAG,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;QACjB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK;YAAE,CAAC,GAAG,KAAK,CAAC;QACnC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;QACZ,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC,IAAI,GAAG,CAAC;QAET,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,KAAK;YAAE,MAAM;IACzC,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,yBAAyB,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS;IAChE,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACrB,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAErB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAC9B,OAAO,GAAG,GAAG,yBAAyB,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IAEvE,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,EAAU;IACrC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,yBAAyB,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;IACxD,OAAO,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,aAAa,CAAC,CAAS,EAAE,EAAU;IAC1C,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,QAAQ,CAAC;IAC7B,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAE5B,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;IACd,IAAI,EAAE,GAAG,GAAG,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,EAAE,GAAG,GAAG,CAAC;QACX,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,GAAG,CAAC;QACX,CAAC;IACH,CAAC;IACD,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,QAAkB;IAChE,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC1B,MAAM,KAAK,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAEjB,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;QACd,OAAO;YACL,UAAU,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ;YACtC,gBAAgB,EAAE,EAAE;YACpB,MAAM,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3B,WAAW,EAAE,KAAK,KAAK,CAAC;YACxB,kBAAkB,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC;SACnC,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,KAAK,GAAG,EAAE,CAAC;IAEzB,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IAEhC,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC3C,MAAM,EAAE,GAAqB,CAAC,KAAK,GAAG,SAAS,GAAG,EAAE,EAAE,KAAK,GAAG,SAAS,GAAG,EAAE,CAAC,CAAC;IAE9E,OAAO;QACL,UAAU,EAAE,KAAK;QACjB,gBAAgB,EAAE,EAAE;QACpB,MAAM;QACN,WAAW,EAAE,MAAM,GAAG,IAAI;QAC1B,kBAAkB,EAAE,EAAE;KACvB,CAAC;AACJ,CAAC"}

package/dist/tools.d.ts

@@ -0,0 +1,16 @@
+import { z } from 'zod';
+import type { EvalSuite } from './eval-suite';
+export interface EvalTool<TParams = unknown> {
+    name: string;
+    description: string;
+    parameters: z.ZodType<TParams>;
+    execute: (params: TParams) => Promise<unknown>;
+}
+declare const RunEvalParamsSchema: z.ZodObject<{
+    maxCases: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+type RunEvalParams = z.infer<typeof RunEvalParamsSchema>;
+export declare function createRunEvalTool(suite: EvalSuite): EvalTool<RunEvalParams>;
+export declare function evalTools(suite: EvalSuite): [EvalTool<RunEvalParams>];
+export {};
+//# sourceMappingURL=tools.d.ts.map

package/dist/tools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAmB,MAAM,cAAc,CAAC;AAE/D,MAAM,WAAW,QAAQ,CAAC,OAAO,GAAG,OAAO;IACzC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,QAAA,MAAM,mBAAmB;;iBAEvB,CAAC;AAEH,KAAK,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAwCzD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,SAAS,GAAG,QAAQ,CAAC,aAAa,CAAC,CAc3E;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAErE"}

package/dist/tools.js

@@ -0,0 +1,58 @@
+import { z } from 'zod';
+const RunEvalParamsSchema = z.object({
+    maxCases: z.number().int().positive().optional(),
+});
+function buildSummary(result, maxCases) {
+    const capped = maxCases ? result.results.slice(0, maxCases) : result.results;
+    const total = capped.length;
+    const duration = maxCases
+        ? capped.reduce((sum, r) => sum + r.duration, 0)
+        : result.stats.duration;
+    const cost = maxCases
+        ? capped.reduce((sum, r) => sum + (r.usage?.cost ?? 0), 0)
+        : result.stats.cost;
+    const metrics = {};
+    if (maxCases && total > 0) {
+        const scoresByMetric = new Map();
+        for (const r of capped) {
+            for (const s of r.scores) {
+                let arr = scoresByMetric.get(s.name);
+                if (!arr) {
+                    arr = [];
+                    scoresByMetric.set(s.name, arr);
+                }
+                arr.push(s.score);
+            }
+        }
+        for (const [name, values] of scoresByMetric) {
+            metrics[name] = values.reduce((a, b) => a + b, 0) / values.length;
+        }
+    }
+    else {
+        for (const [name, agg] of Object.entries(result.aggregated)) {
+            metrics[name] = agg.mean;
+        }
+    }
+    const assertionsPassed = maxCases ? true : result.assertions.every((a) => a.passed);
+    return { success: true, total, duration, cost, metrics, assertionsPassed };
+}
+export function createRunEvalTool(suite) {
+    return {
+        name: 'run_eval',
+        description: 'Run an evaluation suite against the configured dataset and target',
+        parameters: RunEvalParamsSchema,
+        execute: async ({ maxCases }) => {
+            try {
+                const result = await suite.run();
+                return buildSummary(result, maxCases);
+            }
+            catch (err) {
+                return { success: false, error: err instanceof Error ? err.message : String(err) };
+            }
+        },
+    };
+}
+export function evalTools(suite) {
+    return [createRunEvalTool(suite)];
+}
+//# sourceMappingURL=tools.js.map

package/dist/tools.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACjD,CAAC,CAAC;AAIH,SAAS,YAAY,CAAC,MAAuB,EAAE,QAAiB;IAC9D,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;IAE7E,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,MAAM,QAAQ,GAAG,QAAQ;QACvB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAChD,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;IAC1B,MAAM,IAAI,GAAG,QAAQ;QACnB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1D,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;IAEtB,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,QAAQ,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QACnD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,EAAE,CAAC;oBACT,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACpE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5D,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,MAAM,gBAAgB,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAEpF,OAAO,EAAE,OAAO,EAAE,IAAa,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;AACtF,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAgB;IAChD,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,mEAAmE;QAChF,UAAU,EAAE,mBAAmB;QAC/B,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC9B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;gBACjC,OAAO,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAgB;IACxC,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;AACpC,CAAC"}

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@cogitator-ai/evals",
+  "version": "0.1.0",
+  "description": "Evaluation framework for Cogitator AI agents",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "clean": "rm -rf dist",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "dependencies": {
+    "@cogitator-ai/types": "workspace:*",
+    "nanoid": "^5.0.4",
+    "zod": "^4.3.6"
+  },
+  "peerDependencies": {
+    "@cogitator-ai/core": "workspace:*"
+  },
+  "peerDependenciesMeta": {
+    "@cogitator-ai/core": {
+      "optional": true
+    }
+  },
+  "optionalDependencies": {
+    "papaparse": "^5.5.0"
+  },
+  "devDependencies": {
+    "@cogitator-ai/core": "workspace:*",
+    "@types/papaparse": "^5.3.15",
+    "typescript": "^5.7.2",
+    "vitest": "^4.0.18"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/cogitator-ai/Cogitator-AI.git",
+    "directory": "packages/evals"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://npm.pkg.github.com"
+  },
+  "license": "MIT"
+}