@cogitator-ai/evals 0.7.0 → 0.9.0

package/dist/reporters/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAgDhC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,SAAyB;IAC5E,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;IAClE,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAwDxC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,SAAiC;IACpF,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;IACtE,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,OAAO;gBACV,WAAW,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,QAAQ;gBACX,YAAY,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/reporters/junit.d.ts

@@ -0,0 +1,8 @@
+import type { EvalSuiteResult } from './index.js';
+/** The JUnit XML document of a suite run: one testcase per case and one per assertion */
+export declare function junitXml(result: EvalSuiteResult): string;
+/** Writes the suite run as JUnit XML, which CI systems show as a test report */
+export declare function junitReport(result: EvalSuiteResult, options: {
+    path: string;
+}): void;
+//# sourceMappingURL=junit.d.ts.map

package/dist/reporters/junit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit.d.ts","sourceRoot":"","sources":["../../src/reporters/junit.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiG/C,yFAAyF;AACzF,wBAAgB,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAqBxD;AAED,gFAAgF;AAChF,wBAAgB,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAEpF"}

package/dist/reporters/junit.js

@@ -0,0 +1,107 @@
+import { writeFileSync } from 'node:fs';
+import { caseFlags, caseLabel } from './format.js';
+const SUITE_NAME = 'eval';
+function isXmlChar(code) {
+    return (code === 0x9 ||
+        code === 0xa ||
+        code === 0xd ||
+        (code >= 0x20 && code <= 0xd7ff) ||
+        (code >= 0xe000 && code <= 0xfffd) ||
+        (code >= 0x10000 && code <= 0x10ffff));
+}
+/** Replaces the characters XML 1.0 does not allow, lone surrogates included, with U+FFFD */
+function xmlSafe(value) {
+    let out = '';
+    for (const ch of value) {
+        const code = ch.codePointAt(0) ?? 0;
+        out += isXmlChar(code) ? ch : '�';
+    }
+    return out;
+}
+/** Escapes text content, `\r` kept as a reference since parsers fold a raw one into `\n` */
+function escapeText(value) {
+    return xmlSafe(value)
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/\r/g, '&#13;');
+}
+/** Escapes an attribute value, keeping tabs and line breaks that a parser would fold to spaces */
+function escapeAttr(value) {
+    return escapeText(value)
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;')
+        .replace(/\t/g, '&#9;')
+        .replace(/\n/g, '&#10;');
+}
+function attrs(values) {
+    return Object.entries(values)
+        .map(([key, value]) => ` ${key}="${escapeAttr(String(value))}"`)
+        .join('');
+}
+function seconds(ms) {
+    return (Math.max(0, ms) / 1000).toFixed(3);
+}
+function firstLine(text) {
+    return text.split(/\r?\n/, 1)[0] ?? '';
+}
+function caseOutput(result) {
+    const lines = result.scores.map((s) => {
+        const details = s.details ? ` (${s.details})` : '';
+        const error = s.error !== undefined ? ` [error: ${s.error}]` : '';
+        return `${s.name}: ${s.score}${details}${error}`;
+    });
+    for (const flag of caseFlags(result))
+        lines.push(`${flag}: true`);
+    return lines.join('\n');
+}
+function caseTestcase(result, index) {
+    const head = `    <testcase${attrs({
+        name: caseLabel(index, result.case.input),
+        classname: `${SUITE_NAME}.cases`,
+        time: seconds(result.duration),
+    })}`;
+    const body = [];
+    if (result.error !== undefined) {
+        body.push(`      <failure${attrs({ message: firstLine(result.error), type: 'CaseError' })}>${escapeText(result.error)}</failure>`);
+    }
+    const output = caseOutput(result);
+    if (output !== '')
+        body.push(`      <system-out>${escapeText(output)}</system-out>`);
+    return body.length === 0 ? `${head}/>` : `${head}>\n${body.join('\n')}\n    </testcase>`;
+}
+function assertionTestcase(assertion) {
+    const head = `    <testcase${attrs({
+        name: assertion.name,
+        classname: `${SUITE_NAME}.assertions`,
+        time: seconds(0),
+    })}`;
+    if (assertion.passed)
+        return `${head}/>`;
+    return `${head}>\n      <failure${attrs({ message: firstLine(assertion.message), type: 'AssertionFailure' })}>${escapeText(assertion.message)}</failure>\n    </testcase>`;
+}
+/** The JUnit XML document of a suite run: one testcase per case and one per assertion */
+export function junitXml(result) {
+    const tests = result.results.length + result.assertions.length;
+    const failures = result.results.filter((r) => r.error !== undefined).length +
+        result.assertions.filter((a) => !a.passed).length;
+    const time = seconds(result.stats.duration);
+    const counts = { tests, failures, errors: 0, skipped: 0, time };
+    const testcases = [
+        ...result.results.map((r, i) => caseTestcase(r, i)),
+        ...result.assertions.map((a) => assertionTestcase(a)),
+    ];
+    return [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        `<testsuites${attrs({ name: SUITE_NAME, tests, failures, errors: 0, time })}>`,
+        `  <testsuite${attrs({ name: SUITE_NAME, ...counts })}${testcases.length === 0 ? '/>' : '>'}`,
+        ...(testcases.length === 0 ? [] : [...testcases, '  </testsuite>']),
+        '</testsuites>',
+        '',
+    ].join('\n');
+}
+/** Writes the suite run as JUnit XML, which CI systems show as a test report */
+export function junitReport(result, options) {
+    writeFileSync(options.path, junitXml(result), 'utf-8');
+}
+//# sourceMappingURL=junit.js.map

package/dist/reporters/junit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit.js","sourceRoot":"","sources":["../../src/reporters/junit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAEhD,MAAM,UAAU,GAAG,MAAM,CAAC;AAE1B,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,CACL,IAAI,KAAK,GAAG;QACZ,IAAI,KAAK,GAAG;QACZ,IAAI,KAAK,GAAG;QACZ,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,CAAC;QAChC,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC;QAClC,CAAC,IAAI,IAAI,OAAO,IAAI,IAAI,IAAI,QAAQ,CAAC,CACtC,CAAC;AACJ,CAAC;AAED,4FAA4F;AAC5F,SAAS,OAAO,CAAC,KAAa;IAC5B,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACpC,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IACpC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4FAA4F;AAC5F,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,OAAO,CAAC,KAAK,CAAC;SAClB,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,kGAAkG;AAClG,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,UAAU,CAAC,KAAK,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,KAAK,CAAC,MAAuC;IACpD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SAC1B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC;SAC/D,IAAI,CAAC,EAAE,CAAC,CAAC;AACd,CAAC;AAED,SAAS,OAAO,CAAC,EAAU;IACzB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC;AAID,SAAS,UAAU,CAAC,MAAkB;IACpC,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,OAAO,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,GAAG,OAAO,GAAG,KAAK,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,CAAC;IAClE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,MAAkB,EAAE,KAAa;IACrD,MAAM,IAAI,GAAG,gBAAgB,KAAK,CAAC;QACjC,IAAI,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QACzC,SAAS,EAAE,GAAG,UAAU,QAAQ;QAChC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;KAC/B,CAAC,EAAE,CAAC;IACL,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CACP,iBAAiB,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CACxH,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAClC,IAAI,MAAM,KAAK,EAAE;QAAE,IAAI,CAAC,IAAI,CAAC,qBAAqB,UAAU,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACrF,OAAO,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC;AAC3F,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAgD;IACzE,MAAM,IAAI,GAAG,gBAAgB,KAAK,CAAC;QACjC,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,SAAS,EAAE,GAAG,UAAU,aAAa;QACrC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;KACjB,CAAC,EAAE,CAAC;IACL,IAAI,SAAS,CAAC,MAAM;QAAE,OAAO,GAAG,IAAI,IAAI,CAAC;IACzC,OAAO,GAAG,IAAI,oBAAoB,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC,IAAI,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,6BAA6B,CAAC;AAC7K,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,QAAQ,CAAC,MAAuB;IAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;IAC/D,MAAM,QAAQ,GACZ,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;QAC1D,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACpD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC;IAEhE,MAAM,SAAS,GAAG;QAChB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnD,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;KACtD,CAAC;IAEF,OAAO;QACL,wCAAwC;QACxC,cAAc,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG;QAC9E,eAAe,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE;QAC7F,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,EAAE,gBAAgB,CAAC,CAAC;QACnE,eAAe;QACf,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,WAAW,CAAC,MAAuB,EAAE,OAAyB;IAC5E,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AACzD,CAAC"}

package/dist/schema.d.ts

@@ -26,11 +26,22 @@ export interface JudgeCogitator {
         };
     }>;
 }
+/**
+ * Where the judge keeps the verdicts it already gave, by a key of its model, settings and the
+ * whole prompt: a case judged again with the same answer reuses the verdict and costs nothing.
+ * A store of your own (a file, Redis) keeps verdicts across runs
+ */
+export interface JudgeCache {
+    get(key: string): string | undefined | Promise<string | undefined>;
+    set(key: string, verdict: string): void | Promise<void>;
+}
 export declare const JudgeConfigSchema: z.ZodObject<{
     model: z.ZodString;
     temperature: z.ZodDefault<z.ZodNumber>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
     cogitator: z.ZodOptional<z.ZodCustom<JudgeCogitator, JudgeCogitator>>;
+    cache: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<JudgeCache, JudgeCache>]>>;
+    concurrency: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 export declare const EvalComparisonConfigSchema: z.ZodObject<{
     concurrency: z.ZodDefault<z.ZodNumber>;

package/dist/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAUD,eAAO,MAAM,iBAAiB;;;;;iBAM5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;iBAIrC,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACnE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAmBD,eAAO,MAAM,iBAAiB;;;;;;;iBAgB5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;iBAIrC,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}

package/dist/schema.js

@@ -10,6 +10,12 @@ export const EvalSuiteConfigSchema = z.object({
     timeout: z.number().int().min(1000).default(30000),
     retries: z.number().int().min(0).max(10).default(0),
 });
+function isJudgeCache(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        typeof value.get === 'function' &&
+        typeof value.set === 'function');
+}
 function isJudgeCogitator(value) {
     return (typeof value === 'object' &&
         value !== null &&
@@ -21,6 +27,16 @@ export const JudgeConfigSchema = z.object({
     maxTokens: z.number().int().positive().optional(),
     /** Runtime that runs the judge; defaults to the `cogitator` of an agent target */
     cogitator: z.custom(isJudgeCogitator).optional(),
+    /**
+     * Reuse verdicts: `true` keeps them in memory for the suite, a `JudgeCache` of your own keeps
+     * them where it likes. Default: every case is judged afresh
+     */
+    cache: z.union([z.boolean(), z.custom(isJudgeCache)]).optional(),
+    /**
+     * Most judge calls at once across the run, below the suite's own concurrency times its judge
+     * metrics. Default: no limit of its own
+     */
+    concurrency: z.number().int().positive().optional(),
 });
 export const EvalComparisonConfigSchema = z.object({
     concurrency: z.number().int().min(1).default(5),

package/dist/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC;AAaH,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;CACjE,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC"}
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC;AAuBH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU;QACtD,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;IAChE;;;OAGG;IACH,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAa,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5E;;;OAGG;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC"}

package/dist/vitest.d.ts

@@ -0,0 +1,35 @@
+import type { EvalRunOptions, EvalSuite, EvalSuiteResult } from './eval-suite.js';
+export interface DescribeEvalOptions {
+    /** The suite to run */
+    suite: EvalSuite;
+    /** Passed to `suite.run()`: `maxCases` also decides which cases get a test */
+    run?: EvalRunOptions;
+    /**
+     * Milliseconds the suite run may take, 10 minutes by default. When it runs out the run is
+     * cancelled and every test fails with the timeout.
+     */
+    timeout?: number;
+}
+export interface DescribeEvalHandle {
+    /**
+     * The result of the suite run. Call it inside a test or an `afterAll` hook of the same
+     * describe block: it throws while the run has not finished, or with the cause when it failed.
+     */
+    result: () => EvalSuiteResult;
+}
+/**
+ * Registers a Vitest describe block for an eval suite. The suite runs once in `beforeAll`, each
+ * case becomes a test that fails when the case errored, and an `assertions` test fails with
+ * every assertion that did not pass.
+ *
+ * @example
+ * ```ts
+ * import { describeEval } from '@cogitator-ai/evals/vitest';
+ *
+ * const qa = describeEval('qa agent', { suite, run: { maxCases: 50 } });
+ *
+ * afterAll(() => qa.result().report('junit', { path: './reports/qa' }));
+ * ```
+ */
+export declare function describeEval(name: string, options: DescribeEvalOptions): DescribeEvalHandle;
+//# sourceMappingURL=vitest.d.ts.map

package/dist/vitest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vitest.d.ts","sourceRoot":"","sources":["../src/vitest.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAS/E,MAAM,WAAW,mBAAmB;IAClC,uBAAuB;IACvB,KAAK,EAAE,SAAS,CAAC;IACjB,8EAA8E;IAC9E,GAAG,CAAC,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,MAAM,EAAE,MAAM,eAAe,CAAC;CAC/B;AAWD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,mBAAmB,GAAG,kBAAkB,CAsE3F"}

package/dist/vitest.js

@@ -0,0 +1,85 @@
+import { beforeAll, describe, it } from 'vitest';
+import { caseLabel } from './reporters/format.js';
+const DEFAULT_TIMEOUT = 10 * 60 * 1000;
+/** Time the hook keeps after the deadline settled the outcome, for Vitest itself */
+const HOOK_GRACE = 5 * 1000;
+/** The longest delay `setTimeout` keeps, less the grace the hook adds to it */
+const MAX_TIMEOUT = 2_147_483_647 - HOOK_GRACE;
+function messageOf(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+/**
+ * Registers a Vitest describe block for an eval suite. The suite runs once in `beforeAll`, each
+ * case becomes a test that fails when the case errored, and an `assertions` test fails with
+ * every assertion that did not pass.
+ *
+ * @example
+ * ```ts
+ * import { describeEval } from '@cogitator-ai/evals/vitest';
+ *
+ * const qa = describeEval('qa agent', { suite, run: { maxCases: 50 } });
+ *
+ * afterAll(() => qa.result().report('junit', { path: './reports/qa' }));
+ * ```
+ */
+export function describeEval(name, options) {
+    const { suite, run = {}, timeout = DEFAULT_TIMEOUT } = options;
+    if (!Number.isFinite(timeout) || timeout <= 0 || timeout > MAX_TIMEOUT) {
+        throw new Error(`describeEval timeout must be a positive number of milliseconds up to ${MAX_TIMEOUT}, got ${timeout}`);
+    }
+    let outcome = { state: 'pending' };
+    const result = () => {
+        switch (outcome.state) {
+            case 'done':
+                return outcome.result;
+            case 'failed':
+                throw new Error(`Eval suite '${name}' failed to run: ${messageOf(outcome.error)}`, {
+                    cause: outcome.error,
+                });
+            case 'pending':
+                throw new Error(`Eval suite '${name}' has not finished running: read the result inside its tests or afterAll`);
+        }
+    };
+    describe(name, () => {
+        beforeAll(async () => {
+            const deadline = new AbortController();
+            const late = new Error(`the run did not finish within ${timeout} ms`);
+            let timer;
+            const expired = new Promise((_resolve, reject) => {
+                timer = setTimeout(() => {
+                    deadline.abort(late);
+                    reject(late);
+                }, timeout);
+            });
+            const signal = run.signal ? AbortSignal.any([run.signal, deadline.signal]) : deadline.signal;
+            const running = suite.run({ ...run, signal });
+            running.catch(() => undefined);
+            try {
+                outcome = { state: 'done', result: await Promise.race([running, expired]) };
+            }
+            catch (error) {
+                outcome = { state: 'failed', error };
+            }
+            finally {
+                clearTimeout(timer);
+            }
+        }, timeout + HOOK_GRACE);
+        suite.cases(run.maxCases).forEach((evalCase, index) => {
+            it(caseLabel(index, evalCase.input), () => {
+                const caseResult = result().results[index];
+                if (caseResult?.error !== undefined) {
+                    const attempts = caseResult.attempts === undefined ? '' : ` after ${caseResult.attempts} attempt(s)`;
+                    throw new Error(`Case failed${attempts}: ${caseResult.error}`);
+                }
+            });
+        });
+        it('assertions', () => {
+            const failed = result().assertions.filter((a) => !a.passed);
+            if (failed.length > 0) {
+                throw new Error(`${failed.length} assertion(s) failed:\n${failed.map((a) => `  ${a.name}: ${a.message}`).join('\n')}`);
+            }
+        });
+    });
+    return { result };
+}
+//# sourceMappingURL=vitest.js.map

package/dist/vitest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vitest.js","sourceRoot":"","sources":["../src/vitest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAEjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,MAAM,eAAe,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACvC,oFAAoF;AACpF,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC;AAC5B,+EAA+E;AAC/E,MAAM,WAAW,GAAG,aAAa,GAAG,UAAU,CAAC;AA2B/C,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,OAA4B;IACrE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,EAAE,EAAE,OAAO,GAAG,eAAe,EAAE,GAAG,OAAO,CAAC;IAC/D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CACb,wEAAwE,WAAW,SAAS,OAAO,EAAE,CACtG,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,GAAY,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,GAAoB,EAAE;QACnC,QAAQ,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,KAAK,MAAM;gBACT,OAAO,OAAO,CAAC,MAAM,CAAC;YACxB,KAAK,QAAQ;gBACX,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,oBAAoB,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE;oBACjF,KAAK,EAAE,OAAO,CAAC,KAAK;iBACrB,CAAC,CAAC;YACL,KAAK,SAAS;gBACZ,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,0EAA0E,CAC9F,CAAC;QACN,CAAC;IACH,CAAC,CAAC;IAEF,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE;QAClB,SAAS,CAAC,KAAK,IAAI,EAAE;YACnB,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,iCAAiC,OAAO,KAAK,CAAC,CAAC;YACtE,IAAI,KAAgD,CAAC;YACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;gBACtD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,CAAC,CAAC;gBACf,CAAC,EAAE,OAAO,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC7F,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9C,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC/B,IAAI,CAAC;gBACH,OAAO,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC;YAC9E,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YACvC,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;QACH,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC,CAAC;QAEzB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;YACpD,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE;gBACxC,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,UAAU,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,QAAQ,GACZ,UAAU,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,UAAU,CAAC,QAAQ,aAAa,CAAC;oBACtF,MAAM,IAAI,KAAK,CAAC,cAAc,QAAQ,KAAK,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;gBACjE,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE;YACpB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC5D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,MAAM,0BAA0B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACtG,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,CAAC;AACpB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cogitator-ai/evals",
-  "version": "0.7.0",
+  "version": "0.9.0",
   "description": "Evaluation framework for Cogitator AI agents",
   "keywords": [
     "cogitator",
@@ -21,6 +21,11 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
       "default": "./dist/index.js"
+    },
+    "./vitest": {
+      "types": "./dist/vitest.d.ts",
+      "import": "./dist/vitest.js",
+      "default": "./dist/vitest.js"
     }
   },
   "sideEffects": false,
@@ -34,18 +39,22 @@
     "zod": "^4.6.5"
   },
   "peerDependencies": {
-    "@cogitator-ai/core": ">=0.41.0"
+    "@cogitator-ai/core": ">=0.56.4",
+    "vitest": ">=3.0.0"
   },
   "peerDependenciesMeta": {
     "@cogitator-ai/core": {
       "optional": true
+    },
+    "vitest": {
+      "optional": true
     }
   },
   "optionalDependencies": {
     "papaparse": "^5.7.0"
   },
   "devDependencies": {
-    "@cogitator-ai/core": "0.41.0",
+    "@cogitator-ai/core": "0.56.4",
     "@types/papaparse": "^5.3.15",
     "typescript": "^6.0.3",
     "vitest": "^5.0.3"