@cogitator-ai/evals 0.8.0 → 0.9.0

package/dist/vitest.d.ts

@@ -0,0 +1,35 @@
+import type { EvalRunOptions, EvalSuite, EvalSuiteResult } from './eval-suite.js';
+export interface DescribeEvalOptions {
+    /** The suite to run */
+    suite: EvalSuite;
+    /** Passed to `suite.run()`: `maxCases` also decides which cases get a test */
+    run?: EvalRunOptions;
+    /**
+     * Milliseconds the suite run may take, 10 minutes by default. When it runs out the run is
+     * cancelled and every test fails with the timeout.
+     */
+    timeout?: number;
+}
+export interface DescribeEvalHandle {
+    /**
+     * The result of the suite run. Call it inside a test or an `afterAll` hook of the same
+     * describe block: it throws while the run has not finished, or with the cause when it failed.
+     */
+    result: () => EvalSuiteResult;
+}
+/**
+ * Registers a Vitest describe block for an eval suite. The suite runs once in `beforeAll`, each
+ * case becomes a test that fails when the case errored, and an `assertions` test fails with
+ * every assertion that did not pass.
+ *
+ * @example
+ * ```ts
+ * import { describeEval } from '@cogitator-ai/evals/vitest';
+ *
+ * const qa = describeEval('qa agent', { suite, run: { maxCases: 50 } });
+ *
+ * afterAll(() => qa.result().report('junit', { path: './reports/qa' }));
+ * ```
+ */
+export declare function describeEval(name: string, options: DescribeEvalOptions): DescribeEvalHandle;
+//# sourceMappingURL=vitest.d.ts.map

package/dist/vitest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vitest.d.ts","sourceRoot":"","sources":["../src/vitest.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAS/E,MAAM,WAAW,mBAAmB;IAClC,uBAAuB;IACvB,KAAK,EAAE,SAAS,CAAC;IACjB,8EAA8E;IAC9E,GAAG,CAAC,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,MAAM,EAAE,MAAM,eAAe,CAAC;CAC/B;AAWD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,mBAAmB,GAAG,kBAAkB,CAsE3F"}

package/dist/vitest.js

@@ -0,0 +1,85 @@
+import { beforeAll, describe, it } from 'vitest';
+import { caseLabel } from './reporters/format.js';
+const DEFAULT_TIMEOUT = 10 * 60 * 1000;
+/** Time the hook keeps after the deadline settled the outcome, for Vitest itself */
+const HOOK_GRACE = 5 * 1000;
+/** The longest delay `setTimeout` keeps, less the grace the hook adds to it */
+const MAX_TIMEOUT = 2_147_483_647 - HOOK_GRACE;
+function messageOf(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+/**
+ * Registers a Vitest describe block for an eval suite. The suite runs once in `beforeAll`, each
+ * case becomes a test that fails when the case errored, and an `assertions` test fails with
+ * every assertion that did not pass.
+ *
+ * @example
+ * ```ts
+ * import { describeEval } from '@cogitator-ai/evals/vitest';
+ *
+ * const qa = describeEval('qa agent', { suite, run: { maxCases: 50 } });
+ *
+ * afterAll(() => qa.result().report('junit', { path: './reports/qa' }));
+ * ```
+ */
+export function describeEval(name, options) {
+    const { suite, run = {}, timeout = DEFAULT_TIMEOUT } = options;
+    if (!Number.isFinite(timeout) || timeout <= 0 || timeout > MAX_TIMEOUT) {
+        throw new Error(`describeEval timeout must be a positive number of milliseconds up to ${MAX_TIMEOUT}, got ${timeout}`);
+    }
+    let outcome = { state: 'pending' };
+    const result = () => {
+        switch (outcome.state) {
+            case 'done':
+                return outcome.result;
+            case 'failed':
+                throw new Error(`Eval suite '${name}' failed to run: ${messageOf(outcome.error)}`, {
+                    cause: outcome.error,
+                });
+            case 'pending':
+                throw new Error(`Eval suite '${name}' has not finished running: read the result inside its tests or afterAll`);
+        }
+    };
+    describe(name, () => {
+        beforeAll(async () => {
+            const deadline = new AbortController();
+            const late = new Error(`the run did not finish within ${timeout} ms`);
+            let timer;
+            const expired = new Promise((_resolve, reject) => {
+                timer = setTimeout(() => {
+                    deadline.abort(late);
+                    reject(late);
+                }, timeout);
+            });
+            const signal = run.signal ? AbortSignal.any([run.signal, deadline.signal]) : deadline.signal;
+            const running = suite.run({ ...run, signal });
+            running.catch(() => undefined);
+            try {
+                outcome = { state: 'done', result: await Promise.race([running, expired]) };
+            }
+            catch (error) {
+                outcome = { state: 'failed', error };
+            }
+            finally {
+                clearTimeout(timer);
+            }
+        }, timeout + HOOK_GRACE);
+        suite.cases(run.maxCases).forEach((evalCase, index) => {
+            it(caseLabel(index, evalCase.input), () => {
+                const caseResult = result().results[index];
+                if (caseResult?.error !== undefined) {
+                    const attempts = caseResult.attempts === undefined ? '' : ` after ${caseResult.attempts} attempt(s)`;
+                    throw new Error(`Case failed${attempts}: ${caseResult.error}`);
+                }
+            });
+        });
+        it('assertions', () => {
+            const failed = result().assertions.filter((a) => !a.passed);
+            if (failed.length > 0) {
+                throw new Error(`${failed.length} assertion(s) failed:\n${failed.map((a) => `  ${a.name}: ${a.message}`).join('\n')}`);
+            }
+        });
+    });
+    return { result };
+}
+//# sourceMappingURL=vitest.js.map

package/dist/vitest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vitest.js","sourceRoot":"","sources":["../src/vitest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAEjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,MAAM,eAAe,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACvC,oFAAoF;AACpF,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC;AAC5B,+EAA+E;AAC/E,MAAM,WAAW,GAAG,aAAa,GAAG,UAAU,CAAC;AA2B/C,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,OAA4B;IACrE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,EAAE,EAAE,OAAO,GAAG,eAAe,EAAE,GAAG,OAAO,CAAC;IAC/D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CACb,wEAAwE,WAAW,SAAS,OAAO,EAAE,CACtG,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,GAAY,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,GAAoB,EAAE;QACnC,QAAQ,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,KAAK,MAAM;gBACT,OAAO,OAAO,CAAC,MAAM,CAAC;YACxB,KAAK,QAAQ;gBACX,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,oBAAoB,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE;oBACjF,KAAK,EAAE,OAAO,CAAC,KAAK;iBACrB,CAAC,CAAC;YACL,KAAK,SAAS;gBACZ,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,0EAA0E,CAC9F,CAAC;QACN,CAAC;IACH,CAAC,CAAC;IAEF,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE;QAClB,SAAS,CAAC,KAAK,IAAI,EAAE;YACnB,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,iCAAiC,OAAO,KAAK,CAAC,CAAC;YACtE,IAAI,KAAgD,CAAC;YACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;gBACtD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,CAAC,CAAC;gBACf,CAAC,EAAE,OAAO,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC7F,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9C,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC/B,IAAI,CAAC;gBACH,OAAO,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC;YAC9E,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YACvC,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;QACH,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC,CAAC;QAEzB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;YACpD,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE;gBACxC,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,UAAU,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,QAAQ,GACZ,UAAU,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,UAAU,CAAC,QAAQ,aAAa,CAAC;oBACtF,MAAM,IAAI,KAAK,CAAC,cAAc,QAAQ,KAAK,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;gBACjE,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE;YACpB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC5D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,MAAM,0BAA0B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACtG,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,CAAC;AACpB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cogitator-ai/evals",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Evaluation framework for Cogitator AI agents",
   "keywords": [
     "cogitator",
@@ -21,6 +21,11 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
       "default": "./dist/index.js"
+    },
+    "./vitest": {
+      "types": "./dist/vitest.d.ts",
+      "import": "./dist/vitest.js",
+      "default": "./dist/vitest.js"
     }
   },
   "sideEffects": false,
@@ -34,18 +39,22 @@
     "zod": "^4.6.5"
   },
   "peerDependencies": {
-    "@cogitator-ai/core": ">=0.53.0"
+    "@cogitator-ai/core": ">=0.56.4",
+    "vitest": ">=3.0.0"
   },
   "peerDependenciesMeta": {
     "@cogitator-ai/core": {
       "optional": true
+    },
+    "vitest": {
+      "optional": true
     }
   },
   "optionalDependencies": {
     "papaparse": "^5.7.0"
   },
   "devDependencies": {
-    "@cogitator-ai/core": "0.53.0",
+    "@cogitator-ai/core": "0.56.4",
     "@types/papaparse": "^5.3.15",
     "typescript": "^6.0.3",
     "vitest": "^5.0.3"