@vercel/agent-eval 0.2.0 → 0.3.0

package/dist/lib/results.js

@@ -1,7 +1,7 @@
 /**
  * Results storage and reporting for eval experiments.
  */
-import { mkdirSync, writeFileSync } from 'fs';
+import { mkdirSync, writeFileSync, readdirSync, readFileSync, existsSync, statSync } from 'fs';
 import { join } from 'path';
 import chalk from 'chalk';
 import { parseTranscript } from './o11y/index.js';
@@ -86,12 +86,24 @@ export function saveResults(results, options) {
         const evalDir = join(experimentDir, evalSummary.name);
         mkdirSync(evalDir, { recursive: true });
         // Save summary (simplified format per design)
+        const fingerprint = options.fingerprints?.[evalSummary.name];
+        const classification = options.classifications?.[evalSummary.name];
+        const valid = options.validity?.[evalSummary.name];
         const summaryForFile = {
             totalRuns: evalSummary.totalRuns,
             passedRuns: evalSummary.passedRuns,
             passRate: `${evalSummary.passRate.toFixed(0)}%`,
             meanDuration: evalSummary.meanDuration,
         };
+        if (fingerprint) {
+            summaryForFile.fingerprint = fingerprint;
+        }
+        if (classification) {
+            summaryForFile.classification = classification;
+        }
+        if (valid === false) {
+            summaryForFile.valid = false;
+        }
         writeFileSync(join(evalDir, 'summary.json'), JSON.stringify(summaryForFile, null, 2));
         // Save individual run results
         for (let i = 0; i < evalSummary.runs.length; i++) {
@@ -127,14 +139,16 @@ export function saveResults(results, options) {
                     writeFileSync(join(outputsDir, 'eval.txt'), runData.outputContent.eval);
                     outputPaths.eval = './outputs/eval.txt';
                 }
-                // Save npm script outputs (nested to avoid collision)
+                // Save npm script outputs under outputs/scripts/ to avoid collision with eval.txt
                 if (runData.outputContent.scripts) {
+                    const scriptsDir = join(outputsDir, 'scripts');
+                    mkdirSync(scriptsDir, { recursive: true });
                     outputPaths.scripts = {};
                     for (const [name, content] of Object.entries(runData.outputContent.scripts)) {
                         if (content) {
                             const fileName = `${name}.txt`;
-                            writeFileSync(join(outputsDir, fileName), content);
-                            outputPaths.scripts[name] = `./outputs/${fileName}`;
+                            writeFileSync(join(scriptsDir, fileName), content);
+                            outputPaths.scripts[name] = `./outputs/scripts/${fileName}`;
                         }
                     }
                 }
@@ -197,4 +211,75 @@ export function formatRunResult(evalName, runNumber, totalRuns, result) {
 export function createProgressDisplay(evalName, runNumber, totalRuns) {
     return chalk.blue(`Running ${evalName} [${runNumber}/${totalRuns}]...`);
 }
+/**
+ * Scan existing results for an experiment to find reusable eval results.
+ *
+ * A result is reusable if:
+ * 1. Its fingerprint matches the current fingerprint
+ * 2. It is "valid" (not marked as invalid by the classifier)
+ * 3. It has passedRuns > 0 (successful result worth reusing)
+ *
+ * Scans all timestamps newest-first and returns the latest match per eval.
+ */
+export function scanReusableResults(resultsDir, experimentName, fingerprints) {
+    const reusable = new Map();
+    const experimentDir = join(resultsDir, experimentName);
+    if (!existsSync(experimentDir))
+        return reusable;
+    // Get all timestamps, sorted newest first
+    let timestamps;
+    try {
+        timestamps = readdirSync(experimentDir)
+            .filter((t) => !t.startsWith('.'))
+            .sort()
+            .reverse();
+    }
+    catch {
+        return reusable;
+    }
+    for (const timestamp of timestamps) {
+        const tsDir = join(experimentDir, timestamp);
+        if (!statSync(tsDir).isDirectory())
+            continue;
+        let evalDirs;
+        try {
+            evalDirs = readdirSync(tsDir).filter((d) => !d.startsWith('.'));
+        }
+        catch {
+            continue;
+        }
+        for (const evalDir of evalDirs) {
+            // Already found a reusable result for this eval
+            if (reusable.has(evalDir))
+                continue;
+            // Check if we have a fingerprint for this eval
+            const expectedFingerprint = fingerprints[evalDir];
+            if (!expectedFingerprint)
+                continue;
+            const summaryPath = join(tsDir, evalDir, 'summary.json');
+            try {
+                const summary = JSON.parse(readFileSync(summaryPath, 'utf-8'));
+                // Check fingerprint match
+                if (summary.fingerprint !== expectedFingerprint)
+                    continue;
+                // Check validity (valid defaults to true if not explicitly set to false)
+                if (summary.valid === false)
+                    continue;
+                // Check that it has at least some passed runs
+                if (summary.passedRuns <= 0)
+                    continue;
+                reusable.set(evalDir, {
+                    evalName: evalDir,
+                    fingerprint: summary.fingerprint,
+                    passRate: summary.passRate,
+                    timestamp,
+                });
+            }
+            catch {
+                // Skip invalid summaries
+            }
+        }
+    }
+    return reusable;
+}
 //# sourceMappingURL=results.js.map

package/dist/lib/results.js.map

@@ -1 +1 @@
-{"version":3,"file":"results.js","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,KAAK,MAAM,OAAO,CAAC;AAS1B,OAAO,EAAE,eAAe,EAAmB,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,WAA2B;IAClE,gDAAgD;IAChD,MAAM,aAAa,GAAiC,EAAE,CAAC;IAEvD,0BAA0B;IAC1B,IAAI,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACnC,aAAa,CAAC,IAAI,GAAG,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;IACrD,CAAC;IAED,qEAAqE;IACrE,IAAI,WAAW,CAAC,cAAc,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrF,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,EAAE,CAAC;YACxE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAClB,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE;YACN,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACjD,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE,qBAAqB;SAC7D;QACD,UAAU,EAAE,WAAW,CAAC,UAAU;QAClC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;KACjF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,OAAsB;IACpE,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEnE,OAAO;QACL,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM;QACtB,UAAU;QACV,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAChE,YAAY,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,EAAE,OAAO;KACd,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACrC,MAAgC,EAChC,KAAoB,EACpB,SAAe,EACf,WAAiB;IAEjB,OAAO;QACL,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE;QACtC,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAYD;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,WAAW,CACzB,OAA0B,EAC1B,OAA2B;IAE3B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvD,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;IAElF,8BAA8B;IAC9B,SAAS,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9C,wBAAwB;IACxB,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACtD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAExC,8CAA8C;QAC9C,MAAM,cAAc,GAAG;YACrB,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,UAAU,EAAE,WAAW,CAAC,UAAU;YAClC,QAAQ,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YAC/C,YAAY,EAAE,WAAW,CAAC,YAAY;SACvC,CAAC;QACF,aAAa,CACX,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAC7B,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CACxC,CAAC;QAEF,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7C,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEvC,+CAA+C;YAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YACnC,MAAM,eAAe,GAAqD;gBACxE,GAAG,OAAO,CAAC,MAAM;gBACjB,KAAK;aACN,CAAC;YAEF,gCAAgC;YAChC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,2BAA2B;gBAC3B,MAAM,UAAU,GAAG,eAAe,CAChC,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,MAAM,CAAC,KAAK,EACpB,KAAK,CACN,CAAC;gBAEF,6DAA6D;gBAC7D,aAAa,CACX,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,EAC/B,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CACpC,CAAC;gBACF,eAAe,CAAC,cAAc,GAAG,mBAAmB,CAAC;gBAErD,2DAA2D;gBAC3D,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;gBACxE,eAAe,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;gBAE7D,kDAAkD;gBAClD,eAAe,CAAC,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC;YAC5C,CAAC;YAED,uCAAuC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3C,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC1B,MAAM,WAAW,GAAiC,EAAE,CAAC;gBAErD,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;oBAC/B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;oBACxE,WAAW,CAAC,IAAI,GAAG,oBAAoB,CAAC;gBAC1C,CAAC;gBAED,sDAAsD;gBACtD,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;oBAClC,WAAW,CAAC,OAAO,GAAG,EAAE,CAAC;oBACzB,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC5E,IAAI,OAAO,EAAE,CAAC;4BACZ,MAAM,QAAQ,GAAG,GAAG,IAAI,MAAM,CAAC;4BAC/B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;4BACnD,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,QAAQ,EAAE,CAAC;wBACtD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,WAAW,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;oBAC7F,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC5C,CAAC;YACH,CAAC;YAED,+CAA+C;YAC/C,aAAa,CACX,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAC3B,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CACzC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA0B;IAC3D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,0BAA0B;IAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9E,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QAE7F,KAAK,CAAC,IAAI,CACR,SAAS,CACP,GAAG,QAAQ,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,UAAU,IAAI,WAAW,CAAC,SAAS,YAAY,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACnI,CACF,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACnF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,YAAY,GAAG,eAAe,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IAC9G,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,WAAW,IAAI,SAAS,YAAY,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEzG,MAAM,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC;IAC1G,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,MAAqB;IAErB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IAEnE,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,QAAQ,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,CAAC;IACpE,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAExD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/F,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,QAAgB,EAChB,SAAiB,EACjB,SAAiB;IAEjB,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,QAAQ,KAAK,SAAS,IAAI,SAAS,MAAM,CAAC,CAAC;AAC1E,CAAC"}
+{"version":3,"file":"results.js","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAC/F,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,KAAK,MAAM,OAAO,CAAC;AAS1B,OAAO,EAAE,eAAe,EAAmB,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,WAA2B;IAClE,gDAAgD;IAChD,MAAM,aAAa,GAAiC,EAAE,CAAC;IAEvD,0BAA0B;IAC1B,IAAI,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACnC,aAAa,CAAC,IAAI,GAAG,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;IACrD,CAAC;IAED,qEAAqE;IACrE,IAAI,WAAW,CAAC,cAAc,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrF,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,EAAE,CAAC;YACxE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAClB,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE;YACN,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACjD,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE,qBAAqB;SAC7D;QACD,UAAU,EAAE,WAAW,CAAC,UAAU;QAClC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;KACjF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,OAAsB;IACpE,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEnE,OAAO;QACL,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM;QACtB,UAAU;QACV,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAChE,YAAY,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,EAAE,OAAO;KACd,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACrC,MAAgC,EAChC,KAAoB,EACpB,SAAe,EACf,WAAiB;IAEjB,OAAO;QACL,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE;QACtC,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAkBD;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,WAAW,CACzB,OAA0B,EAC1B,OAA2B;IAE3B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvD,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;IAElF,8BAA8B;IAC9B,SAAS,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9C,wBAAwB;IACxB,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACtD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAExC,8CAA8C;QAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,cAAc,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,cAAc,GAA4B;YAC9C,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,UAAU,EAAE,WAAW,CAAC,UAAU;YAClC,QAAQ,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YAC/C,YAAY,EAAE,WAAW,CAAC,YAAY;SACvC,CAAC;QACF,IAAI,WAAW,EAAE,CAAC;YAChB,cAAc,CAAC,WAAW,GAAG,WAAW,CAAC;QAC3C,CAAC;QACD,IAAI,cAAc,EAAE,CAAC;YACnB,cAAc,CAAC,cAAc,GAAG,cAAc,CAAC;QACjD,CAAC;QACD,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACpB,cAAc,CAAC,KAAK,GAAG,KAAK,CAAC;QAC/B,CAAC;QACD,aAAa,CACX,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAC7B,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CACxC,CAAC;QAEF,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7C,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEvC,+CAA+C;YAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YACnC,MAAM,eAAe,GAAqD;gBACxE,GAAG,OAAO,CAAC,MAAM;gBACjB,KAAK;aACN,CAAC;YAEF,gCAAgC;YAChC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,2BAA2B;gBAC3B,MAAM,UAAU,GAAG,eAAe,CAChC,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,MAAM,CAAC,KAAK,EACpB,KAAK,CACN,CAAC;gBAEF,6DAA6D;gBAC7D,aAAa,CACX,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,EAC/B,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CACpC,CAAC;gBACF,eAAe,CAAC,cAAc,GAAG,mBAAmB,CAAC;gBAErD,2DAA2D;gBAC3D,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;gBACxE,eAAe,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;gBAE7D,kDAAkD;gBAClD,eAAe,CAAC,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC;YAC5C,CAAC;YAED,uCAAuC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3C,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC1B,MAAM,WAAW,GAAiC,EAAE,CAAC;gBAErD,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;oBAC/B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;oBACxE,WAAW,CAAC,IAAI,GAAG,oBAAoB,CAAC;gBAC1C,CAAC;gBAED,kFAAkF;gBAClF,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;oBAClC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC/C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC3C,WAAW,CAAC,OAAO,GAAG,EAAE,CAAC;oBACzB,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC5E,IAAI,OAAO,EAAE,CAAC;4BACZ,MAAM,QAAQ,GAAG,GAAG,IAAI,MAAM,CAAC;4BAC/B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;4BACnD,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,qBAAqB,QAAQ,EAAE,CAAC;wBAC9D,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,WAAW,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;oBAC7F,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC5C,CAAC;YACH,CAAC;YAED,+CAA+C;YAC/C,aAAa,CACX,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAC3B,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CACzC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA0B;IAC3D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,0BAA0B;IAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9E,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QAE7F,KAAK,CAAC,IAAI,CACR,SAAS,CACP,GAAG,QAAQ,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,UAAU,IAAI,WAAW,CAAC,SAAS,YAAY,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACnI,CACF,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACnF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,YAAY,GAAG,eAAe,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IAC9G,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,WAAW,IAAI,SAAS,YAAY,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEzG,MAAM,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC;IAC1G,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,MAAqB;IAErB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IAEnE,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,QAAQ,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,CAAC;IACpE,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAExD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/F,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,QAAgB,EAChB,SAAiB,EACjB,SAAiB;IAEjB,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,QAAQ,KAAK,SAAS,IAAI,SAAS,MAAM,CAAC,CAAC;AAC1E,CAAC;AAYD;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CACjC,UAAkB,EAClB,cAAsB,EACtB,YAAoC;IAEpC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IACnD,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAEvD,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,QAAQ,CAAC;IAEhD,0CAA0C;IAC1C,IAAI,UAAoB,CAAC;IACzB,IAAI,CAAC;QACH,UAAU,GAAG,WAAW,CAAC,aAAa,CAAC;aACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACjC,IAAI,EAAE;aACN,OAAO,EAAE,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE;YAAE,SAAS;QAE7C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,gDAAgD;YAChD,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YAEpC,+CAA+C;YAC/C,MAAM,mBAAmB,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,CAAC,mBAAmB;gBAAE,SAAS;YAEnC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;YACzD,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;gBAE/D,0BAA0B;gBAC1B,IAAI,OAAO,CAAC,WAAW,KAAK,mBAAmB;oBAAE,SAAS;gBAE1D,yEAAyE;gBACzE,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK;oBAAE,SAAS;gBAEtC,8CAA8C;gBAC9C,IAAI,OAAO,CAAC,UAAU,IAAI,CAAC;oBAAE,SAAS;gBAEtC,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE;oBACpB,QAAQ,EAAE,OAAO;oBACjB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,SAAS;iBACV,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,yBAAyB;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/lib/types.d.ts

@@ -182,6 +182,17 @@ export interface EvalSummary {
     /** Individual run data (internal, not all fields saved to summary.json) */
     runs: EvalRunData[];
 }
+/**
+ * Failure classification for a failed eval run.
+ */
+export type FailureType = 'model' | 'infra' | 'timeout';
+/**
+ * Classification result for a failed eval.
+ */
+export interface Classification {
+    failureType: FailureType;
+    failureReason: string;
+}
 /**
  * Complete experiment results.
  */

package/dist/lib/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,GACP,4BAA4B,CAAC;AAEjC;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GACzC,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB;;6FAEyF;IACzF,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAEhC,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;IAEtB,qFAAqF;IACrF,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;IAElC,+FAA+F;IAC/F,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAC/B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACzC;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,uEAAwE,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8DAA8D;IAC9D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,GACP,4BAA4B,CAAC;AAEjC;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GACzC,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB;;6FAEyF;IACzF,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAEhC,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;IAEtB,qFAAqF;IACrF,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;IAElC,+FAA+F;IAC/F,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAC/B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACzC;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,uEAAwE,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8DAA8D;IAC9D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,WAAW,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vercel/agent-eval",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Framework for testing AI coding agents in isolated sandboxes",
   "repository": {
     "type": "git",