@m4trix/evals 0.4.0 → 0.7.0

package/README.md

@@ -116,7 +116,8 @@ export const myTestCase = TestCase.describe({
   tags: ['demo'],
   inputSchema: S.Struct({ prompt: S.String }),
   input: { prompt: 'Hello from my first eval' },
-  outputDefinition: { expectedMinScore: 50 },
+  outputSchema: S.Struct({ expectedMinScore: S.Number }),
+  output: { expectedMinScore: 50 },
 });
 ```
 

package/dist/cli-simple.cjs

@@ -416,12 +416,12 @@ function normalizeResult(result) {
   const metrics = Array.isArray(obj.metrics) ? obj.metrics : void 0;
   return { scores, metrics };
 }
-function readOutputDefinition(testCase) {
+function readOutput(testCase) {
   const candidate = testCase;
-  if (typeof candidate.getOutputDefinition !== "function") {
+  if (typeof candidate.getOutput !== "function") {
     return void 0;
   }
-  return candidate.getOutputDefinition();
+  return candidate.getOutput();
 }
 function nowIsoForFile() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -451,7 +451,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
     const started = Date.now();
     const evaluatorScores = [];
     let testCaseError;
-    const outputDefinition = readOutputDefinition(testCaseItem.testCase);
+    const output = readOutput(testCaseItem.testCase);
     for (const { id: evaluatorId, evaluator } of task.evaluators) {
       const evaluateFn = evaluator.getEvaluateFn();
       if (!evaluateFn) {
@@ -466,7 +466,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
             evaluateFn({
               input: testCaseItem.testCase.getInput(),
               ctx,
-              output: outputDefinition
+              output
             })
           )
         );
@@ -499,7 +499,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => e
       passed: testCasePassed,
       durationMs: Date.now() - started,
       evaluatorScores,
-      outputDefinition,
+      output,
       errorMessage: testCaseError
     };
     updateSnapshot(task.runId, (snapshot) => ({
@@ -633,9 +633,18 @@ function mergeRunnerOverrides(base, next) {
   if (!base) {
     return next;
   }
-  {
+  if (!next) {
     return base;
   }
+  const discovery = base.discovery || next.discovery ? {
+    ...base.discovery ?? {},
+    ...next.discovery ?? {}
+  } : void 0;
+  return {
+    ...base,
+    ...next,
+    discovery
+  };
 }
 function createRunner(overrides) {
   const fileOverrides = loadRunnerConfigFile();
@@ -880,11 +889,11 @@ function getSimpleCliUsage() {
     '  "/score/i"            regex literal'
   ].join("\n");
 }
-function readOutputDefinition2(testCase) {
-  if (typeof testCase.getOutputDefinition !== "function") {
+function readOutput2(testCase) {
+  if (typeof testCase.getOutput !== "function") {
     return void 0;
   }
-  return testCase.getOutputDefinition();
+  return testCase.getOutput();
 }
 function createOutputPath(datasetFilePath) {
   const parsed = path.parse(datasetFilePath);
@@ -899,7 +908,7 @@ async function generateDatasetJsonCommand(runner, datasetName) {
   const payload = testCases.map((item) => ({
     name: item.testCase.getName(),
     input: item.testCase.getInput(),
-    outputDefinition: readOutputDefinition2(item.testCase)
+    output: readOutput2(item.testCase)
   }));
   const absoluteDatasetPath = path.resolve(process.cwd(), dataset.filePath);
   const outputPath = createOutputPath(absoluteDatasetPath);

package/dist/cli-simple.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/runner/api.ts","../src/runner/config.ts","../src/runner/config-loader.ts","../src/runner/discovery.ts","../src/runner/execution.ts","../src/evals/metric.ts","../src/evals/score.ts","../src/evals/metrics/standard.ts","../src/evals/scores/standard.ts","../src/runner/score-utils.ts","../src/runner/persistence.ts","../src/runner/search.ts","../src/cli-simple/args.ts","../src/cli-simple/generate.ts","../src/cli-simple/run.ts","../src/cli-simple/index.ts"],"names":["Effect","Queue","createJiti","resolve","jitiModule","loaded","registry","join","readOutputDefinition"],"mappings":";;;AAAA,SAAS,kBAAkB;AAE3B,SAAS,UAAAA,SAAQ,OAAO,QAAQ,SAAAC,cAAa;;;ACyCtC,IAAM,sBAAoC;AAAA,EAC/C,WAAW;AAAA,IACT,SAAS,QAAQ,IAAI;AAAA,IACrB,iBAAiB,CAAC,eAAe,gBAAgB,eAAe,cAAc;AAAA,IAC9E,mBAAmB;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,IACA,kBAAkB;AAAA,MAChB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,IACA,oBAAoB,CAAC,gBAAgB,QAAQ,SAAS,QAAQ,aAAa;AAAA,EAC7E;AAAA,EACA,mBAAmB;AACrB;AAEO,SAAS,wBACd,QACmC;AACnC,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,EACT;AAEA,QAAM,eAAe,OAAO;AAC5B,QAAM,YAA4C,CAAC;AACnD,MAAI,cAAc,YAAY,QAAW;AACvC,cAAU,UAAU,aAAa;AAAA,EACnC;AACA,MAAI,cAAc,wBAAwB,QAAW;AACnD,cAAU,kBAAkB,aAAa;AAAA,EAC3C,WAAW,cAAc,oBAAoB,QAAW;AACtD,cAAU,kBAAkB,aAAa;AAAA,EAC3C;AACA,MAAI,cAAc,0BAA0B,QAAW;AACrD,cAAU,oBAAoB,aAAa;AAAA,EAC7C,WAAW,cAAc,sBAAsB,QAAW;AACxD,cAAU,oBAAoB,aAAa;AAAA,EAC7C;AACA,MAAI,cAAc,yBAAyB,QAAW;AACpD,cAAU,mBAAmB,aAAa;AAAA,EAC5C,WAAW,cAAc,qBAAqB,QAAW;AACvD,cAAU,mBAAmB,aAAa;AAAA,EAC5C;AACA,MAAI,cAAc,uBAAuB,QAAW;AAClD,cAAU,qBAAqB,aAAa;AAAA,EAC9C;AAEA,QAAM,YAAmC,CAAC;AAC1C,MAAI,OAAO,sBAAsB,QAAW;AAC1C,cAAU,oBAAoB,OAAO;AAAA,EACvC;AACA,MAAI,OAAO,KAAK,SAAS,EAAE,SAAS,GAAG;AACrC,cAAU,YAAY;AAAA,EACxB;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,WAAiD;AAChF,MAAI,CAAC,WAAW;AACd,WAAO;AAAA,EACT;AACA,QAAM,YAAY,UAAU,YACxB;AAAA,IACE,GAAG,oBAAoB;AAAA,IACvB,GAAG,UAAU;AAAA,EACf,IACA,oBAAoB;AAExB,SAAO;AAAA,IACL,GAAG;AAAA,IACH,GAAG;AAAA,IACH;AAAA,EACF;AACF;;;ACzHA,SAAS,kBAAkB;AAC3B,SAAS,eAAe;AAExB,YAAY,gBAAgB;AAS5B,IAAM,mBAAmB;AAOzB,IAAI;AAEJ,SAAS,gBAA4B;AACnC,MAAI,cAAc;AAChB,WAAO;AAAA,EACT;AACA,QAAMC,cACwD,yBACtB;AACxC,MAAI,OAAOA,gBAAe,YAAY;AACpC,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,iBACEA,YACA,YAAY,KAAK;AAAA,IACjB,gBAAgB;AAAA,IAChB,aAAa;AAAA,EACf,CAAC;AACD,SAAO;AACT;AAEA,SAAS,0BAA0B,cAAgC;AACjE,MACE,gBACA,OAAO,iBAAiB,YACxB,aAAa,cACb;AACA,WAAQ,aAAsC;AAAA,EAChD;AACA,SAAO;AACT;AAEA,SAAS,mBAAmB,OAAwC;AAClE,MAAI,UAAU,UAAa,UAAU,MAAM;AACzC,WAAO;AAAA,EACT;AACA,MAAI,OAAO,UAAU,YAAY;AAC/B,WAAQ,MAA8C;AAAA,EACxD;AACA,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,qBACd,MAAM,QAAQ,IAAI,GACiB;AACnC,QAAM,aAAa,QAAQ,KAAK,gBAAgB;AAChD,MAAI,CAAC,WAAW,UAAU,GAAG;AAC3B,WAAO;AAAA,EACT;AACA,QAAM,SAAS,cAAc;AAC7B,QAAM,SAAS,OAAO,UAAU;AAChC,QAAM,gBAAgB,0BAA0B,MAAM;AACtD,QAAM,SAAS,mBAAmB,aAAa;AAC/C,SAAO,wBAAwB,MAAM;AACvC;;;AC/EA,SAAS,eAAe;AACxB,SAAS,WAAAC,UAAS,gBAAgB;AAClC,SAAS,qBAAqB;AAiB9B,IAAI;AAEJ,SAAS,KAAK,QAAgB,UAAkB,MAAuB;AACrE,QAAM,SAAS,QAAQ,KAAK,KAAK,EAAE,SAAS,IAAI,OAAO;AACvD,SAAO,GAAG,MAAM,IAAI,MAAM,GACvB,YAAY,EACZ,QAAQ,eAAe,GAAG,EAC1B,QAAQ,YAAY,EAAE;AAC3B;AAEA,SAAS,UAAU,OAAgB,YAA6B;AAC9D,SACE,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,OAAQ,MAAkC,UAAU,MAAM;AAE9D;AAEA,SAAS,cAAc,OAAkC;AACvD,SAAO,UAAU,OAAO,SAAS,KAAK,UAAU,OAAO,iBAAiB;AAC1E;AAEA,SAAS,gBACP,OACwD;AACxD,SACE,UAAU,OAAO,SAAS,KAC1B,UAAU,OAAO,gBAAgB,KACjC,UAAU,OAAO,eAAe;AAEpC;AAEA,SAAS,eAAe,OAA4C;AAClE,SACE,UAAU,OAAO,SAAS,KAC1B,UAAU,OAAO,SAAS,KAC1B,UAAU,OAAO,UAAU;AAE/B;AAEA,eAAe,cACb,SACA,oBACmB;AACnB,QAAM,MAAgB,CAAC;AAEvB,iBAAe,KAAK,YAAmC;AACrD,QAAI;AACJ,QAAI;AACF,gBAAU,MAAM,QAAQ,YAAY,EAAE,eAAe,KAAK,CAAC;AAAA,IAC7D,QAAQ;AACN;AAAA,IACF;AAEA,UAAM,QAAQ;AAAA,MACZ,QAAQ,IAAI,OAAO,UAAU;AAC3B,cAAM,WAAWA,SAAQ,YAAY,MAAM,IAAI;AAC/C,YAAI,MAAM,YAAY,GAAG;AACvB,cAAI,mBAAmB,SAAS,MAAM,IAAI,GAAG;AAC3C;AAAA,UACF;AACA,gBAAM,KAAK,QAAQ;AACnB;AAAA,QACF;AAEA,YAAI,MAAM,OAAO,GAAG;AAClB,cAAI,KAAK,QAAQ;AAAA,QACnB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,KAAK,OAAO;AAClB,SAAO;AACT;AAEA,SAAS,aACP,UACA,UACS;AACT,SAAO,SAAS,KAAK,CAAC,WAAW,SAAS,SAAS,MAAM,CAAC;AAC5D;AAEA,eAAe,kBAAkB,UAAsC;AACrE,MAAI,SAAS,SAAS,KAAK,KAAK,SAAS,SAAS,MAAM,GAAG;AACzD,QAAI,CAAC,YAAY;AACf,YAAMC,cAAc,MAAM,OAAO,MAAM;AAIvC,YAAMF,cAAaE,YAAW,cAAcA,YAAW;AACvD,UAAI,CAACF,aAAY;AACf,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AACA,mBAAaA,YAAW,YAAY,KAAK;AAAA,QACvC,gBAAgB;AAAA,QAChB,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AACA,UAAMG,UAAS,WAAW,SACtB,MAAM,WAAW,OAAO,QAAQ,IAChC,MAAM,QAAQ,QAAQ,WAAW,QAAQ,CAAC;AAC9C,WAAO,OAAO,OAAOA,OAAiC;AAAA,EACxD;AAEA,QAAM,YAAY,cAAc,QAAQ,EAAE;AAC1C,QAAM,SAAU,MAAM,OAAO;AAC7B,SAAO,OAAO,OAAO,MAAM;AAC7B;AAEA,eAAsB,yBACpB,QAC0C;AAC1C,QAAM,QAAQ,MAAM,cAAc,OAAO,SAAS,OAAO,kBAAkB;AAC3E,QAAM,UAAU,MAAM;AAAA,IAAO,CAAC,aAC5B,aAAa,UAAU,OAAO,eAAe;AAAA,EAC/C;AAEA,QAAM,QAAQ,MAAM,QAAQ;AAAA,IAC1B,QAAQ,IAAI,OAAO,iBAAiB;AAClC,YAAM,UAAU,MAAM,kBAAkB,YAAY;AACpD,YAAM,WAAW,QAAQ,OAAO,aAAa;AAC7C,YAAM,UAAU,SAAS,OAAO,SAAS,YAAY;AACrD,aAAO,SAAS,IAAI,CAAC,aAAa;AAAA,QAChC,IAAI,KAAK,WAAW,SAAS,QAAQ,QAAQ,CAAC;AAAA,QAC9C,UAAU;AAAA,QACV;AAAA,MACF,EAAE;AAAA,IACJ,CAAC;AAAA,EACH;AAEA,SAAO,MAAM,KAAK;AACpB;AAEA,eAAsB,2BACpB,QAC4C;AAC5C,QAAM,QAAQ,MAAM,cAAc,OAAO,SAAS,OAAO,kBAAkB;AAC3E,QAAM,UAAU,MAAM;AAAA,IAAO,CAAC,aAC5B,aAAa,UAAU,OAAO,iBAAiB;AAAA,EACjD;AAEA,QAAM,QAAQ,MAAM,QAAQ;AAAA,IAC1B,QAAQ,IAAI,OAAO,iBAAiB;AAClC,YAAM,UAAU,MAAM,kBAAkB,YAAY;AACpD,YAAM,aAAa,QAAQ,OAAO,eAAe;AACjD,YAAM,UAAU,SAAS,OAAO,SAAS,YAAY;AACrD,aAAO,WAAW,IAAI,CAAC,eAAe;AAAA,QACpC,IAAI,KAAK,aAAa,SAAS,UAAU,QAAQ,CAAC;AAAA,QAClD,UAAU;AAAA,QACV;AAAA,MACF,EAAE;AAAA,IACJ,CAAC;AAAA,EACH;AAEA,SAAO,MAAM,KAAK;AACpB;AAEA,eAAsB,0BACpB,QAC2C;AAC3C,QAAM,QAAQ,MAAM,cAAc,OAAO,SAAS,OAAO,kBAAkB;AAC3E,QAAM,UAAU,MAAM;AAAA,IAAO,CAAC,aAC5B,aAAa,UAAU,OAAO,gBAAgB;AAAA,EAChD;AAEA,QAAM,QAAQ,MAAM,QAAQ;AAAA,IAC1B,QAAQ,IAAI,OAAO,iBAAiB;AAClC,YAAM,UAAU,MAAM,kBAAkB,YAAY;AACpD,YAAM,YAAY,QAAQ,OAAO,cAAc;AAC/C,YAAM,UAAU,SAAS,OAAO,SAAS,YAAY;AACrD,aAAO,UAAU,IAAI,CAAC,cAAc;AAAA,QAClC,IAAI,KAAK,aAAa,SAAS,SAAS,QAAQ,CAAC;AAAA,QACjD,UAAU;AAAA,QACV;AAAA,MACF,EAAE;AAAA,IACJ,CAAC;AAAA,EACH;AAEA,SAAO,MAAM,KAAK;AACpB;;;ACzMA,SAAS,YAAY;AAErB,SAAS,QAAQ,aAAa;;;ACF9B,IAAM,WAAW,oBAAI,IAAgC;AAc9C,IAAM,SAAS;AAAA,EACpB,GAAU,QAIW;AACnB,UAAM,MAAwB;AAAA,MAC5B,IAAI,OAAO;AAAA,MACX,MAAM,OAAO;AAAA,MACb,QAAQ,OAAO;AAAA,MACf,MAAM,CAAC,UAAiB,EAAE,IAAI,OAAO,IAAI,KAAK;AAAA,IAChD;AACA,aAAS,IAAI,OAAO,IAAI,GAAyB;AACjD,WAAO;AAAA,EACT;AACF;AAEO,SAAS,cAAc,IAA4C;AACxE,SAAO,SAAS,IAAI,EAAE;AACxB;;;ACjCA,IAAMC,YAAW,oBAAI,IAA+B;AAqB7C,IAAM,QAAQ;AAAA,EACnB,GAAU,QAKU;AAClB,UAAM,MAAuB;AAAA,MAC3B,IAAI,OAAO;AAAA,MACX,MAAM,OAAO;AAAA,MACb,iBAAiB,OAAO;AAAA,MACxB,QAAQ,OAAO;AAAA,MACf,MAAM,CAAC,MAAa,YAA0D;AAC5E,cAAM,SACJ,SAAS,iBAAiB,SACtB,QAAQ,aAAa,IAAI,IACzB;AACN,eAAO;AAAA,UACL,IAAI,OAAO;AAAA,UACX;AAAA,UACA,GAAI,WAAW,UAAa,EAAE,OAAO;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AACA,IAAAA,UAAS,IAAI,OAAO,IAAI,GAAwB;AAChD,WAAO;AAAA,EACT;AACF;AAEO,SAAS,aAAa,IAA2C;AACtE,SAAOA,UAAS,IAAI,EAAE;AACxB;;;AC3CO,IAAM,mBAAmB,OAAO,GAAmB;AAAA,EACxD,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,QAAQ,CAAC,SAAS;AAChB,UAAM,QAAQ,KAAK,SAAS;AAC5B,UAAM,SAAS,KAAK,UAAU;AAC9B,UAAM,cAAc,KAAK,eAAe;AACxC,UAAM,eAAe,KAAK,gBAAgB;AAC1C,UAAM,SAAS,cAAc;AAC7B,WAAO,MAAM,KAAK,QAAQ,MAAM,WAAW,MAAM;AAAA,EACnD;AACF,CAAC;AAMM,IAAM,gBAAgB,OAAO,GAAgB;AAAA,EAClD,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,QAAQ,CAAC,SAAS,GAAG,KAAK,EAAE;AAC9B,CAAC;;;ACxBM,IAAM,eAAe,MAAM,GAAqB;AAAA,EACrD,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,iBAAiB;AAAA,EACjB,QAAQ,CAAC,SAAS,KAAK,MAAM,QAAQ,CAAC;AACxC,CAAC;AAMM,IAAM,cAAc,MAAM,GAAoB;AAAA,EACnD,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,iBAAiB;AAAA,EACjB,QAAQ,CAAC,SAAU,KAAK,SAAS,WAAW;AAC9C,CAAC;;;ACnBM,SAAS,yBACd,QACoB;AACpB,aAAW,QAAQ,QAAQ;AACzB,UAAM,MAAM,aAAa,KAAK,EAAE;AAChC,QAAI,OAAO,IAAI,oBAAoB,SAAS,OAAO,KAAK,SAAS,YAAY,KAAK,SAAS,QAAQ,WAAW,KAAK,MAAM;AACvH,YAAM,QAAS,KAAK,KAA4B;AAChD,UAAI,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,GAAG;AACvD,eAAO;AAAA,MACT;AAAA,IACF;AACA,UAAM,UAAU,eAAe,KAAK,IAAI;AACxC,QAAI,YAAY,QAAW;AACzB,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,eAAe,OAAoC;AACjE,MAAI,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,GAAG;AACvD,WAAO;AAAA,EACT;AACA,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AACA,QAAM,MAAM;AACZ,MACE,WAAW,OACX,OAAO,IAAI,UAAU,YACrB,OAAO,SAAS,IAAI,KAAK,GACzB;AACA,WAAO,IAAI;AAAA,EACb;AACA,QAAM,eAAe,OAAO,OAAO,KAAK,EAAE;AAAA,IACxC,CAAC,UACC,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAAA,EACtD;AACA,MAAI,aAAa,WAAW,GAAG;AAC7B,WAAO;AAAA,EACT;AACA,SACE,aAAa,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI,aAAa;AAEvE;;;ALnCA,SAAS,uBACP,WACA,QACA,QACS;AACT,QAAM,mBAAmB,OAAO,OAAO,CAAC,MAAM,YAAY,KAAK,EAAE,WAAW,MAAS;AACrF,MAAI,iBAAiB,SAAS,GAAG;AAC/B,WAAO,iBAAiB,MAAM,CAAC,MAAM,EAAE,WAAW,IAAI;AAAA,EACxD;AACA,QAAM,gBAAgB,UAAU,iBAAiB;AACjD,MAAI,eAAe;AACjB,WAAO,cAAc,MAAM;AAAA,EAC7B;AACA,QAAM,gBAAgB,UAAU,iBAAiB;AACjD,MAAI,kBAAkB,QAAW;AAC/B,UAAM,UAAU,yBAAyB,MAAM;AAC/C,WAAO,YAAY,UAAa,WAAW;AAAA,EAC7C;AACA,SAAO;AACT;AAEA,SAAS,gBACP,QAIA;AACA,MAAI,OAAO,WAAW,YAAY,WAAW,MAAM;AACjD,WAAO,EAAE,QAAQ,CAAC,EAAE;AAAA,EACtB;AACA,QAAM,MAAM;AACZ,QAAM,SAAS,MAAM,QAAQ,IAAI,MAAM,IAClC,IAAI,SACL,CAAC;AACL,QAAM,UAAU,MAAM,QAAQ,IAAI,OAAO,IACpC,IAAI,UACL;AACJ,SAAO,EAAE,QAAQ,QAAQ;AAC3B;AAEA,SAAS,qBAAqB,UAAkD;AAC9E,QAAM,YAAY;AAGlB,MAAI,OAAO,UAAU,wBAAwB,YAAY;AACvD,WAAO;AAAA,EACT;AACA,SAAO,UAAU,oBAAoB;AACvC;AAcA,SAAS,gBAAwB;AAC/B,UAAO,oBAAI,KAAK,GAAE,YAAY,EAAE,QAAQ,SAAS,GAAG;AACtD;AAEO,SAAS,mBACd,mBACA,WACA,OACQ;AACR,SAAO;AAAA,IACL;AAAA,IACA,GAAG,SAAS,IAAI,KAAK,IAAI,cAAc,CAAC;AAAA,EAC1C;AACF;AAEO,IAAM,iBAAiB,CAC5B,MACA,cACA,kBACA,mBAKA,OAAO,IAAI,aAAa;AACtB,QAAM,YAAY,KAAK,IAAI;AAC3B,iBAAe,KAAK,OAAO,CAAC,cAAc;AAAA,IACxC,GAAG;AAAA,IACH,QAAQ;AAAA,IACR;AAAA,EACF,EAAE;AACF,SAAO,aAAa;AAAA,IAClB,MAAM;AAAA,IACN,OAAO,KAAK;AAAA,IACZ;AAAA,EACF,CAAC;AAED,MAAI,qBAAqB;AACzB,MAAI,kBAAkB;AACtB,MAAI,kBAAkB;AAEtB,aAAW,gBAAgB,KAAK,WAAW;AACzC,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,kBAKD,CAAC;AACN,QAAI;AACJ,UAAM,mBAAmB,qBAAqB,aAAa,QAAQ;AAEnE,eAAW,EAAE,IAAI,aAAa,UAAU,KAAK,KAAK,YAAY;AAC5D,YAAM,aAAa,UAAU,cAAc;AAC3C,UAAI,CAAC,YAAY;AACf;AAAA,MACF;AAEA,UAAI;AACF,cAAM,MAAM,OAAO,OAAO;AAAA,UAAQ,MAChC,QAAQ,QAAQ,UAAU,eAAe,CAAC;AAAA,QAC5C;AACA,cAAM,SAAS,OAAO,OAAO;AAAA,UAAQ,MACnC,QAAQ;AAAA,YACN,WAAW;AAAA,cACT,OAAO,aAAa,SAAS,SAAS;AAAA,cACtC;AAAA,cACA,QAAQ;AAAA,YACV,CAAC;AAAA,UACH;AAAA,QACF;AACA,cAAM,EAAE,QAAQ,QAAQ,IAAI,gBAAgB,MAAM;AAClD,cAAM,SAAS,uBAAuB,WAAW,QAAQ,MAAM;AAC/D,wBAAgB,KAAK,EAAE,aAAa,QAAQ,QAAQ,QAAQ,CAAC;AAAA,MAC/D,SAAS,OAAO;AACd,wBACE,iBAAiB,QACb,MAAM,UACN;AACN,wBAAgB,KAAK;AAAA,UACnB;AAAA,UACA,QAAQ,CAAC;AAAA,UACT,QAAQ;AAAA,QACV,CAAC;AAAA,MACH;AAAA,IACF;AAEA,UAAM,iBAAiB,gBAAgB,MAAM,CAAC,MAAM,EAAE,MAAM;AAC5D,0BAAsB;AACtB,QAAI,gBAAgB;AAClB,yBAAmB;AAAA,IACrB,OAAO;AACL,yBAAmB;AAAA,IACrB;AAEA,UAAM,gBAA6B;AAAA,MACjC,MAAM;AAAA,MACN,OAAO,KAAK;AAAA,MACZ,YAAY,aAAa;AAAA,MACzB,cAAc,aAAa,SAAS,QAAQ;AAAA,MAC5C;AAAA,MACA,gBAAgB,KAAK,UAAU;AAAA,MAC/B,QAAQ;AAAA,MACR,YAAY,KAAK,IAAI,IAAI;AAAA,MACzB;AAAA,MACA;AAAA,MACA,cAAc;AAAA,IAChB;AAEA,mBAAe,KAAK,OAAO,CAAC,cAAc;AAAA,MACxC,GAAG;AAAA,MACH;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE;AAEF,WAAO,aAAa,aAAa;AACjC,WAAO,MAAM,MAAM,kBAAkB;AAAA,MACnC,OAAO,KAAK;AAAA,MACZ,cAAc,KAAK,SAAS;AAAA,MAC5B,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,QAAM,aAAa,KAAK,IAAI;AAC5B,QAAM,iBAA8B;AAAA,IAClC,MAAM;AAAA,IACN,OAAO,KAAK;AAAA,IACZ;AAAA,IACA;AAAA,IACA;AAAA,IACA,gBAAgB,KAAK,UAAU;AAAA,IAC/B,cAAc,KAAK,SAAS;AAAA,EAC9B;AAEA,iBAAe,KAAK,OAAO,CAAC,cAAc;AAAA,IACxC,GAAG;AAAA,IACH,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE;AAEF,SAAO,aAAa,cAAc;AAClC,SAAO,MAAM,MAAM,kBAAkB;AAAA,IACnC,OAAO,KAAK;AAAA,IACZ,cAAc,KAAK,SAAS;AAAA,IAC5B,SAAS;AAAA,EACX,CAAC;AACD,SAAO,aAAa;AAAA,IAClB,MAAM;AAAA,IACN,OAAO,KAAK;AAAA,IACZ,cAAc,KAAK,SAAS;AAAA,EAC9B,CAAC;AACH,CAAC;;;AMrOH,SAAS,YAAY,aAAa;AAClC,SAAS,eAAe;AAExB,SAAS,UAAAN,SAAQ,SAAAC,cAAa;AAQ9B,eAAe,eACb,cACA,SACe;AACf,QAAM,MAAM,QAAQ,YAAY,GAAG,EAAE,WAAW,KAAK,CAAC;AACtD,QAAM,WAAW,cAAc,GAAG,KAAK,UAAU,OAAO,CAAC;AAAA,GAAM,MAAM;AACvE;AAEO,IAAM,0BAA0B,CACrC,UAEAD,QAAO;AAAA,EACLA,QAAO,IAAI,aAAa;AACtB,UAAM,UAAU,OAAOC,OAAM,KAAK,KAAK;AACvC,WAAOD,QAAO;AAAA,MAAQ,MACpB,eAAe,QAAQ,cAAc;AAAA,QACnC,OAAO,QAAQ;AAAA,QACf,IAAI,KAAK,IAAI;AAAA,QACb,GAAG,QAAQ;AAAA,MACb,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACH;;;AC/BF,SAAS,WACP,OACA,UACS;AACT,MAAI,CAAC,YAAY,SAAS,WAAW,GAAG;AACtC,WAAO;AAAA,EACT;AACA,SAAO,SAAS;AAAA,IAAK,CAAC,YACpB,OAAO,YAAY,WAAW,YAAY,QAAQ,QAAQ,KAAK,KAAK;AAAA,EACtE;AACF;AAEA,SAAS,YACP,OACA,UACS;AACT,MAAI,CAAC,YAAY,SAAS,WAAW,GAAG;AACtC,WAAO;AAAA,EACT;AACA,SAAO,SAAS,KAAK,CAAC,YAAY;AAChC,QAAI,OAAO,YAAY,UAAU;AAC/B,aAAO,MAAM,SAAS,OAAO;AAAA,IAC/B;AACA,WAAO,QAAQ,KAAK,KAAK;AAAA,EAC3B,CAAC;AACH;AAEO,SAAS,yBACd,KACA,OACkC;AAClC,MAAI,CAAC,OAAO;AACV,WAAO;AAAA,EACT;AAEA,SAAO,IAAI,OAAO,CAAC,SAAS;AAC1B,UAAM,OAAO,KAAK,SAAS,QAAQ;AAEnC,QACE,MAAM,gBACN,KAAK,KAAK,CAAC,QAAQ,WAAW,KAAK,MAAM,YAAY,CAAC,GACtD;AACA,aAAO;AAAA,IACT;AACA,QACE,MAAM,iBACN,YAAY,KAAK,UAAU,MAAM,aAAa,GAC9C;AACA,aAAO;AAAA,IACT;AAEA,UAAM,oBACJ,CAAC,MAAM,gBACP,MAAM,aAAa,WAAW,KAC9B,KAAK,KAAK,CAAC,QAAQ,WAAW,KAAK,MAAM,YAAY,CAAC;AAExD,UAAM,qBACJ,CAAC,MAAM,iBACP,MAAM,cAAc,WAAW,KAC/B,YAAY,KAAK,UAAU,MAAM,aAAa;AAEhD,WAAO,qBAAqB;AAAA,EAC9B,CAAC;AACH;;;AXpCA,SAAS,kBACP,SAC+C;AAC/C,MAAI,CAAC,QAAQ,WAAW,GAAG,GAAG;AAC5B,WAAO;AAAA,EACT;AACA,QAAM,YAAY,QAAQ,YAAY,GAAG;AACzC,MAAI,aAAa,GAAG;AAClB,WAAO;AAAA,EACT;AACA,SAAO;AAAA,IACL,QAAQ,QAAQ,MAAM,GAAG,SAAS;AAAA,IAClC,OAAO,QAAQ,MAAM,YAAY,CAAC;AAAA,EACpC;AACF;AAEA,SAAS,kBAAkB,SAA6C;AACtE,QAAM,oBAAoB,QAAQ,KAAK;AACvC,QAAM,eAAe,kBAAkB,iBAAiB;AACxD,MAAI,cAAc;AAChB,UAAM,QAAQ,IAAI,OAAO,aAAa,QAAQ,aAAa,KAAK;AAChE,WAAO,CAAC,UAAkB,MAAM,KAAK,KAAK;AAAA,EAC5C;AAEA,MAAI,kBAAkB,SAAS,GAAG,GAAG;AACnC,UAAM,UAAU,kBACb,QAAQ,qBAAqB,MAAM,EACnC,QAAQ,OAAO,IAAI;AACtB,UAAM,QAAQ,IAAI,OAAO,IAAI,OAAO,KAAK,GAAG;AAC5C,WAAO,CAAC,UAAkB,MAAM,KAAK,KAAK;AAAA,EAC5C;AAEA,SAAO,CAAC,UAAkB,MAAM,YAAY,MAAM,kBAAkB,YAAY;AAClF;AAuBA,SAAS,qBACP,MACA,MACmC;AACnC,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,QAAM,YAAY,KAAK,aAAa,KAAK,YACrC;AAAA,IACE,GAAI,KAAK,aAAa,CAAC;AAAA,IACvB,GAAI,KAAK,aAAa,CAAC;AAAA,EACzB,IACA;AACJ,SAAO;AAAA,IACL,GAAG;AAAA,IACH,GAAG;AAAA,IACH;AAAA,EACF;AACF;AAEO,SAAS,aAAa,WAA8C;AACzE,QAAM,gBAAgB,qBAAqB;AAC3C,QAAM,SAAS,qBAAqB,eAAe,SAAS;AAC5D,SAAO,IAAI,aAAa,iBAAiB,MAAM,CAAC;AAClD;AAEA,IAAM,eAAN,MAAwC;AAAA,EAiCtC,YAAY,QAAsB;AA9BlC,SAAiB,WAAWA,QAAO,QAAQ,OAAO,UAAuB,CAAC;AAE1E,SAAiB,WAAWA,QAAO,QAAQC,OAAM,UAAmB,CAAC;AAErE,SAAiB,mBAAmBD,QAAO;AAAA,MACzCC,OAAM,UAIH;AAAA,IACL;AAEA,SAAiB,YAAY,oBAAI,IAAyB;AAC1D,SAAiB,YAAY,oBAAI,IAG9B;AAEH,SAAiB,eAAe,oBAAI,IAA8B;AAElE,SAAiB,iBAAiB,oBAAI,IAAgC;AAEtE,SAAiB,iBAAiBD,QAAO;AAAA,MACvC,KAAK,sBAAsB;AAAA,IAC7B;AAEA,SAAiB,mBAAmBA,QAAO;AAAA,MACzC,wBAAwB,KAAK,gBAAgB;AAAA,IAC/C;AAGE,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,kBAA4D;AAChE,UAAM,WAAW,MAAM,yBAAyB,KAAK,OAAO,SAAS;AACrE,SAAK,aAAa,MAAM;AACxB,eAAW,WAAW,UAAU;AAC9B,WAAK,aAAa,IAAI,QAAQ,IAAI,OAAO;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAAgE;AACpE,UAAM,aAAa,MAAM,2BAA2B,KAAK,OAAO,SAAS;AACzE,SAAK,eAAe,MAAM;AAC1B,eAAW,aAAa,YAAY;AAClC,WAAK,eAAe,IAAI,UAAU,IAAI,SAAS;AAAA,IACjD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,qBAAqB,MAAqD;AAC9E,QAAI,KAAK,aAAa,SAAS,GAAG;AAChC,YAAM,KAAK,gBAAgB;AAAA,IAC7B;AACA,UAAM,aAAa,KAAK,KAAK,EAAE,YAAY;AAC3C,WAAO,MAAM,KAAK,KAAK,aAAa,OAAO,CAAC,EAAE;AAAA,MAC5C,CAAC,SAAS,KAAK,QAAQ,QAAQ,EAAE,YAAY,MAAM;AAAA,IACrD;AAAA,EACF;AAAA,EAEA,MAAM,+BACJ,SAC4C;AAC5C,QAAI,KAAK,eAAe,SAAS,GAAG;AAClC,YAAM,KAAK,kBAAkB;AAAA,IAC/B;AACA,UAAM,UAAU,kBAAkB,OAAO;AACzC,WAAO,MAAM,KAAK,KAAK,eAAe,OAAO,CAAC,EAAE;AAAA,MAAO,CAAC,SACtD,QAAQ,KAAK,UAAU,QAAQ,KAAK,EAAE;AAAA,IACxC;AAAA,EACF;AAAA,EAEA,MAAM,gBACJ,OAC2C;AAC3C,UAAM,YAAY,MAAM,0BAA0B,KAAK,OAAO,SAAS;AACvE,WAAO,yBAAyB,WAAW,KAAK;AAAA,EAClD;AAAA,EAEA,MAAM,wBACJ,WAC2C;AAC3C,QAAI,KAAK,aAAa,SAAS,GAAG;AAChC,YAAM,KAAK,gBAAgB;AAAA,IAC7B;AACA,UAAM,UAAU,KAAK,aAAa,IAAI,SAAS;AAC/C,QAAI,CAAC,SAAS;AACZ,YAAM,IAAI,MAAM,oBAAoB,SAAS,EAAE;AAAA,IACjD;AACA,UAAM,eAAe,MAAM,0BAA0B,KAAK,OAAO,SAAS;AAC1E,WAAO,aAAa;AAAA,MAAO,CAAC,aAC1B,QAAQ,QAAQ,gBAAgB,SAAS,UAAU,SAAS,QAAQ;AAAA,IACtE;AAAA,EACF;AAAA,EAEA,MAAM,eAAe,SAAkD;AACrE,QAAI,KAAK,aAAa,SAAS,GAAG;AAChC,YAAM,KAAK,gBAAgB;AAAA,IAC7B;AACA,QAAI,KAAK,eAAe,SAAS,GAAG;AAClC,YAAM,KAAK,kBAAkB;AAAA,IAC/B;AAEA,UAAM,UAAU,KAAK,aAAa,IAAI,QAAQ,SAAS;AACvD,QAAI,CAAC,SAAS;AACZ,YAAM,IAAI,MAAM,oBAAoB,QAAQ,SAAS,EAAE;AAAA,IACzD;AAEA,UAAM,qBAAqB,QAAQ,aAChC,IAAI,CAAC,OAAO,KAAK,eAAe,IAAI,EAAE,CAAC,EACvC,OAAO,CAAC,UAAuC,QAAQ,KAAK,CAAC,EAC7D,IAAI,CAAC,WAAW,EAAE,IAAI,MAAM,IAAI,WAAW,MAAM,UAAU,EAAE;AAEhE,QAAI,mBAAmB,WAAW,GAAG;AACnC,YAAM,IAAI,MAAM,gCAAgC;AAAA,IAClD;AAEA,UAAM,oBAAoB,MAAM,KAAK,wBAAwB,QAAQ,SAAS;AAE9E,UAAM,QAAQ,OAAO,WAAW,CAAC;AACjC,UAAM,eAAe;AAAA,MACnB,KAAK,OAAO;AAAA,MACZ,QAAQ;AAAA,MACR;AAAA,IACF;AACA,UAAM,WAAwB;AAAA,MAC5B;AAAA,MACA,WAAW,QAAQ;AAAA,MACnB,aAAa,QAAQ,QAAQ,QAAQ;AAAA,MACrC,cAAc,mBAAmB,IAAI,CAAC,SAAS,KAAK,EAAE;AAAA,MACtD,UAAU,KAAK,IAAI;AAAA,MACnB,gBAAgB,kBAAkB;AAAA,MAClC,oBAAoB;AAAA,MACpB,iBAAiB;AAAA,MACjB,iBAAiB;AAAA,MACjB,QAAQ;AAAA,MACR;AAAA,IACF;AAEA,SAAK,UAAU,IAAI,OAAO,QAAQ;AAClC,UAAM,cAA2B;AAAA,MAC/B,MAAM;AAAA,MACN;AAAA,MACA,WAAW,QAAQ;AAAA,MACnB,aAAa,QAAQ,QAAQ,QAAQ;AAAA,MACrC,cAAc,mBAAmB,IAAI,CAAC,SAAS,KAAK,EAAE;AAAA,MACtD,gBAAgB,kBAAkB;AAAA,MAClC;AAAA,IACF;AACA,UAAMA,QAAO,WAAW,KAAK,aAAa,WAAW,CAAC;AACtD,UAAMA,QAAO;AAAA,MACXC,OAAM,MAAM,KAAK,kBAAkB;AAAA,QACjC;AAAA,QACA;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAEA,UAAMD,QAAO;AAAA,MACXC,OAAM,MAAM,KAAK,UAAU;AAAA,QACzB;AAAA,QACA,WAAW,QAAQ;AAAA,QACnB,SAAS,QAAQ;AAAA,QACjB,YAAY;AAAA,QACZ,WAAW;AAAA,QACX;AAAA,MACF,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,mBACE,UACA,SACY;AACZ,UAAM,QAAQ,EAAE,OAAO,SAAS,OAAO,SAAS;AAChD,SAAK,UAAU,IAAI,KAAK;AACxB,WAAO,MAAM;AACX,WAAK,UAAU,OAAO,KAAK;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,eAAe,OAAwC;AACrD,WAAO,KAAK,UAAU,IAAI,KAAK;AAAA,EACjC;AAAA,EAEA,qBAAiD;AAC/C,WAAO,MAAM,KAAK,KAAK,UAAU,OAAO,CAAC,EAAE;AAAA,MACzC,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,MAAM,WAA0B;AAC9B,UAAMD,QAAO,WAAW,MAAM,UAAU,KAAK,cAAc,CAAC;AAC5D,UAAMA,QAAO,WAAW,MAAM,UAAU,KAAK,gBAAgB,CAAC;AAC9D,UAAMA,QAAO,WAAWC,OAAM,SAAS,KAAK,QAAQ,CAAC;AACrD,UAAMD,QAAO,WAAWC,OAAM,SAAS,KAAK,gBAAgB,CAAC;AAC7D,UAAMD,QAAO,WAAW,OAAO,SAAS,KAAK,QAAQ,CAAC;AAAA,EACxD;AAAA,EAEQ,wBAAwB;AAC9B,UAAM,OAAO;AACb,WAAOA,QAAO;AAAA,MACZA,QAAO,IAAI,aAAa;AACtB,cAAM,OAAO,OAAOC,OAAM,KAAK,KAAK,QAAQ;AAC5C,eAAOD,QAAO;AAAA,UACZ;AAAA,YACE;AAAA,YACA,KAAK,aAAa,KAAK,IAAI;AAAA,YAC3B,KAAK;AAAA,YACL,KAAK,eAAe,KAAK,IAAI;AAAA,UAC/B;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,eACN,OACA,SACM;AACN,UAAM,WAAW,KAAK,UAAU,IAAI,KAAK;AACzC,QAAI,CAAC,UAAU;AACb;AAAA,IACF;AACA,SAAK,UAAU,IAAI,OAAO,QAAQ,QAAQ,CAAC;AAAA,EAC7C;AAAA,EAEQ,aAAa,OAAuD;AAC1E,WAAOA,QAAO,KAAK,MAAM;AACvB,iBAAW,SAAS,KAAK,WAAW;AAClC,YAAI,MAAM,SAAS,MAAM,UAAU,MAAM,OAAO;AAC9C;AAAA,QACF;AACA,cAAM,SAAS,KAAK;AAAA,MACtB;AAAA,IACF,CAAC,EAAE;AAAA,MACDA,QAAO,QAAQ,MAAM,OAAO,QAAQ,KAAK,UAAU,KAAK,CAAC;AAAA,MACzDA,QAAO;AAAA,IACT;AAAA,EACF;AACF;;;AY/VO,SAAS,mBAAmB,MAA+B;AAChE,QAAM,OAAsB;AAAA,IAC1B,MAAM;AAAA,IACN,aAAa,CAAC;AAAA,EAChB;AACA,MAAI,QAAQ;AACZ,MAAI,KAAK,CAAC,MAAM,SAAS,KAAK,CAAC,MAAM,YAAY;AAC/C,SAAK,UAAU,KAAK,CAAC;AACrB,YAAQ;AAAA,EACV;AAEA,SAAO,QAAQ,KAAK,QAAQ,SAAS,GAAG;AACtC,UAAM,QAAQ,KAAK,KAAK;AACxB,QAAI,UAAU,YAAY,UAAU,MAAM;AACxC,WAAK,OAAO;AACZ;AAAA,IACF;AACA,SAAK,UAAU,eAAe,UAAU,oBAAoB,KAAK,QAAQ,CAAC,GAAG;AAC3E,WAAK,cAAc,KAAK,QAAQ,CAAC;AACjC,eAAS;AACT;AAAA,IACF;AACA,SAAK,UAAU,iBAAiB,UAAU,aAAa,KAAK,QAAQ,CAAC,GAAG;AACtE,WAAK,mBAAmB,KAAK,QAAQ,CAAC;AACtC,eAAS;AACT;AAAA,IACF;AACA,SAAK,YAAY,KAAK,KAAK;AAAA,EAC7B;AAEA,SAAO;AACT;AAEO,SAAS,oBAA4B;AAC1C,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;ACtDA,SAAS,iBAAiB;AAC1B,SAAS,QAAAO,OAAM,OAAO,WAAAJ,gBAAe;AAUrC,SAASK,sBAAqB,UAElB;AACV,MAAI,OAAO,SAAS,wBAAwB,YAAY;AACtD,WAAO;AAAA,EACT;AACA,SAAO,SAAS,oBAAoB;AACtC;AAEA,SAAS,iBAAiB,iBAAiC;AACzD,QAAM,SAAS,MAAM,eAAe;AACpC,SAAOD,MAAK,OAAO,KAAK,GAAG,OAAO,IAAI,aAAa;AACrD;AAEA,eAAsB,2BACpB,QACA,aACe;AACf,QAAM,UAAU,MAAM,OAAO,qBAAqB,WAAW;AAC7D,MAAI,CAAC,SAAS;AACZ,UAAM,IAAI,MAAM,YAAY,WAAW,cAAc;AAAA,EACvD;AAEA,QAAM,YAAY,MAAM,OAAO,wBAAwB,QAAQ,EAAE;AACjE,QAAM,UAAkC,UAAU,IAAI,CAAC,UAAU;AAAA,IAC/D,MAAM,KAAK,SAAS,QAAQ;AAAA,IAC5B,OAAO,KAAK,SAAS,SAAS;AAAA,IAC9B,kBAAkBC,sBAAqB,KAAK,QAAQ;AAAA,EACtD,EAAE;AAEF,QAAM,sBAAsBL,SAAQ,QAAQ,IAAI,GAAG,QAAQ,QAAQ;AACnE,QAAM,aAAa,iBAAiB,mBAAmB;AAEvD,QAAM,UAAU,YAAY,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AAAA,GAAM,MAAM;AAE3E,UAAQ,IAAI,aAAa,QAAQ,MAAM,4BAA4B,QAAQ,QAAQ,QAAQ,CAAC,IAAI;AAChG,UAAQ,IAAI,SAAS,UAAU,EAAE;AACnC;;;AC1BA,IAAM,OAAO;AAAA,EACX,OAAO;AAAA,EACP,MAAM;AAAA,EACN,KAAK;AAAA,EACL,OAAO;AAAA,EACP,QAAQ;AAAA,EACR,KAAK;AAAA,EACL,MAAM;AAAA,EACN,SAAS;AACX;AAEA,SAAS,SAAS,MAAc,OAAuB;AACrD,SAAO,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,KAAK;AACrC;AAEA,SAAS,aAAa,OAAuB;AAC3C,MAAI,SAAS,IAAI;AACf,WAAO,KAAK;AAAA,EACd;AACA,MAAI,SAAS,IAAI;AACf,WAAO,KAAK;AAAA,EACd;AACA,SAAO,KAAK;AACd;AAEA,SAAS,wBACP,eACA,WACQ;AACR,MAAI,CAAC,aAAa,UAAU,UAAU,GAAG;AACvC,WAAO,KAAK,cAAc,OAAO,EAAE,CAAC;AAAA,EACtC;AACA,QAAM,OAAO,UAAU,QAAQ,UAAU;AACzC,SAAO,KAAK,cAAc,OAAO,EAAE,CAAC,QAAQ,SAAS,KAAK,QAAQ,CAAC,GAAG,aAAa,IAAI,CAAC,CAAC,WAAW,UAAU,MAAM,WAAW,UAAU,MAAM;AACjJ;AAEA,SAAS,UAAU,OAAe,MAAM,KAAK,QAAQ,IAAY;AAC/D,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,KAAK,CAAC;AAC7C,QAAM,SAAS,KAAK,MAAO,OAAO,MAAO,KAAK;AAC9C,SAAO,GAAG,SAAI,OAAO,MAAM,CAAC,GAAG,SAAI,OAAO,QAAQ,MAAM,CAAC;AAC3D;AAEA,SAAS,yBACP,MACA,QACA,QACA,SACQ;AACR,QAAM,YAAY,SACd,SAAS,QAAQ,GAAG,KAAK,IAAI,GAAG,KAAK,KAAK,EAAE,IAC5C,SAAS,QAAQ,GAAG,KAAK,IAAI,GAAG,KAAK,GAAG,EAAE;AAC9C,QAAM,aAAuB,CAAC;AAC9B,aAAW,QAAQ,QAAQ;AACzB,UAAM,MAAM,aAAa,KAAK,EAAE;AAChC,QAAI,CAAC,KAAK;AACR,YAAM,UAAU,eAAe,KAAK,IAAI;AACxC,iBAAW;AAAA,QACT,YAAY,SACR,SAAS,QAAQ,QAAQ,CAAC,GAAG,aAAa,OAAO,CAAC,IAClD;AAAA,MACN;AACA;AAAA,IACF;AACA,UAAM,YAAY,IAAI,OAAO,KAAK,IAAI;AACtC,YAAQ,IAAI,iBAAiB;AAAA,MAC3B,KAAK,OAAO;AACV,cAAM,UACJ,OAAO,KAAK,SAAS,YACrB,KAAK,SAAS,QACd,WAAW,KAAK,OACX,KAAK,KAA4B,QAClC,eAAe,KAAK,IAAI;AAC9B,YAAI,OAAO,YAAY,YAAY,OAAO,SAAS,OAAO,GAAG;AAC3D,qBAAW;AAAA,YACT,GAAG,SAAS,WAAW,aAAa,OAAO,CAAC,CAAC,IAAI,SAAS,UAAU,OAAO,GAAG,KAAK,GAAG,CAAC;AAAA,UACzF;AAAA,QACF,OAAO;AACL,qBAAW,KAAK,SAAS;AAAA,QAC3B;AACA;AAAA,MACF;AAAA,MACA,KAAK;AACH,mBAAW,KAAK,SAAS;AACzB;AAAA,MACF,KAAK;AACH,mBAAW;AAAA,UACT;AAAA,YACE;AAAA,YACA,KAAK,WAAW,OACZ,GAAG,KAAK,IAAI,GAAG,KAAK,KAAK,KACzB,KAAK,WAAW,QACd,GAAG,KAAK,IAAI,GAAG,KAAK,GAAG,KACvB,KAAK;AAAA,UACb;AAAA,QACF;AACA;AAAA,IACJ;AAAA,EACF;AACA,QAAM,WAAW,WAAW,SAAS,IAAI,WAAW,KAAK,GAAG,IAAI;AAChE,MAAI,OAAO,MAAM,IAAI,KAAK,SAAS,IAAI,QAAQ;AAC/C,MAAI,WAAW,QAAQ,SAAS,GAAG;AACjC,UAAM,cAAwB,CAAC;AAC/B,eAAW,EAAE,IAAI,KAAK,KAAK,SAAS;AAClC,YAAM,MAAM,cAAc,EAAE;AAC5B,UAAI,KAAK;AACP,cAAM,YAAY,IAAI,OAAO,IAAI;AACjC,oBAAY;AAAA,UACV,IAAI,OAAO,IAAI,IAAI,IAAI,KAAK,SAAS,MAAM,IAAI,SAAS;AAAA,QAC1D;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY,SAAS,GAAG;AAC1B,cAAQ,IAAI,YAAY,KAAK,GAAG,CAAC;AAAA,IACnC;AAAA,EACF;AACA,SAAO;AACT;AAEA,eAAsB,qBACpB,QACA,aACA,kBACe;AACf,QAAM,UAAU,MAAM,OAAO,qBAAqB,WAAW;AAC7D,MAAI,CAAC,SAAS;AACZ,UAAM,QAAQ,MAAM,OAAO,gBAAgB;AAC3C,UAAM,YAAY,MAAM,IAAI,CAAC,SAAS,KAAK,QAAQ,QAAQ,CAAC,EAAE,KAAK;AACnE,UAAM,IAAI;AAAA,MACR,UAAU,SAAS,IACf,YAAY,WAAW,oCAAoC,UAAU,KAAK,IAAI,CAAC,KAC/E,YAAY,WAAW;AAAA,IAC7B;AAAA,EACF;AAEA,QAAM,aACJ,MAAM,OAAO,+BAA+B,gBAAgB;AAC9D,MAAI,WAAW,WAAW,GAAG;AAC3B,UAAM,QAAQ,MAAM,OAAO,kBAAkB;AAC7C,UAAM,YAAY,MACf,IAAI,CAAC,SAAS,KAAK,UAAU,QAAQ,CAAC,EACtC,OAAO,CAAC,SAAyB,OAAO,SAAS,QAAQ,EACzD,KAAK;AACR,UAAM,IAAI;AAAA,MACR,UAAU,SAAS,IACf,yBAAyB,gBAAgB,4BAA4B,UAAU,KAAK,IAAI,CAAC,KACzF,yBAAyB,gBAAgB;AAAA,IAC/C;AAAA,EACF;AAEA,QAAM,oBAAoB,IAAI;AAAA,IAC5B,WAAW,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,KAAK,UAAU,QAAQ,KAAK,KAAK,EAAE,CAAC;AAAA,EACzE;AACA,QAAM,aAAa,oBAAI,IAAgC;AACvD,QAAM,oBAA4C,CAAC;AACnD,MAAI,oBAAoB;AACxB,MAAI,oBAAoB;AACxB,MAAI,iBAAiB;AACrB,MAAI,aAAa;AACjB,MAAI,cAAc;AAClB,QAAM,gBAAgB,CAAC,UAAK,UAAK,UAAK,UAAK,UAAK,QAAG;AACnD,MAAI,eAAe;AAEnB,WAAS,YAAkB;AACzB,QAAI,CAAC,QAAQ,OAAO,OAAO;AACzB;AAAA,IACF;AACA,YAAQ,OAAO,MAAM,WAAW;AAAA,EAClC;AAEA,WAAS,cAAoB;AAC3B,QAAI,CAAC,QAAQ,OAAO,SAAS,aAAa;AACxC;AAAA,IACF;AACA,UAAM,QAAQ,cAAc,eAAe,cAAc,MAAM;AAC/D,oBAAgB;AAChB,YAAQ,OAAO;AAAA,MACb,KAAK,SAAS,OAAO,KAAK,IAAI,CAAC,wBAAwB;AAAA,QACrD,GAAG,cAAc,IAAI,UAAU;AAAA,QAC/B,KAAK;AAAA,MACP,CAAC,IAAI,SAAS,UAAU,KAAK,GAAG,CAAC;AAAA,IACnC;AAAA,EACF;AAEA,MAAI;AACJ,QAAM,OAAO,IAAI,QAAqB,CAACA,aAAY;AACjD,UAAM,cAAc,OAAO,mBAAmB,CAAC,UAAU;AACvD,UAAI,MAAM,SAAS,oBAAoB;AACrC,yBAAiB,MAAM;AACvB,cAAM,gBAAgB,MAAM,gBACzB,IAAI,CAAC,SAAS,yBAAyB,KAAK,MAAM,CAAC,EACnD,OAAO,CAAC,SAAyB,SAAS,MAAS;AACtD,cAAM,eACJ,cAAc,SAAS,IACnB,cAAc,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC,IACnD,cAAc,SACd;AAEN,kBAAU;AACV,gBAAQ;AAAA,UACN,GAAG,SAAS,IAAI,MAAM,kBAAkB,IAAI,MAAM,cAAc,KAAK,KAAK,IAAI,CAAC,IAAI,MAAM,YAAY,IAAI,SAAS,IAAI,MAAM,UAAU,OAAO,KAAK,GAAG,CAAC;AAAA,QACxJ;AACA,mBAAW,QAAQ,MAAM,iBAAiB;AACxC,gBAAM,OACJ,kBAAkB,IAAI,KAAK,WAAW,KAAK,KAAK;AAClD,kBAAQ;AAAA,YACN;AAAA,cACE;AAAA,cACA,KAAK;AAAA,cACL,KAAK;AAAA,cACL,KAAK;AAAA,YACP;AAAA,UACF;AAEA,gBAAM,UAAU,yBAAyB,KAAK,MAAM;AACpD,cAAI,YAAY,QAAW;AACzB,kBAAM,UAAU,WAAW,IAAI,KAAK,WAAW,KAAK;AAAA,cAClD,OAAO;AAAA,cACP,OAAO;AAAA,cACP,QAAQ;AAAA,cACR,QAAQ;AAAA,YACV;AACA,uBAAW,IAAI,KAAK,aAAa;AAAA,cAC/B,OAAO,QAAQ,QAAQ;AAAA,cACvB,OAAO,QAAQ,QAAQ;AAAA,cACvB,QAAQ,QAAQ,UAAU,KAAK,SAAS,IAAI;AAAA,cAC5C,QAAQ,QAAQ,UAAU,KAAK,SAAS,IAAI;AAAA,YAC9C,CAAC;AACD,iCAAqB;AACrB,iCAAqB;AAAA,UACvB;AAAA,QACF;AAEA,0BAAkB,KAAK;AAAA,UACrB,MAAM,MAAM;AAAA,UACZ;AAAA,UACA,YAAY,MAAM;AAAA,UAClB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAED,oBAAY;AAAA,MACd;AACA,UAAI,MAAM,SAAS,kBAAkB,MAAM,SAAS,aAAa;AAC/D,sBAAc;AACd,kBAAU;AACV,oBAAY;AACZ,QAAAA,SAAQ,KAAK;AAAA,MACf;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,QAAM,WAAW,MAAM,OAAO,eAAe;AAAA,IAC3C,WAAW,QAAQ;AAAA,IACnB,cAAc,WAAW,IAAI,CAAC,SAAS,KAAK,EAAE;AAAA,EAChD,CAAC;AACD,eAAa,SAAS;AAEtB,UAAQ,IAAI,SAAS,4BAA4B,GAAG,KAAK,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;AAC5E,UAAQ,IAAI,QAAQ,SAAS,SAAS,OAAO,KAAK,IAAI,CAAC,EAAE;AACzD,UAAQ,IAAI,YAAY,SAAS,SAAS,aAAa,KAAK,IAAI,CAAC,EAAE;AACnE,UAAQ;AAAA,IACN,eAAe,WACZ,IAAI,CAAC,SAAS,KAAK,UAAU,QAAQ,KAAK,KAAK,EAAE,EACjD,KAAK,IAAI,CAAC;AAAA,EACf;AACA,UAAQ;AAAA,IACN,qBAAqB,SAAS,OAAO,SAAS,cAAc,GAAG,KAAK,IAAI,CAAC;AAAA,EAC3E;AACA,UAAQ,IAAI,EAAE;AACd,cAAY;AACZ,iBAAe,YAAY,aAAa,GAAG;AAE3C,QAAM,aAAa,MAAM;AACzB,MAAI,cAAc;AAChB,kBAAc,YAAY;AAAA,EAC5B;AAEA,MAAI,WAAW,SAAS,aAAa;AACnC,UAAM,IAAI,MAAM,eAAe,WAAW,YAAY,EAAE;AAAA,EAC1D;AAEA,UAAQ,IAAI,EAAE;AACd,UAAQ,IAAI,SAAS,uBAAuB,GAAG,KAAK,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;AACvE,UAAQ;AAAA,IACN,aAAa;AAAA,MACX,GAAG,WAAW,eAAe,IAAI,WAAW,cAAc;AAAA,MAC1D,KAAK;AAAA,IACP,CAAC;AAAA,EACH;AACA,UAAQ;AAAA,IACN,aAAa;AAAA,MACX,GAAG,WAAW,eAAe,IAAI,WAAW,cAAc;AAAA,MAC1D,WAAW,kBAAkB,IAAI,KAAK,MAAM,KAAK;AAAA,IACnD,CAAC;AAAA,EACH;AACA,MAAI,oBAAoB,GAAG;AACzB,UAAM,iBAAiB,oBAAoB;AAC3C,YAAQ;AAAA,MACN,wBAAwB;AAAA,QACtB,eAAe,QAAQ,CAAC;AAAA,QACxB,aAAa,cAAc;AAAA,MAC7B,CAAC,IAAI,SAAS,UAAU,cAAc,GAAG,KAAK,GAAG,CAAC;AAAA,IACpD;AAAA,EACF;AACA,UAAQ,IAAI,SAAS,yBAAyB,KAAK,OAAO,CAAC;AAC3D,aAAW,CAAC,aAAa,aAAa,KAAK,kBAAkB,QAAQ,GAAG;AACtE,YAAQ;AAAA,MACN,wBAAwB,eAAe,WAAW,IAAI,WAAW,CAAC;AAAA,IACpE;AAAA,EACF;AACA,MAAI,kBAAkB,SAAS,GAAG;AAChC,YAAQ,IAAI,SAAS,uBAAuB,KAAK,OAAO,CAAC;AACzD,eAAW,WAAW,mBAAmB;AACvC,YAAM,SAAS,QAAQ,SACnB,SAAS,QAAQ,KAAK,KAAK,IAC3B,SAAS,QAAQ,KAAK,GAAG;AAC7B,UAAI,QAAQ,iBAAiB,QAAW;AACtC,gBAAQ;AAAA,UACN,KAAK,MAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC,cAAc,SAAS,IAAI,QAAQ,UAAU,OAAO,KAAK,GAAG,CAAC;AAAA,QACrG;AACA;AAAA,MACF;AACA,cAAQ;AAAA,QACN,KAAK,MAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC,UAAU;AAAA,UAC9C,QAAQ,aAAa,QAAQ,CAAC;AAAA,UAC9B,aAAa,QAAQ,YAAY;AAAA,QACnC,CAAC,IAAI,SAAS,UAAU,QAAQ,cAAc,KAAK,EAAE,GAAG,KAAK,GAAG,CAAC,IAAI,SAAS,IAAI,QAAQ,UAAU,OAAO,KAAK,GAAG,CAAC;AAAA,MACtH;AAAA,IACF;AAAA,EACF;AACA,UAAQ,IAAI,eAAe,SAAS,WAAW,cAAc,KAAK,GAAG,CAAC,EAAE;AAC1E;;;ACzVA,SAAS,kBAAkB,UAAyB;AAClD,QAAM,UAAU,aAAa,IAAI,QAAQ,MAAM,QAAQ;AACvD,UAAQ,kBAAkB,CAAC;AAC3B,UAAQ,KAAK,QAAQ;AACvB;AAEA,eAAe,OAAsB;AACnC,QAAM,OAAO,mBAAmB,QAAQ,KAAK,MAAM,CAAC,CAAC;AAErD,MAAI,KAAK,MAAM;AACb,sBAAkB,CAAC;AAAA,EACrB;AACA,MAAI,KAAK,YAAY,SAAS,GAAG;AAC/B,YAAQ,MAAM,sBAAsB,KAAK,YAAY,KAAK,IAAI,CAAC,EAAE;AACjE,sBAAkB,CAAC;AAAA,EACrB;AACA,MAAI,CAAC,KAAK,SAAS;AACjB,sBAAkB,CAAC;AAAA,EACrB;AACA,MAAI,CAAC,KAAK,aAAa;AACrB,YAAQ,MAAM,oDAAoD;AAClE,sBAAkB,CAAC;AAAA,EACrB;AAEA,QAAM,SAAS,aAAa;AAC5B,MAAI;AACF,QAAI,KAAK,YAAY,OAAO;AAC1B,UAAI,CAAC,KAAK,kBAAkB;AAC1B,gBAAQ,MAAM,0DAA0D;AACxE,0BAAkB,CAAC;AAAA,MACrB;AACA,YAAM,qBAAqB,QAAQ,KAAK,aAAa,KAAK,gBAAgB;AAC1E;AAAA,IACF;AAEA,UAAM,2BAA2B,QAAQ,KAAK,WAAW;AAAA,EAC3D,UAAE;AACA,UAAM,OAAO,SAAS;AAAA,EACxB;AACF;AAEA,KAAK,KAAK,EAAE,MAAM,CAAC,UAAmB;AACpC,UAAQ,MAAM,iBAAiB,QAAQ,MAAM,UAAU,gBAAgB;AACvE,UAAQ,KAAK,CAAC;AAChB,CAAC","sourcesContent":["import { randomUUID } from 'node:crypto';\n\nimport { Effect, Fiber, PubSub, Queue } from 'effect';\n\nimport type { RunnerConfig, RunnerConfigOverrides } from './config';\nimport { withRunnerConfig } from './config';\nimport { loadRunnerConfigFile } from './config-loader';\nimport {\n  collectDatasetsFromFiles,\n  collectEvaluatorsFromFiles,\n  collectTestCasesFromFiles,\n} from './discovery';\nimport { createArtifactPath, executeRunTask, type RunTask } from './execution';\nimport type {\n  CollectedDataset,\n  CollectedEvaluator,\n  CollectedTestCase,\n  RunDatasetRequest,\n  RunSnapshot,\n  RunnerEvent,\n  SearchTestCasesQuery,\n} from './events';\nimport { createPersistenceWorker } from './persistence';\nimport { searchCollectedTestCases } from './search';\n\ninterface SubscribeOptions {\n  runId?: string;\n}\n\nfunction parseRegexLiteral(\n  pattern: string,\n): { source: string; flags: string } | undefined {\n  if (!pattern.startsWith('/')) {\n    return undefined;\n  }\n  const lastSlash = pattern.lastIndexOf('/');\n  if (lastSlash <= 0) {\n    return undefined;\n  }\n  return {\n    source: pattern.slice(1, lastSlash),\n    flags: pattern.slice(lastSlash + 1),\n  };\n}\n\nfunction createNameMatcher(pattern: string): (value: string) => boolean {\n  const normalizedPattern = pattern.trim();\n  const regexLiteral = parseRegexLiteral(normalizedPattern);\n  if (regexLiteral) {\n    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);\n    return (value: string) => regex.test(value);\n  }\n\n  if (normalizedPattern.includes('*')) {\n    const escaped = normalizedPattern\n      .replace(/[.+^${}()|[\\]\\\\]/g, '\\\\$&')\n      .replace(/\\*/g, '.*');\n    const regex = new RegExp(`^${escaped}$`, 'i');\n    return (value: string) => regex.test(value);\n  }\n\n  return (value: string) => value.toLowerCase() === normalizedPattern.toLowerCase();\n}\n\nexport interface RunnerApi {\n  collectDatasets(): Promise<ReadonlyArray<CollectedDataset>>;\n  collectEvaluators(): Promise<ReadonlyArray<CollectedEvaluator>>;\n  resolveDatasetByName(name: string): Promise<CollectedDataset | undefined>;\n  resolveEvaluatorsByNamePattern(\n    pattern: string,\n  ): Promise<ReadonlyArray<CollectedEvaluator>>;\n  searchTestCases(\n    query?: SearchTestCasesQuery,\n  ): Promise<ReadonlyArray<CollectedTestCase>>;\n  collectDatasetTestCases(datasetId: string): Promise<ReadonlyArray<CollectedTestCase>>;\n  runDatasetWith(request: RunDatasetRequest): Promise<RunSnapshot>;\n  subscribeRunEvents(\n    listener: (event: RunnerEvent) => void,\n    options?: SubscribeOptions,\n  ): () => void;\n  getRunSnapshot(runId: string): RunSnapshot | undefined;\n  getAllRunSnapshots(): ReadonlyArray<RunSnapshot>;\n  shutdown(): Promise<void>;\n}\n\nfunction mergeRunnerOverrides(\n  base?: RunnerConfigOverrides,\n  next?: RunnerConfigOverrides,\n): RunnerConfigOverrides | undefined {\n  if (!base) {\n    return next;\n  }\n  if (!next) {\n    return base;\n  }\n  const discovery = base.discovery || next.discovery\n    ? {\n        ...(base.discovery ?? {}),\n        ...(next.discovery ?? {}),\n      }\n    : undefined;\n  return {\n    ...base,\n    ...next,\n    discovery,\n  };\n}\n\nexport function createRunner(overrides?: RunnerConfigOverrides): RunnerApi {\n  const fileOverrides = loadRunnerConfigFile();\n  const merged = mergeRunnerOverrides(fileOverrides, overrides);\n  return new EffectRunner(withRunnerConfig(merged));\n}\n\nclass EffectRunner implements RunnerApi {\n  private readonly config: RunnerConfig;\n\n  private readonly eventBus = Effect.runSync(PubSub.unbounded<RunnerEvent>());\n\n  private readonly runQueue = Effect.runSync(Queue.unbounded<RunTask>());\n\n  private readonly persistenceQueue = Effect.runSync(\n    Queue.unbounded<{\n      runId: string;\n      artifactPath: string;\n      payload: unknown;\n    }>(),\n  );\n\n  private readonly snapshots = new Map<string, RunSnapshot>();\n  private readonly listeners = new Set<{\n    runId?: string;\n    listener: (event: RunnerEvent) => void;\n  }>();\n\n  private readonly datasetsById = new Map<string, CollectedDataset>();\n\n  private readonly evaluatorsById = new Map<string, CollectedEvaluator>();\n\n  private readonly schedulerFiber = Effect.runFork(\n    this.createSchedulerEffect(),\n  );\n\n  private readonly persistenceFiber = Effect.runFork(\n    createPersistenceWorker(this.persistenceQueue),\n  );\n\n  constructor(config: RunnerConfig) {\n    this.config = config;\n  }\n\n  async collectDatasets(): Promise<ReadonlyArray<CollectedDataset>> {\n    const datasets = await collectDatasetsFromFiles(this.config.discovery);\n    this.datasetsById.clear();\n    for (const dataset of datasets) {\n      this.datasetsById.set(dataset.id, dataset);\n    }\n    return datasets;\n  }\n\n  async collectEvaluators(): Promise<ReadonlyArray<CollectedEvaluator>> {\n    const evaluators = await collectEvaluatorsFromFiles(this.config.discovery);\n    this.evaluatorsById.clear();\n    for (const evaluator of evaluators) {\n      this.evaluatorsById.set(evaluator.id, evaluator);\n    }\n    return evaluators;\n  }\n\n  async resolveDatasetByName(name: string): Promise<CollectedDataset | undefined> {\n    if (this.datasetsById.size === 0) {\n      await this.collectDatasets();\n    }\n    const normalized = name.trim().toLowerCase();\n    return Array.from(this.datasetsById.values()).find(\n      (item) => item.dataset.getName().toLowerCase() === normalized,\n    );\n  }\n\n  async resolveEvaluatorsByNamePattern(\n    pattern: string,\n  ): Promise<ReadonlyArray<CollectedEvaluator>> {\n    if (this.evaluatorsById.size === 0) {\n      await this.collectEvaluators();\n    }\n    const matcher = createNameMatcher(pattern);\n    return Array.from(this.evaluatorsById.values()).filter((item) =>\n      matcher(item.evaluator.getName() ?? ''),\n    );\n  }\n\n  async searchTestCases(\n    query?: SearchTestCasesQuery,\n  ): Promise<ReadonlyArray<CollectedTestCase>> {\n    const testCases = await collectTestCasesFromFiles(this.config.discovery);\n    return searchCollectedTestCases(testCases, query);\n  }\n\n  async collectDatasetTestCases(\n    datasetId: string,\n  ): Promise<ReadonlyArray<CollectedTestCase>> {\n    if (this.datasetsById.size === 0) {\n      await this.collectDatasets();\n    }\n    const dataset = this.datasetsById.get(datasetId);\n    if (!dataset) {\n      throw new Error(`Unknown dataset: ${datasetId}`);\n    }\n    const allTestCases = await collectTestCasesFromFiles(this.config.discovery);\n    return allTestCases.filter((testCase) =>\n      dataset.dataset.matchesTestCase(testCase.testCase, testCase.filePath),\n    );\n  }\n\n  async runDatasetWith(request: RunDatasetRequest): Promise<RunSnapshot> {\n    if (this.datasetsById.size === 0) {\n      await this.collectDatasets();\n    }\n    if (this.evaluatorsById.size === 0) {\n      await this.collectEvaluators();\n    }\n\n    const dataset = this.datasetsById.get(request.datasetId);\n    if (!dataset) {\n      throw new Error(`Unknown dataset: ${request.datasetId}`);\n    }\n\n    const selectedEvaluators = request.evaluatorIds\n      .map((id) => this.evaluatorsById.get(id))\n      .filter((value): value is CollectedEvaluator => Boolean(value))\n      .map((value) => ({ id: value.id, evaluator: value.evaluator }));\n\n    if (selectedEvaluators.length === 0) {\n      throw new Error('No evaluators selected for run');\n    }\n\n    const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);\n\n    const runId = `run-${randomUUID()}`;\n    const artifactPath = createArtifactPath(\n      this.config.artifactDirectory,\n      request.datasetId,\n      runId,\n    );\n    const snapshot: RunSnapshot = {\n      runId,\n      datasetId: request.datasetId,\n      datasetName: dataset.dataset.getName(),\n      evaluatorIds: selectedEvaluators.map((item) => item.id),\n      queuedAt: Date.now(),\n      totalTestCases: selectedTestCases.length,\n      completedTestCases: 0,\n      passedTestCases: 0,\n      failedTestCases: 0,\n      status: 'queued',\n      artifactPath,\n    };\n\n    this.snapshots.set(runId, snapshot);\n    const queuedEvent: RunnerEvent = {\n      type: 'RunQueued',\n      runId,\n      datasetId: request.datasetId,\n      datasetName: dataset.dataset.getName(),\n      evaluatorIds: selectedEvaluators.map((item) => item.id),\n      totalTestCases: selectedTestCases.length,\n      artifactPath,\n    };\n    await Effect.runPromise(this.publishEvent(queuedEvent));\n    await Effect.runPromise(\n      Queue.offer(this.persistenceQueue, {\n        runId,\n        artifactPath,\n        payload: queuedEvent,\n      }),\n    );\n\n    await Effect.runPromise(\n      Queue.offer(this.runQueue, {\n        runId,\n        datasetId: request.datasetId,\n        dataset: dataset.dataset,\n        evaluators: selectedEvaluators,\n        testCases: selectedTestCases,\n        snapshot,\n      }),\n    );\n\n    return snapshot;\n  }\n\n  subscribeRunEvents(\n    listener: (event: RunnerEvent) => void,\n    options?: SubscribeOptions,\n  ): () => void {\n    const entry = { runId: options?.runId, listener };\n    this.listeners.add(entry);\n    return () => {\n      this.listeners.delete(entry);\n    };\n  }\n\n  getRunSnapshot(runId: string): RunSnapshot | undefined {\n    return this.snapshots.get(runId);\n  }\n\n  getAllRunSnapshots(): ReadonlyArray<RunSnapshot> {\n    return Array.from(this.snapshots.values()).sort(\n      (a, b) => b.queuedAt - a.queuedAt,\n    );\n  }\n\n  async shutdown(): Promise<void> {\n    await Effect.runPromise(Fiber.interrupt(this.schedulerFiber));\n    await Effect.runPromise(Fiber.interrupt(this.persistenceFiber));\n    await Effect.runPromise(Queue.shutdown(this.runQueue));\n    await Effect.runPromise(Queue.shutdown(this.persistenceQueue));\n    await Effect.runPromise(PubSub.shutdown(this.eventBus));\n  }\n\n  private createSchedulerEffect() {\n    const self = this;\n    return Effect.forever(\n      Effect.gen(function* () {\n        const task = yield* Queue.take(self.runQueue);\n        yield* Effect.fork(\n          executeRunTask(\n            task,\n            self.publishEvent.bind(self),\n            self.persistenceQueue,\n            self.updateSnapshot.bind(self),\n          ),\n        );\n      }),\n    );\n  }\n\n  private updateSnapshot(\n    runId: string,\n    updater: (snapshot: RunSnapshot) => RunSnapshot,\n  ): void {\n    const existing = this.snapshots.get(runId);\n    if (!existing) {\n      return;\n    }\n    this.snapshots.set(runId, updater(existing));\n  }\n\n  private publishEvent(event: RunnerEvent): Effect.Effect<void, never, never> {\n    return Effect.sync(() => {\n      for (const entry of this.listeners) {\n        if (entry.runId && entry.runId !== event.runId) {\n          continue;\n        }\n        entry.listener(event);\n      }\n    }).pipe(\n      Effect.flatMap(() => PubSub.publish(this.eventBus, event)),\n      Effect.asVoid,\n    );\n  }\n}\n","export interface RunnerDiscoveryConfig {\n  rootDir: string;\n  datasetSuffixes: ReadonlyArray<string>;\n  evaluatorSuffixes: ReadonlyArray<string>;\n  testCaseSuffixes: ReadonlyArray<string>;\n  excludeDirectories: ReadonlyArray<string>;\n}\n\nexport interface RunnerConfig {\n  discovery: RunnerDiscoveryConfig;\n  artifactDirectory: string;\n}\n\nexport type RunnerConfigOverrides = Omit<Partial<RunnerConfig>, 'discovery'> & {\n  discovery?: Partial<RunnerDiscoveryConfig>;\n};\n\nexport interface M4trixEvalConfigDiscovery {\n  rootDir?: string;\n  datasetFilePatterns?: ReadonlyArray<string>;\n  evaluatorFilePatterns?: ReadonlyArray<string>;\n  testCaseFilePatterns?: ReadonlyArray<string>;\n  datasetSuffixes?: ReadonlyArray<string>;\n  evaluatorSuffixes?: ReadonlyArray<string>;\n  testCaseSuffixes?: ReadonlyArray<string>;\n  excludeDirectories?: ReadonlyArray<string>;\n}\n\nexport interface M4trixEvalConfig {\n  discovery?: M4trixEvalConfigDiscovery;\n  artifactDirectory?: string;\n}\n\nexport type ConfigType = M4trixEvalConfig;\n\nexport type M4trixEvalConfigFactory<TConfig extends ConfigType = ConfigType> = () => TConfig;\n\nexport function defineConfig<TConfig extends ConfigType>(\n  factory: M4trixEvalConfigFactory<TConfig>,\n): M4trixEvalConfigFactory<TConfig> {\n  return factory;\n}\n\nexport const defaultRunnerConfig: RunnerConfig = {\n  discovery: {\n    rootDir: process.cwd(),\n    datasetSuffixes: ['.dataset.ts', '.dataset.tsx', '.dataset.js', '.dataset.mjs'],\n    evaluatorSuffixes: [\n      '.evaluator.ts',\n      '.evaluator.tsx',\n      '.evaluator.js',\n      '.evaluator.mjs',\n    ],\n    testCaseSuffixes: [\n      '.test-case.ts',\n      '.test-case.tsx',\n      '.test-case.js',\n      '.test-case.mjs',\n    ],\n    excludeDirectories: ['node_modules', 'dist', '.next', '.git', '.pnpm-store'],\n  },\n  artifactDirectory: '.eval-results',\n};\n\nexport function toRunnerConfigOverrides(\n  config?: ConfigType,\n): RunnerConfigOverrides | undefined {\n  if (!config) {\n    return undefined;\n  }\n\n  const rawDiscovery = config.discovery;\n  const discovery: Partial<RunnerDiscoveryConfig> = {};\n  if (rawDiscovery?.rootDir !== undefined) {\n    discovery.rootDir = rawDiscovery.rootDir;\n  }\n  if (rawDiscovery?.datasetFilePatterns !== undefined) {\n    discovery.datasetSuffixes = rawDiscovery.datasetFilePatterns;\n  } else if (rawDiscovery?.datasetSuffixes !== undefined) {\n    discovery.datasetSuffixes = rawDiscovery.datasetSuffixes;\n  }\n  if (rawDiscovery?.evaluatorFilePatterns !== undefined) {\n    discovery.evaluatorSuffixes = rawDiscovery.evaluatorFilePatterns;\n  } else if (rawDiscovery?.evaluatorSuffixes !== undefined) {\n    discovery.evaluatorSuffixes = rawDiscovery.evaluatorSuffixes;\n  }\n  if (rawDiscovery?.testCaseFilePatterns !== undefined) {\n    discovery.testCaseSuffixes = rawDiscovery.testCaseFilePatterns;\n  } else if (rawDiscovery?.testCaseSuffixes !== undefined) {\n    discovery.testCaseSuffixes = rawDiscovery.testCaseSuffixes;\n  }\n  if (rawDiscovery?.excludeDirectories !== undefined) {\n    discovery.excludeDirectories = rawDiscovery.excludeDirectories;\n  }\n\n  const overrides: RunnerConfigOverrides = {};\n  if (config.artifactDirectory !== undefined) {\n    overrides.artifactDirectory = config.artifactDirectory;\n  }\n  if (Object.keys(discovery).length > 0) {\n    overrides.discovery = discovery;\n  }\n  return overrides;\n}\n\nexport function withRunnerConfig(overrides?: RunnerConfigOverrides): RunnerConfig {\n  if (!overrides) {\n    return defaultRunnerConfig;\n  }\n  const discovery = overrides.discovery\n    ? {\n        ...defaultRunnerConfig.discovery,\n        ...overrides.discovery,\n      }\n    : defaultRunnerConfig.discovery;\n\n  return {\n    ...defaultRunnerConfig,\n    ...overrides,\n    discovery,\n  };\n}\n","import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\n\nimport * as jitiModule from 'jiti';\n\nimport type { RunnerConfigOverrides } from './config';\nimport {\n  toRunnerConfigOverrides,\n  type ConfigType,\n  type M4trixEvalConfigFactory,\n} from './config';\n\nconst CONFIG_FILE_NAME = 'm4trix-eval.config.ts';\n\ntype JitiLoader = {\n  (id: string): unknown;\n  import?: (id: string) => Promise<unknown> | unknown;\n};\n\nlet cachedLoader: JitiLoader | undefined;\n\nfunction getJitiLoader(): JitiLoader {\n  if (cachedLoader) {\n    return cachedLoader;\n  }\n  const createJiti =\n    (jitiModule as { createJiti?: unknown; default?: unknown }).createJiti ??\n    (jitiModule as { default?: unknown }).default;\n  if (typeof createJiti !== 'function') {\n    throw new Error(\n      'Failed to initialize jiti for m4trix eval config loading.',\n    );\n  }\n  cachedLoader = (\n    createJiti as (id: string, options?: Record<string, unknown>) => JitiLoader\n  )(import.meta.url, {\n    interopDefault: true,\n    moduleCache: true,\n  });\n  return cachedLoader;\n}\n\nfunction resolveConfigModuleExport(loadedModule: unknown): unknown {\n  if (\n    loadedModule &&\n    typeof loadedModule === 'object' &&\n    'default' in loadedModule\n  ) {\n    return (loadedModule as { default: unknown }).default;\n  }\n  return loadedModule;\n}\n\nfunction resolveConfigValue(value: unknown): ConfigType | undefined {\n  if (value === undefined || value === null) {\n    return undefined;\n  }\n  if (typeof value === 'function') {\n    return (value as M4trixEvalConfigFactory<ConfigType>)();\n  }\n  if (typeof value !== 'object') {\n    throw new Error(\n      'Invalid m4trix eval config export. Expected an object or defineConfig(() => config).',\n    );\n  }\n  return value as ConfigType;\n}\n\nexport function loadRunnerConfigFile(\n  cwd = process.cwd(),\n): RunnerConfigOverrides | undefined {\n  const configPath = resolve(cwd, CONFIG_FILE_NAME);\n  if (!existsSync(configPath)) {\n    return undefined;\n  }\n  const loader = getJitiLoader();\n  const loaded = loader(configPath);\n  const exportedValue = resolveConfigModuleExport(loaded);\n  const config = resolveConfigValue(exportedValue);\n  return toRunnerConfigOverrides(config);\n}\n","import { Dirent } from 'node:fs';\nimport { readdir } from 'node:fs/promises';\nimport { resolve, relative } from 'node:path';\nimport { pathToFileURL } from 'node:url';\n\nimport type { Dataset } from '../evals/dataset';\nimport type { Evaluator } from '../evals/evaluator';\nimport type { TestCase } from '../evals/test-case';\nimport type {\n  CollectedDataset,\n  CollectedEvaluator,\n  CollectedTestCase,\n} from './events';\nimport type { RunnerDiscoveryConfig } from './config';\n\ntype JitiModuleLoader = {\n  (id: string): unknown;\n  import?: (id: string) => Promise<unknown> | unknown;\n};\n\nlet jitiLoader: JitiModuleLoader | undefined;\n\nfunction toId(prefix: string, filePath: string, name?: string): string {\n  const stable = name && name.trim().length > 0 ? name : filePath;\n  return `${prefix}:${stable}`\n    .toLowerCase()\n    .replace(/[^a-z0-9]+/g, '-')\n    .replace(/^-+|-+$/g, '');\n}\n\nfunction hasMethod(value: unknown, methodName: string): boolean {\n  return (\n    typeof value === 'object' &&\n    value !== null &&\n    methodName in value &&\n    typeof (value as Record<string, unknown>)[methodName] === 'function'\n  );\n}\n\nfunction isDatasetLike(value: unknown): value is Dataset {\n  return hasMethod(value, 'getName') && hasMethod(value, 'matchesTestCase');\n}\n\nfunction isEvaluatorLike(\n  value: unknown,\n): value is Evaluator<unknown, unknown, unknown, unknown> {\n  return (\n    hasMethod(value, 'getName') &&\n    hasMethod(value, 'resolveContext') &&\n    hasMethod(value, 'getEvaluateFn')\n  );\n}\n\nfunction isTestCaseLike(value: unknown): value is TestCase<unknown> {\n  return (\n    hasMethod(value, 'getName') &&\n    hasMethod(value, 'getTags') &&\n    hasMethod(value, 'getInput')\n  );\n}\n\nasync function walkDirectory(\n  rootDir: string,\n  excludeDirectories: ReadonlyArray<string>,\n): Promise<string[]> {\n  const out: string[] = [];\n\n  async function walk(currentDir: string): Promise<void> {\n    let entries: Dirent[];\n    try {\n      entries = await readdir(currentDir, { withFileTypes: true });\n    } catch {\n      return;\n    }\n\n    await Promise.all(\n      entries.map(async (entry) => {\n        const absolute = resolve(currentDir, entry.name);\n        if (entry.isDirectory()) {\n          if (excludeDirectories.includes(entry.name)) {\n            return;\n          }\n          await walk(absolute);\n          return;\n        }\n\n        if (entry.isFile()) {\n          out.push(absolute);\n        }\n      }),\n    );\n  }\n\n  await walk(rootDir);\n  return out;\n}\n\nfunction hasOneSuffix(\n  filePath: string,\n  suffixes: ReadonlyArray<string>,\n): boolean {\n  return suffixes.some((suffix) => filePath.endsWith(suffix));\n}\n\nasync function loadModuleExports(filePath: string): Promise<unknown[]> {\n  if (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) {\n    if (!jitiLoader) {\n      const jitiModule = (await import('jiti')) as {\n        createJiti?: (filename: string, opts?: Record<string, unknown>) => JitiModuleLoader;\n        default?: (filename: string, opts?: Record<string, unknown>) => JitiModuleLoader;\n      };\n      const createJiti = jitiModule.createJiti ?? jitiModule.default;\n      if (!createJiti) {\n        throw new Error('Failed to initialize jiti TypeScript loader');\n      }\n      jitiLoader = createJiti(import.meta.url, {\n        interopDefault: true,\n        moduleCache: true,\n      }) as JitiModuleLoader;\n    }\n    const loaded = jitiLoader.import\n      ? await jitiLoader.import(filePath)\n      : await Promise.resolve(jitiLoader(filePath));\n    return Object.values(loaded as Record<string, unknown>);\n  }\n\n  const moduleUrl = pathToFileURL(filePath).href;\n  const loaded = (await import(moduleUrl)) as Record<string, unknown>;\n  return Object.values(loaded);\n}\n\nexport async function collectDatasetsFromFiles(\n  config: RunnerDiscoveryConfig,\n): Promise<ReadonlyArray<CollectedDataset>> {\n  const files = await walkDirectory(config.rootDir, config.excludeDirectories);\n  const matched = files.filter((filePath) =>\n    hasOneSuffix(filePath, config.datasetSuffixes),\n  );\n\n  const found = await Promise.all(\n    matched.map(async (absolutePath) => {\n      const exports = await loadModuleExports(absolutePath);\n      const datasets = exports.filter(isDatasetLike);\n      const relPath = relative(config.rootDir, absolutePath);\n      return datasets.map((dataset) => ({\n        id: toId('dataset', relPath, dataset.getName()),\n        filePath: relPath,\n        dataset,\n      }));\n    }),\n  );\n\n  return found.flat();\n}\n\nexport async function collectEvaluatorsFromFiles(\n  config: RunnerDiscoveryConfig,\n): Promise<ReadonlyArray<CollectedEvaluator>> {\n  const files = await walkDirectory(config.rootDir, config.excludeDirectories);\n  const matched = files.filter((filePath) =>\n    hasOneSuffix(filePath, config.evaluatorSuffixes),\n  );\n\n  const found = await Promise.all(\n    matched.map(async (absolutePath) => {\n      const exports = await loadModuleExports(absolutePath);\n      const evaluators = exports.filter(isEvaluatorLike);\n      const relPath = relative(config.rootDir, absolutePath);\n      return evaluators.map((evaluator) => ({\n        id: toId('evaluator', relPath, evaluator.getName()),\n        filePath: relPath,\n        evaluator,\n      }));\n    }),\n  );\n\n  return found.flat();\n}\n\nexport async function collectTestCasesFromFiles(\n  config: RunnerDiscoveryConfig,\n): Promise<ReadonlyArray<CollectedTestCase>> {\n  const files = await walkDirectory(config.rootDir, config.excludeDirectories);\n  const matched = files.filter((filePath) =>\n    hasOneSuffix(filePath, config.testCaseSuffixes),\n  );\n\n  const found = await Promise.all(\n    matched.map(async (absolutePath) => {\n      const exports = await loadModuleExports(absolutePath);\n      const testCases = exports.filter(isTestCaseLike);\n      const relPath = relative(config.rootDir, absolutePath);\n      return testCases.map((testCase) => ({\n        id: toId('test-case', relPath, testCase.getName()),\n        filePath: relPath,\n        testCase,\n      }));\n    }),\n  );\n\n  return found.flat();\n}\n","import { join } from 'node:path';\n\nimport { Effect, Queue } from 'effect';\n\nimport type { Dataset } from '../evals/dataset';\nimport type { Evaluator } from '../evals/evaluator';\nimport type { MetricItem } from '../evals/metric';\nimport type { ScoreItem } from '../evals/score';\nimport type { CollectedTestCase, RunSnapshot, RunnerEvent } from './events';\nimport type { PersistenceMessage } from './persistence';\nimport { toNumericScoreFromScores } from './score-utils';\n\nfunction computeEvaluatorPassed(\n  evaluator: Evaluator<unknown, unknown, unknown, unknown>,\n  result: unknown,\n  scores: ReadonlyArray<ScoreItem>,\n): boolean {\n  const scoresWithPassed = scores.filter((s) => 'passed' in s && s.passed !== undefined);\n  if (scoresWithPassed.length > 0) {\n    return scoresWithPassed.every((s) => s.passed === true);\n  }\n  const passCriterion = evaluator.getPassCriterion();\n  if (passCriterion) {\n    return passCriterion(result);\n  }\n  const passThreshold = evaluator.getPassThreshold();\n  if (passThreshold !== undefined) {\n    const numeric = toNumericScoreFromScores(scores);\n    return numeric !== undefined && numeric >= passThreshold;\n  }\n  return true;\n}\n\nfunction normalizeResult(\n  result: unknown,\n): {\n  scores: ReadonlyArray<ScoreItem>;\n  metrics?: ReadonlyArray<MetricItem>;\n} {\n  if (typeof result !== 'object' || result === null) {\n    return { scores: [] };\n  }\n  const obj = result as Record<string, unknown>;\n  const scores = Array.isArray(obj.scores)\n    ? (obj.scores as ReadonlyArray<ScoreItem>)\n    : [];\n  const metrics = Array.isArray(obj.metrics)\n    ? (obj.metrics as ReadonlyArray<MetricItem>)\n    : undefined;\n  return { scores, metrics };\n}\n\nfunction readOutputDefinition(testCase: CollectedTestCase['testCase']): unknown {\n  const candidate = testCase as unknown as {\n    getOutputDefinition?: () => unknown;\n  };\n  if (typeof candidate.getOutputDefinition !== 'function') {\n    return undefined;\n  }\n  return candidate.getOutputDefinition();\n}\n\nexport interface RunTask {\n  runId: string;\n  datasetId: string;\n  dataset: Dataset;\n  evaluators: ReadonlyArray<{\n    id: string;\n    evaluator: Evaluator<unknown, unknown, unknown, unknown>;\n  }>;\n  testCases: ReadonlyArray<CollectedTestCase>;\n  snapshot: RunSnapshot;\n}\n\nfunction nowIsoForFile(): string {\n  return new Date().toISOString().replace(/[:.]/g, '-');\n}\n\nexport function createArtifactPath(\n  artifactDirectory: string,\n  datasetId: string,\n  runId: string,\n): string {\n  return join(\n    artifactDirectory,\n    `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`,\n  );\n}\n\nexport const executeRunTask = (\n  task: RunTask,\n  publishEvent: (event: RunnerEvent) => Effect.Effect<void, never, never>,\n  persistenceQueue: Queue.Queue<PersistenceMessage>,\n  updateSnapshot: (\n    runId: string,\n    updater: (snapshot: RunSnapshot) => RunSnapshot,\n  ) => void,\n): Effect.Effect<void, never, never> =>\n  Effect.gen(function* () {\n    const startedAt = Date.now();\n    updateSnapshot(task.runId, (snapshot) => ({\n      ...snapshot,\n      status: 'running',\n      startedAt,\n    }));\n    yield* publishEvent({\n      type: 'RunStarted',\n      runId: task.runId,\n      startedAt,\n    });\n\n    let completedTestCases = 0;\n    let passedTestCases = 0;\n    let failedTestCases = 0;\n\n    for (const testCaseItem of task.testCases) {\n      const started = Date.now();\n      const evaluatorScores: Array<{\n        evaluatorId: string;\n        scores: ReadonlyArray<ScoreItem>;\n        passed: boolean;\n        metrics?: ReadonlyArray<MetricItem>;\n      }> = [];\n      let testCaseError: string | undefined;\n      const outputDefinition = readOutputDefinition(testCaseItem.testCase);\n\n      for (const { id: evaluatorId, evaluator } of task.evaluators) {\n        const evaluateFn = evaluator.getEvaluateFn();\n        if (!evaluateFn) {\n          continue;\n        }\n\n        try {\n          const ctx = yield* Effect.promise(() =>\n            Promise.resolve(evaluator.resolveContext()),\n          );\n          const result = yield* Effect.promise(() =>\n            Promise.resolve(\n              evaluateFn({\n                input: testCaseItem.testCase.getInput(),\n                ctx,\n                output: outputDefinition,\n              }),\n            ),\n          );\n          const { scores, metrics } = normalizeResult(result);\n          const passed = computeEvaluatorPassed(evaluator, result, scores);\n          evaluatorScores.push({ evaluatorId, scores, passed, metrics });\n        } catch (error) {\n          testCaseError =\n            error instanceof Error\n              ? error.message\n              : 'Evaluator execution failed';\n          evaluatorScores.push({\n            evaluatorId,\n            scores: [],\n            passed: false,\n          });\n        }\n      }\n\n      const testCasePassed = evaluatorScores.every((s) => s.passed);\n      completedTestCases += 1;\n      if (testCasePassed) {\n        passedTestCases += 1;\n      } else {\n        failedTestCases += 1;\n      }\n\n      const progressEvent: RunnerEvent = {\n        type: 'TestCaseProgress',\n        runId: task.runId,\n        testCaseId: testCaseItem.id,\n        testCaseName: testCaseItem.testCase.getName(),\n        completedTestCases,\n        totalTestCases: task.testCases.length,\n        passed: testCasePassed,\n        durationMs: Date.now() - started,\n        evaluatorScores,\n        outputDefinition,\n        errorMessage: testCaseError,\n      };\n\n      updateSnapshot(task.runId, (snapshot) => ({\n        ...snapshot,\n        completedTestCases,\n        passedTestCases,\n        failedTestCases,\n      }));\n\n      yield* publishEvent(progressEvent);\n      yield* Queue.offer(persistenceQueue, {\n        runId: task.runId,\n        artifactPath: task.snapshot.artifactPath,\n        payload: progressEvent,\n      });\n    }\n\n    const finishedAt = Date.now();\n    const completedEvent: RunnerEvent = {\n      type: 'RunCompleted',\n      runId: task.runId,\n      finishedAt,\n      passedTestCases,\n      failedTestCases,\n      totalTestCases: task.testCases.length,\n      artifactPath: task.snapshot.artifactPath,\n    };\n\n    updateSnapshot(task.runId, (snapshot) => ({\n      ...snapshot,\n      status: 'completed',\n      completedTestCases,\n      passedTestCases,\n      failedTestCases,\n      finishedAt,\n    }));\n\n    yield* publishEvent(completedEvent);\n    yield* Queue.offer(persistenceQueue, {\n      runId: task.runId,\n      artifactPath: task.snapshot.artifactPath,\n      payload: completedEvent,\n    });\n    yield* publishEvent({\n      type: 'ArtifactFlushed',\n      runId: task.runId,\n      artifactPath: task.snapshot.artifactPath,\n    });\n  });\n","const registry = new Map<string, MetricDef<unknown>>();\n\nexport interface MetricItem<TData = unknown> {\n  readonly id: string;\n  readonly data: TData;\n}\n\nexport interface MetricDef<TData = unknown> {\n  readonly id: string;\n  readonly name?: string;\n  format(data: TData): string;\n  make(data: TData): MetricItem<TData>;\n}\n\nexport const Metric = {\n  of<TData>(config: {\n    id: string;\n    name?: string;\n    format: (data: TData) => string;\n  }): MetricDef<TData> {\n    const def: MetricDef<TData> = {\n      id: config.id,\n      name: config.name,\n      format: config.format,\n      make: (data: TData) => ({ id: config.id, data }),\n    };\n    registry.set(config.id, def as MetricDef<unknown>);\n    return def;\n  },\n};\n\nexport function getMetricById(id: string): MetricDef<unknown> | undefined {\n  return registry.get(id);\n}\n","const registry = new Map<string, ScoreDef<unknown>>();\n\nexport type ScoreDisplayStrategy = 'bar' | 'number' | 'passFail';\n\nexport interface ScoreItem<TData = unknown> {\n  readonly id: string;\n  readonly data: TData;\n  readonly passed?: boolean;\n}\n\nexport interface ScoreDef<TData = unknown> {\n  readonly id: string;\n  readonly name?: string;\n  readonly displayStrategy: ScoreDisplayStrategy;\n  format(data: TData): string;\n  make(\n    data: TData,\n    options?: { definePassed?: (data: TData) => boolean },\n  ): ScoreItem<TData>;\n}\n\nexport const Score = {\n  of<TData>(config: {\n    id: string;\n    name?: string;\n    displayStrategy: ScoreDisplayStrategy;\n    format: (data: TData) => string;\n  }): ScoreDef<TData> {\n    const def: ScoreDef<TData> = {\n      id: config.id,\n      name: config.name,\n      displayStrategy: config.displayStrategy,\n      format: config.format,\n      make: (data: TData, options?: { definePassed?: (data: TData) => boolean }) => {\n        const passed =\n          options?.definePassed !== undefined\n            ? options.definePassed(data)\n            : undefined;\n        return {\n          id: config.id,\n          data,\n          ...(passed !== undefined && { passed }),\n        };\n      },\n    };\n    registry.set(config.id, def as ScoreDef<unknown>);\n    return def;\n  },\n};\n\nexport function getScoreById(id: string): ScoreDef<unknown> | undefined {\n  return registry.get(id);\n}\n","import { Metric } from '../metric';\n\nexport interface TokenCountData {\n  input?: number;\n  output?: number;\n  inputCached?: number;\n  outputCached?: number;\n}\n\nexport const tokenCountMetric = Metric.of<TokenCountData>({\n  id: 'token-count',\n  name: 'Tokens',\n  format: (data) => {\n    const input = data.input ?? 0;\n    const output = data.output ?? 0;\n    const inputCached = data.inputCached ?? 0;\n    const outputCached = data.outputCached ?? 0;\n    const cached = inputCached + outputCached;\n    return `in:${input} out:${output} cached:${cached}`;\n  },\n});\n\nexport interface LatencyData {\n  ms: number;\n}\n\nexport const latencyMetric = Metric.of<LatencyData>({\n  id: 'latency',\n  name: 'Latency',\n  format: (data) => `${data.ms}ms`,\n});\n","import { Score } from '../score';\n\nexport interface PercentScoreData {\n  value: number;\n}\n\nexport const percentScore = Score.of<PercentScoreData>({\n  id: 'percent',\n  name: 'Score',\n  displayStrategy: 'bar',\n  format: (data) => data.value.toFixed(2),\n});\n\nexport interface BinaryScoreData {\n  passed: boolean;\n}\n\nexport const binaryScore = Score.of<BinaryScoreData>({\n  id: 'binary',\n  name: 'Result',\n  displayStrategy: 'passFail',\n  format: (data) => (data.passed ? 'PASSED' : 'NOT PASSED'),\n});\n","import type { ScoreItem } from '../evals/score';\nimport { getScoreById } from '../evals';\n\nexport function toNumericScoreFromScores(\n  scores: ReadonlyArray<ScoreItem>,\n): number | undefined {\n  for (const item of scores) {\n    const def = getScoreById(item.id);\n    if (def && def.displayStrategy === 'bar' && typeof item.data === 'object' && item.data !== null && 'value' in item.data) {\n      const value = (item.data as { value: unknown }).value;\n      if (typeof value === 'number' && Number.isFinite(value)) {\n        return value;\n      }\n    }\n    const numeric = toNumericScore(item.data);\n    if (numeric !== undefined) {\n      return numeric;\n    }\n  }\n  return undefined;\n}\n\nexport function toNumericScore(value: unknown): number | undefined {\n  if (typeof value === 'number' && Number.isFinite(value)) {\n    return value;\n  }\n  if (typeof value !== 'object' || value === null) {\n    return undefined;\n  }\n  const obj = value as Record<string, unknown>;\n  if (\n    'score' in obj &&\n    typeof obj.score === 'number' &&\n    Number.isFinite(obj.score)\n  ) {\n    return obj.score;\n  }\n  const numberValues = Object.values(value).filter(\n    (entry): entry is number =>\n      typeof entry === 'number' && Number.isFinite(entry),\n  );\n  if (numberValues.length === 0) {\n    return undefined;\n  }\n  return (\n    numberValues.reduce((sum, entry) => sum + entry, 0) / numberValues.length\n  );\n}\n","import { appendFile, mkdir } from 'node:fs/promises';\nimport { dirname } from 'node:path';\n\nimport { Effect, Queue } from 'effect';\n\nexport interface PersistenceMessage {\n  runId: string;\n  artifactPath: string;\n  payload: unknown;\n}\n\nasync function appendJsonLine(\n  artifactPath: string,\n  payload: unknown,\n): Promise<void> {\n  await mkdir(dirname(artifactPath), { recursive: true });\n  await appendFile(artifactPath, `${JSON.stringify(payload)}\\n`, 'utf8');\n}\n\nexport const createPersistenceWorker = (\n  queue: Queue.Queue<PersistenceMessage>,\n): Effect.Effect<never, never, never> =>\n  Effect.forever(\n    Effect.gen(function* () {\n      const message = yield* Queue.take(queue);\n      yield* Effect.promise(() =>\n        appendJsonLine(message.artifactPath, {\n          runId: message.runId,\n          ts: Date.now(),\n          ...message.payload,\n        }),\n      );\n    }),\n  );\n","import type { CollectedTestCase, SearchTestCasesQuery } from './events';\n\nfunction matchesAny(\n  value: string,\n  matchers: ReadonlyArray<string | RegExp> | undefined,\n): boolean {\n  if (!matchers || matchers.length === 0) {\n    return true;\n  }\n  return matchers.some((matcher) =>\n    typeof matcher === 'string' ? matcher === value : matcher.test(value),\n  );\n}\n\nfunction matchesPath(\n  value: string,\n  matchers: ReadonlyArray<string | RegExp> | undefined,\n): boolean {\n  if (!matchers || matchers.length === 0) {\n    return true;\n  }\n  return matchers.some((matcher) => {\n    if (typeof matcher === 'string') {\n      return value.includes(matcher);\n    }\n    return matcher.test(value);\n  });\n}\n\nexport function searchCollectedTestCases(\n  all: ReadonlyArray<CollectedTestCase>,\n  query?: SearchTestCasesQuery,\n): ReadonlyArray<CollectedTestCase> {\n  if (!query) {\n    return all;\n  }\n\n  return all.filter((item) => {\n    const tags = item.testCase.getTags();\n\n    if (\n      query.excludedTags &&\n      tags.some((tag) => matchesAny(tag, query.excludedTags))\n    ) {\n      return false;\n    }\n    if (\n      query.excludedPaths &&\n      matchesPath(item.filePath, query.excludedPaths)\n    ) {\n      return false;\n    }\n\n    const includedTagsMatch =\n      !query.includedTags ||\n      query.includedTags.length === 0 ||\n      tags.some((tag) => matchesAny(tag, query.includedTags));\n\n    const includedPathsMatch =\n      !query.includedPaths ||\n      query.includedPaths.length === 0 ||\n      matchesPath(item.filePath, query.includedPaths);\n\n    return includedTagsMatch && includedPathsMatch;\n  });\n}\n","export type SimpleCliCommand = 'run' | 'generate';\n\nexport interface SimpleCliArgs {\n  command?: SimpleCliCommand;\n  datasetName?: string;\n  evaluatorPattern?: string;\n  help: boolean;\n  unknownArgs: string[];\n}\n\nexport function parseSimpleCliArgs(argv: string[]): SimpleCliArgs {\n  const args: SimpleCliArgs = {\n    help: false,\n    unknownArgs: [],\n  };\n  let index = 0;\n  if (argv[0] === 'run' || argv[0] === 'generate') {\n    args.command = argv[0];\n    index = 1;\n  }\n\n  for (; index < argv.length; index += 1) {\n    const token = argv[index];\n    if (token === '--help' || token === '-h') {\n      args.help = true;\n      continue;\n    }\n    if ((token === '--dataset' || token === '--datasetName') && argv[index + 1]) {\n      args.datasetName = argv[index + 1];\n      index += 1;\n      continue;\n    }\n    if ((token === '--evaluator' || token === '--name') && argv[index + 1]) {\n      args.evaluatorPattern = argv[index + 1];\n      index += 1;\n      continue;\n    }\n    args.unknownArgs.push(token);\n  }\n\n  return args;\n}\n\nexport function getSimpleCliUsage(): string {\n  return [\n    'Usage:',\n    '  eval-agents-simple run --dataset <datasetName> --evaluator <name-or-pattern>',\n    '  eval-agents-simple generate --dataset <datasetName>',\n    '',\n    'Pattern examples for --evaluator:',\n    '  score-evaluator       exact name (case-insensitive)',\n    '  \"*score*\"             wildcard pattern',\n    '  \"/score/i\"            regex literal',\n  ].join('\\n');\n}\n","import { writeFile } from 'node:fs/promises';\nimport { join, parse, resolve } from 'node:path';\n\nimport type { RunnerApi } from '../runner';\n\ninterface GeneratedDatasetCase {\n  name: string;\n  input: unknown;\n  outputDefinition?: unknown;\n}\n\nfunction readOutputDefinition(testCase: {\n  getOutputDefinition?: () => unknown;\n}): unknown {\n  if (typeof testCase.getOutputDefinition !== 'function') {\n    return undefined;\n  }\n  return testCase.getOutputDefinition();\n}\n\nfunction createOutputPath(datasetFilePath: string): string {\n  const parsed = parse(datasetFilePath);\n  return join(parsed.dir, `${parsed.name}.cases.json`);\n}\n\nexport async function generateDatasetJsonCommand(\n  runner: RunnerApi,\n  datasetName: string,\n): Promise<void> {\n  const dataset = await runner.resolveDatasetByName(datasetName);\n  if (!dataset) {\n    throw new Error(`Dataset \"${datasetName}\" not found.`);\n  }\n\n  const testCases = await runner.collectDatasetTestCases(dataset.id);\n  const payload: GeneratedDatasetCase[] = testCases.map((item) => ({\n    name: item.testCase.getName(),\n    input: item.testCase.getInput(),\n    outputDefinition: readOutputDefinition(item.testCase),\n  }));\n\n  const absoluteDatasetPath = resolve(process.cwd(), dataset.filePath);\n  const outputPath = createOutputPath(absoluteDatasetPath);\n\n  await writeFile(outputPath, `${JSON.stringify(payload, null, 2)}\\n`, 'utf8');\n\n  console.log(`Generated ${payload.length} test cases for dataset \"${dataset.dataset.getName()}\".`);\n  console.log(`Wrote ${outputPath}`);\n}\n","import { getMetricById, getScoreById } from '../evals';\nimport type { ScoreItem } from '../evals/score';\nimport type { RunnerApi, RunnerEvent } from '../runner';\nimport {\n  toNumericScore,\n  toNumericScoreFromScores,\n} from '../runner/score-utils';\n\ninterface EvaluatorAggregate {\n  total: number;\n  count: number;\n  passed: number;\n  failed: number;\n}\n\ninterface TestCaseScoreSummary {\n  name: string;\n  averageScore?: number;\n  durationMs: number;\n  passed: boolean;\n}\n\nconst ansi = {\n  reset: '\\x1b[0m',\n  bold: '\\x1b[1m',\n  dim: '\\x1b[2m',\n  green: '\\x1b[32m',\n  yellow: '\\x1b[33m',\n  red: '\\x1b[31m',\n  cyan: '\\x1b[36m',\n  magenta: '\\x1b[35m',\n} as const;\n\nfunction colorize(text: string, color: string): string {\n  return `${color}${text}${ansi.reset}`;\n}\n\nfunction scoreToColor(score: number): string {\n  if (score >= 80) {\n    return ansi.green;\n  }\n  if (score >= 50) {\n    return ansi.yellow;\n  }\n  return ansi.red;\n}\n\nfunction getEvaluatorSummaryLine(\n  evaluatorName: string,\n  aggregate: EvaluatorAggregate | undefined,\n): string {\n  if (!aggregate || aggregate.count === 0) {\n    return `- ${evaluatorName.padEnd(28)} no numeric scores`;\n  }\n  const mean = aggregate.total / aggregate.count;\n  return `- ${evaluatorName.padEnd(28)} avg=${colorize(mean.toFixed(2), scoreToColor(mean))} passed=${aggregate.passed} failed=${aggregate.failed}`;\n}\n\nfunction createBar(value: number, max = 100, width = 20): string {\n  const safe = Math.max(0, Math.min(max, value));\n  const filled = Math.round((safe / max) * width);\n  return `${'█'.repeat(filled)}${'░'.repeat(width - filled)}`;\n}\n\nfunction formatEvaluatorScoreLine(\n  name: string,\n  scores: ReadonlyArray<ScoreItem>,\n  passed: boolean,\n  metrics?: ReadonlyArray<{ id: string; data: unknown }>,\n): string {\n  const passLabel = passed\n    ? colorize('PASS', `${ansi.bold}${ansi.green}`)\n    : colorize('FAIL', `${ansi.bold}${ansi.red}`);\n  const scoreParts: string[] = [];\n  for (const item of scores) {\n    const def = getScoreById(item.id);\n    if (!def) {\n      const numeric = toNumericScore(item.data);\n      scoreParts.push(\n        numeric !== undefined\n          ? colorize(numeric.toFixed(2), scoreToColor(numeric))\n          : 'n/a',\n      );\n      continue;\n    }\n    const formatted = def.format(item.data);\n    switch (def.displayStrategy) {\n      case 'bar': {\n        const numeric =\n          typeof item.data === 'object' &&\n          item.data !== null &&\n          'value' in item.data\n            ? (item.data as { value: unknown }).value\n            : toNumericScore(item.data);\n        if (typeof numeric === 'number' && Number.isFinite(numeric)) {\n          scoreParts.push(\n            `${colorize(formatted, scoreToColor(numeric))} ${colorize(createBar(numeric), ansi.dim)}`,\n          );\n        } else {\n          scoreParts.push(formatted);\n        }\n        break;\n      }\n      case 'number':\n        scoreParts.push(formatted);\n        break;\n      case 'passFail':\n        scoreParts.push(\n          colorize(\n            formatted,\n            item.passed === true\n              ? `${ansi.bold}${ansi.green}`\n              : item.passed === false\n                ? `${ansi.bold}${ansi.red}`\n                : ansi.dim,\n          ),\n        );\n        break;\n    }\n  }\n  const scoreStr = scoreParts.length > 0 ? scoreParts.join(' ') : 'n/a';\n  let line = `   ${name}: ${passLabel} ${scoreStr}`;\n  if (metrics && metrics.length > 0) {\n    const metricParts: string[] = [];\n    for (const { id, data } of metrics) {\n      const def = getMetricById(id);\n      if (def) {\n        const formatted = def.format(data);\n        metricParts.push(\n          def.name ? `[${def.name}: ${formatted}]` : `[${formatted}]`,\n        );\n      }\n    }\n    if (metricParts.length > 0) {\n      line += ` ${metricParts.join(' ')}`;\n    }\n  }\n  return line;\n}\n\nexport async function runSimpleEvalCommand(\n  runner: RunnerApi,\n  datasetName: string,\n  evaluatorPattern: string,\n): Promise<void> {\n  const dataset = await runner.resolveDatasetByName(datasetName);\n  if (!dataset) {\n    const known = await runner.collectDatasets();\n    const available = known.map((item) => item.dataset.getName()).sort();\n    throw new Error(\n      available.length > 0\n        ? `Dataset \"${datasetName}\" not found. Available datasets: ${available.join(', ')}`\n        : `Dataset \"${datasetName}\" not found and no datasets were discovered.`,\n    );\n  }\n\n  const evaluators =\n    await runner.resolveEvaluatorsByNamePattern(evaluatorPattern);\n  if (evaluators.length === 0) {\n    const known = await runner.collectEvaluators();\n    const available = known\n      .map((item) => item.evaluator.getName())\n      .filter((name): name is string => typeof name === 'string')\n      .sort();\n    throw new Error(\n      available.length > 0\n        ? `No evaluator matched \"${evaluatorPattern}\". Available evaluators: ${available.join(', ')}`\n        : `No evaluator matched \"${evaluatorPattern}\" and no evaluators were discovered.`,\n    );\n  }\n\n  const evaluatorNameById = new Map(\n    evaluators.map((item) => [item.id, item.evaluator.getName() ?? item.id]),\n  );\n  const aggregates = new Map<string, EvaluatorAggregate>();\n  const testCaseSummaries: TestCaseScoreSummary[] = [];\n  let overallScoreTotal = 0;\n  let overallScoreCount = 0;\n  let completedCount = 0;\n  let totalCount = 0;\n  let runFinished = false;\n  const spinnerFrames = ['⠋', '⠙', '⠸', '⠴', '⠦', '⠇'];\n  let spinnerIndex = 0;\n\n  function clearLine(): void {\n    if (!process.stdout.isTTY) {\n      return;\n    }\n    process.stdout.write('\\r\\x1b[2K');\n  }\n\n  function drawSpinner(): void {\n    if (!process.stdout.isTTY || runFinished) {\n      return;\n    }\n    const frame = spinnerFrames[spinnerIndex % spinnerFrames.length];\n    spinnerIndex += 1;\n    process.stdout.write(\n      `\\r${colorize(frame, ansi.cyan)} Running evaluations ${colorize(\n        `${completedCount}/${totalCount}`,\n        ansi.bold,\n      )} ${colorize('(live)', ansi.dim)}`,\n    );\n  }\n\n  let spinnerTimer: NodeJS.Timeout | undefined;\n  const done = new Promise<RunnerEvent>((resolve) => {\n    const unsubscribe = runner.subscribeRunEvents((event) => {\n      if (event.type === 'TestCaseProgress') {\n        completedCount = event.completedTestCases;\n        const numericScores = event.evaluatorScores\n          .map((item) => toNumericScoreFromScores(item.scores))\n          .filter((item): item is number => item !== undefined);\n        const averageScore =\n          numericScores.length > 0\n            ? numericScores.reduce((sum, value) => sum + value, 0) /\n              numericScores.length\n            : undefined;\n\n        clearLine();\n        console.log(\n          `${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi.cyan)} ${event.testCaseName} ${colorize(`(${event.durationMs}ms)`, ansi.dim)}`,\n        );\n        for (const item of event.evaluatorScores) {\n          const name =\n            evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;\n          console.log(\n            formatEvaluatorScoreLine(\n              name,\n              item.scores,\n              item.passed,\n              item.metrics,\n            ),\n          );\n\n          const numeric = toNumericScoreFromScores(item.scores);\n          if (numeric !== undefined) {\n            const current = aggregates.get(item.evaluatorId) ?? {\n              total: 0,\n              count: 0,\n              passed: 0,\n              failed: 0,\n            };\n            aggregates.set(item.evaluatorId, {\n              total: current.total + numeric,\n              count: current.count + 1,\n              passed: current.passed + (item.passed ? 1 : 0),\n              failed: current.failed + (item.passed ? 0 : 1),\n            });\n            overallScoreTotal += numeric;\n            overallScoreCount += 1;\n          }\n        }\n\n        testCaseSummaries.push({\n          name: event.testCaseName,\n          averageScore,\n          durationMs: event.durationMs,\n          passed: event.passed,\n        });\n\n        drawSpinner();\n      }\n      if (event.type === 'RunCompleted' || event.type === 'RunFailed') {\n        runFinished = true;\n        clearLine();\n        unsubscribe();\n        resolve(event);\n      }\n    });\n  });\n\n  const snapshot = await runner.runDatasetWith({\n    datasetId: dataset.id,\n    evaluatorIds: evaluators.map((item) => item.id),\n  });\n  totalCount = snapshot.totalTestCases;\n\n  console.log(colorize('=== Eval Run Started ===', `${ansi.bold}${ansi.cyan}`));\n  console.log(`Run: ${colorize(snapshot.runId, ansi.cyan)}`);\n  console.log(`Dataset: ${colorize(snapshot.datasetName, ansi.bold)}`);\n  console.log(\n    `Evaluators: ${evaluators\n      .map((item) => item.evaluator.getName() ?? item.id)\n      .join(', ')}`,\n  );\n  console.log(\n    `Total test cases: ${colorize(String(snapshot.totalTestCases), ansi.bold)}`,\n  );\n  console.log('');\n  drawSpinner();\n  spinnerTimer = setInterval(drawSpinner, 100);\n\n  const finalEvent = await done;\n  if (spinnerTimer) {\n    clearInterval(spinnerTimer);\n  }\n\n  if (finalEvent.type === 'RunFailed') {\n    throw new Error(`Run failed: ${finalEvent.errorMessage}`);\n  }\n\n  console.log('');\n  console.log(colorize('=== Run Summary ===', `${ansi.bold}${ansi.cyan}`));\n  console.log(\n    `- passed: ${colorize(\n      `${finalEvent.passedTestCases}/${finalEvent.totalTestCases}`,\n      ansi.green,\n    )}`,\n  );\n  console.log(\n    `- failed: ${colorize(\n      `${finalEvent.failedTestCases}/${finalEvent.totalTestCases}`,\n      finalEvent.failedTestCases > 0 ? ansi.red : ansi.dim,\n    )}`,\n  );\n  if (overallScoreCount > 0) {\n    const overallAverage = overallScoreTotal / overallScoreCount;\n    console.log(\n      `- overall avg score: ${colorize(\n        overallAverage.toFixed(2),\n        scoreToColor(overallAverage),\n      )} ${colorize(createBar(overallAverage), ansi.dim)}`,\n    );\n  }\n  console.log(colorize('- evaluator averages:', ansi.magenta));\n  for (const [evaluatorId, evaluatorName] of evaluatorNameById.entries()) {\n    console.log(\n      getEvaluatorSummaryLine(evaluatorName, aggregates.get(evaluatorId)),\n    );\n  }\n  if (testCaseSummaries.length > 0) {\n    console.log(colorize('- test case scores:', ansi.magenta));\n    for (const summary of testCaseSummaries) {\n      const status = summary.passed\n        ? colorize('PASS', ansi.green)\n        : colorize('FAIL', ansi.red);\n      if (summary.averageScore === undefined) {\n        console.log(\n          `  ${status} ${summary.name.padEnd(24)} score=n/a ${colorize(`(${summary.durationMs}ms)`, ansi.dim)}`,\n        );\n        continue;\n      }\n      console.log(\n        `  ${status} ${summary.name.padEnd(24)} score=${colorize(\n          summary.averageScore.toFixed(2),\n          scoreToColor(summary.averageScore),\n        )} ${colorize(createBar(summary.averageScore, 100, 14), ansi.dim)} ${colorize(`(${summary.durationMs}ms)`, ansi.dim)}`,\n      );\n    }\n  }\n  console.log(`- artifact: ${colorize(finalEvent.artifactPath, ansi.dim)}`);\n}\n","#!/usr/bin/env node\n\nimport { createRunner } from '../runner';\nimport { getSimpleCliUsage, parseSimpleCliArgs } from './args';\nimport { generateDatasetJsonCommand } from './generate';\nimport { runSimpleEvalCommand } from './run';\n\nfunction printUsageAndExit(exitCode: number): never {\n  const printer = exitCode === 0 ? console.log : console.error;\n  printer(getSimpleCliUsage());\n  process.exit(exitCode);\n}\n\nasync function main(): Promise<void> {\n  const args = parseSimpleCliArgs(process.argv.slice(2));\n\n  if (args.help) {\n    printUsageAndExit(0);\n  }\n  if (args.unknownArgs.length > 0) {\n    console.error(`Unknown arguments: ${args.unknownArgs.join(', ')}`);\n    printUsageAndExit(1);\n  }\n  if (!args.command) {\n    printUsageAndExit(1);\n  }\n  if (!args.datasetName) {\n    console.error('Missing required --dataset <datasetName> argument.');\n    printUsageAndExit(1);\n  }\n\n  const runner = createRunner();\n  try {\n    if (args.command === 'run') {\n      if (!args.evaluatorPattern) {\n        console.error('Missing required --evaluator <name-or-pattern> argument.');\n        printUsageAndExit(1);\n      }\n      await runSimpleEvalCommand(runner, args.datasetName, args.evaluatorPattern);\n      return;\n    }\n\n    await generateDatasetJsonCommand(runner, args.datasetName);\n  } finally {\n    await runner.shutdown();\n  }\n}\n\nvoid main().catch((error: unknown) => {\n  console.error(error instanceof Error ? error.message : 'Command failed');\n  process.exit(1);\n});\n"]}
+{"version":3,"sources":["../src/runner/api.ts","../src/runner/config.ts","../src/runner/config-loader.ts","../src/runner/discovery.ts","../src/runner/execution.ts","../src/evals/metric.ts","../src/evals/score.ts","../src/evals/metrics/standard.ts","../src/evals/scores/standard.ts","../src/runner/score-utils.ts","../src/runner/persistence.ts","../src/runner/search.ts","../src/cli-simple/args.ts","../src/cli-simple/generate.ts","../src/cli-simple/run.ts","../src/cli-simple/index.ts"],"names":["Effect","Queue","createJiti","resolve","jitiModule","loaded","registry","join","readOutput"],"mappings":";;;AAAA,SAAS,kBAAkB;AAE3B,SAAS,UAAAA,SAAQ,OAAO,QAAQ,SAAAC,cAAa;;;ACyCtC,IAAM,sBAAoC;AAAA,EAC/C,WAAW;AAAA,IACT,SAAS,QAAQ,IAAI;AAAA,IACrB,iBAAiB,CAAC,eAAe,gBAAgB,eAAe,cAAc;AAAA,IAC9E,mBAAmB;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,IACA,kBAAkB;AAAA,MAChB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,IACA,oBAAoB,CAAC,gBAAgB,QAAQ,SAAS,QAAQ,aAAa;AAAA,EAC7E;AAAA,EACA,mBAAmB;AACrB;AAEO,SAAS,wBACd,QACmC;AACnC,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,EACT;AAEA,QAAM,eAAe,OAAO;AAC5B,QAAM,YAA4C,CAAC;AACnD,MAAI,cAAc,YAAY,QAAW;AACvC,cAAU,UAAU,aAAa;AAAA,EACnC;AACA,MAAI,cAAc,wBAAwB,QAAW;AACnD,cAAU,kBAAkB,aAAa;AAAA,EAC3C,WAAW,cAAc,oBAAoB,QAAW;AACtD,cAAU,kBAAkB,aAAa;AAAA,EAC3C;AACA,MAAI,cAAc,0BAA0B,QAAW;AACrD,cAAU,oBAAoB,aAAa;AAAA,EAC7C,WAAW,cAAc,sBAAsB,QAAW;AACxD,cAAU,oBAAoB,aAAa;AAAA,EAC7C;AACA,MAAI,cAAc,yBAAyB,QAAW;AACpD,cAAU,mBAAmB,aAAa;AAAA,EAC5C,WAAW,cAAc,qBAAqB,QAAW;AACvD,cAAU,mBAAmB,aAAa;AAAA,EAC5C;AACA,MAAI,cAAc,uBAAuB,QAAW;AAClD,cAAU,qBAAqB,aAAa;AAAA,EAC9C;AAEA,QAAM,YAAmC,CAAC;AAC1C,MAAI,OAAO,sBAAsB,QAAW;AAC1C,cAAU,oBAAoB,OAAO;AAAA,EACvC;AACA,MAAI,OAAO,KAAK,SAAS,EAAE,SAAS,GAAG;AACrC,cAAU,YAAY;AAAA,EACxB;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,WAAiD;AAChF,MAAI,CAAC,WAAW;AACd,WAAO;AAAA,EACT;AACA,QAAM,YAAY,UAAU,YACxB;AAAA,IACE,GAAG,oBAAoB;AAAA,IACvB,GAAG,UAAU;AAAA,EACf,IACA,oBAAoB;AAExB,SAAO;AAAA,IACL,GAAG;AAAA,IACH,GAAG;AAAA,IACH;AAAA,EACF;AACF;;;ACzHA,SAAS,kBAAkB;AAC3B,SAAS,eAAe;AAExB,YAAY,gBAAgB;AAS5B,IAAM,mBAAmB;AAOzB,IAAI;AAEJ,SAAS,gBAA4B;AACnC,MAAI,cAAc;AAChB,WAAO;AAAA,EACT;AACA,QAAMC,cACwD,yBACtB;AACxC,MAAI,OAAOA,gBAAe,YAAY;AACpC,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,iBACEA,YACA,YAAY,KAAK;AAAA,IACjB,gBAAgB;AAAA,IAChB,aAAa;AAAA,EACf,CAAC;AACD,SAAO;AACT;AAEA,SAAS,0BAA0B,cAAgC;AACjE,MACE,gBACA,OAAO,iBAAiB,YACxB,aAAa,cACb;AACA,WAAQ,aAAsC;AAAA,EAChD;AACA,SAAO;AACT;AAEA,SAAS,mBAAmB,OAAwC;AAClE,MAAI,UAAU,UAAa,UAAU,MAAM;AACzC,WAAO;AAAA,EACT;AACA,MAAI,OAAO,UAAU,YAAY;AAC/B,WAAQ,MAA8C;AAAA,EACxD;AACA,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,qBACd,MAAM,QAAQ,IAAI,GACiB;AACnC,QAAM,aAAa,QAAQ,KAAK,gBAAgB;AAChD,MAAI,CAAC,WAAW,UAAU,GAAG;AAC3B,WAAO;AAAA,EACT;AACA,QAAM,SAAS,cAAc;AAC7B,QAAM,SAAS,OAAO,UAAU;AAChC,QAAM,gBAAgB,0BAA0B,MAAM;AACtD,QAAM,SAAS,mBAAmB,aAAa;AAC/C,SAAO,wBAAwB,MAAM;AACvC;;;AC/EA,SAAS,eAAe;AACxB,SAAS,WAAAC,UAAS,gBAAgB;AAClC,SAAS,qBAAqB;AAiB9B,IAAI;AAEJ,SAAS,KAAK,QAAgB,UAAkB,MAAuB;AACrE,QAAM,SAAS,QAAQ,KAAK,KAAK,EAAE,SAAS,IAAI,OAAO;AACvD,SAAO,GAAG,MAAM,IAAI,MAAM,GACvB,YAAY,EACZ,QAAQ,eAAe,GAAG,EAC1B,QAAQ,YAAY,EAAE;AAC3B;AAEA,SAAS,UAAU,OAAgB,YAA6B;AAC9D,SACE,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,OAAQ,MAAkC,UAAU,MAAM;AAE9D;AAEA,SAAS,cAAc,OAAkC;AACvD,SAAO,UAAU,OAAO,SAAS,KAAK,UAAU,OAAO,iBAAiB;AAC1E;AAEA,SAAS,gBACP,OACwD;AACxD,SACE,UAAU,OAAO,SAAS,KAC1B,UAAU,OAAO,gBAAgB,KACjC,UAAU,OAAO,eAAe;AAEpC;AAEA,SAAS,eAAe,OAA4C;AAClE,SACE,UAAU,OAAO,SAAS,KAC1B,UAAU,OAAO,SAAS,KAC1B,UAAU,OAAO,UAAU;AAE/B;AAEA,eAAe,cACb,SACA,oBACmB;AACnB,QAAM,MAAgB,CAAC;AAEvB,iBAAe,KAAK,YAAmC;AACrD,QAAI;AACJ,QAAI;AACF,gBAAU,MAAM,QAAQ,YAAY,EAAE,eAAe,KAAK,CAAC;AAAA,IAC7D,QAAQ;AACN;AAAA,IACF;AAEA,UAAM,QAAQ;AAAA,MACZ,QAAQ,IAAI,OAAO,UAAU;AAC3B,cAAM,WAAWA,SAAQ,YAAY,MAAM,IAAI;AAC/C,YAAI,MAAM,YAAY,GAAG;AACvB,cAAI,mBAAmB,SAAS,MAAM,IAAI,GAAG;AAC3C;AAAA,UACF;AACA,gBAAM,KAAK,QAAQ;AACnB;AAAA,QACF;AAEA,YAAI,MAAM,OAAO,GAAG;AAClB,cAAI,KAAK,QAAQ;AAAA,QACnB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,KAAK,OAAO;AAClB,SAAO;AACT;AAEA,SAAS,aACP,UACA,UACS;AACT,SAAO,SAAS,KAAK,CAAC,WAAW,SAAS,SAAS,MAAM,CAAC;AAC5D;AAEA,eAAe,kBAAkB,UAAsC;AACrE,MAAI,SAAS,SAAS,KAAK,KAAK,SAAS,SAAS,MAAM,GAAG;AACzD,QAAI,CAAC,YAAY;AACf,YAAMC,cAAc,MAAM,OAAO,MAAM;AAIvC,YAAMF,cAAaE,YAAW,cAAcA,YAAW;AACvD,UAAI,CAACF,aAAY;AACf,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AACA,mBAAaA,YAAW,YAAY,KAAK;AAAA,QACvC,gBAAgB;AAAA,QAChB,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AACA,UAAMG,UAAS,WAAW,SACtB,MAAM,WAAW,OAAO,QAAQ,IAChC,MAAM,QAAQ,QAAQ,WAAW,QAAQ,CAAC;AAC9C,WAAO,OAAO,OAAOA,OAAiC;AAAA,EACxD;AAEA,QAAM,YAAY,cAAc,QAAQ,EAAE;AAC1C,QAAM,SAAU,MAAM,OAAO;AAC7B,SAAO,OAAO,OAAO,MAAM;AAC7B;AAEA,eAAsB,yBACpB,QAC0C;AAC1C,QAAM,QAAQ,MAAM,cAAc,OAAO,SAAS,OAAO,kBAAkB;AAC3E,QAAM,UAAU,MAAM;AAAA,IAAO,CAAC,aAC5B,aAAa,UAAU,OAAO,eAAe;AAAA,EAC/C;AAEA,QAAM,QAAQ,MAAM,QAAQ;AAAA,IAC1B,QAAQ,IAAI,OAAO,iBAAiB;AAClC,YAAM,UAAU,MAAM,kBAAkB,YAAY;AACpD,YAAM,WAAW,QAAQ,OAAO,aAAa;AAC7C,YAAM,UAAU,SAAS,OAAO,SAAS,YAAY;AACrD,aAAO,SAAS,IAAI,CAAC,aAAa;AAAA,QAChC,IAAI,KAAK,WAAW,SAAS,QAAQ,QAAQ,CAAC;AAAA,QAC9C,UAAU;AAAA,QACV;AAAA,MACF,EAAE;AAAA,IACJ,CAAC;AAAA,EACH;AAEA,SAAO,MAAM,KAAK;AACpB;AAEA,eAAsB,2BACpB,QAC4C;AAC5C,QAAM,QAAQ,MAAM,cAAc,OAAO,SAAS,OAAO,kBAAkB;AAC3E,QAAM,UAAU,MAAM;AAAA,IAAO,CAAC,aAC5B,aAAa,UAAU,OAAO,iBAAiB;AAAA,EACjD;AAEA,QAAM,QAAQ,MAAM,QAAQ;AAAA,IAC1B,QAAQ,IAAI,OAAO,iBAAiB;AAClC,YAAM,UAAU,MAAM,kBAAkB,YAAY;AACpD,YAAM,aAAa,QAAQ,OAAO,eAAe;AACjD,YAAM,UAAU,SAAS,OAAO,SAAS,YAAY;AACrD,aAAO,WAAW,IAAI,CAAC,eAAe;AAAA,QACpC,IAAI,KAAK,aAAa,SAAS,UAAU,QAAQ,CAAC;AAAA,QAClD,UAAU;AAAA,QACV;AAAA,MACF,EAAE;AAAA,IACJ,CAAC;AAAA,EACH;AAEA,SAAO,MAAM,KAAK;AACpB;AAEA,eAAsB,0BACpB,QAC2C;AAC3C,QAAM,QAAQ,MAAM,cAAc,OAAO,SAAS,OAAO,kBAAkB;AAC3E,QAAM,UAAU,MAAM;AAAA,IAAO,CAAC,aAC5B,aAAa,UAAU,OAAO,gBAAgB;AAAA,EAChD;AAEA,QAAM,QAAQ,MAAM,QAAQ;AAAA,IAC1B,QAAQ,IAAI,OAAO,iBAAiB;AAClC,YAAM,UAAU,MAAM,kBAAkB,YAAY;AACpD,YAAM,YAAY,QAAQ,OAAO,cAAc;AAC/C,YAAM,UAAU,SAAS,OAAO,SAAS,YAAY;AACrD,aAAO,UAAU,IAAI,CAAC,cAAc;AAAA,QAClC,IAAI,KAAK,aAAa,SAAS,SAAS,QAAQ,CAAC;AAAA,QACjD,UAAU;AAAA,QACV;AAAA,MACF,EAAE;AAAA,IACJ,CAAC;AAAA,EACH;AAEA,SAAO,MAAM,KAAK;AACpB;;;ACzMA,SAAS,YAAY;AAErB,SAAS,QAAQ,aAAa;;;ACF9B,IAAM,WAAW,oBAAI,IAAgC;AAc9C,IAAM,SAAS;AAAA,EACpB,GAAU,QAIW;AACnB,UAAM,MAAwB;AAAA,MAC5B,IAAI,OAAO;AAAA,MACX,MAAM,OAAO;AAAA,MACb,QAAQ,OAAO;AAAA,MACf,MAAM,CAAC,UAAiB,EAAE,IAAI,OAAO,IAAI,KAAK;AAAA,IAChD;AACA,aAAS,IAAI,OAAO,IAAI,GAAyB;AACjD,WAAO;AAAA,EACT;AACF;AAEO,SAAS,cAAc,IAA4C;AACxE,SAAO,SAAS,IAAI,EAAE;AACxB;;;ACjCA,IAAMC,YAAW,oBAAI,IAA+B;AAqB7C,IAAM,QAAQ;AAAA,EACnB,GAAU,QAKU;AAClB,UAAM,MAAuB;AAAA,MAC3B,IAAI,OAAO;AAAA,MACX,MAAM,OAAO;AAAA,MACb,iBAAiB,OAAO;AAAA,MACxB,QAAQ,OAAO;AAAA,MACf,MAAM,CAAC,MAAa,YAA0D;AAC5E,cAAM,SACJ,SAAS,iBAAiB,SACtB,QAAQ,aAAa,IAAI,IACzB;AACN,eAAO;AAAA,UACL,IAAI,OAAO;AAAA,UACX;AAAA,UACA,GAAI,WAAW,UAAa,EAAE,OAAO;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AACA,IAAAA,UAAS,IAAI,OAAO,IAAI,GAAwB;AAChD,WAAO;AAAA,EACT;AACF;AAEO,SAAS,aAAa,IAA2C;AACtE,SAAOA,UAAS,IAAI,EAAE;AACxB;;;AC3CO,IAAM,mBAAmB,OAAO,GAAmB;AAAA,EACxD,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,QAAQ,CAAC,SAAS;AAChB,UAAM,QAAQ,KAAK,SAAS;AAC5B,UAAM,SAAS,KAAK,UAAU;AAC9B,UAAM,cAAc,KAAK,eAAe;AACxC,UAAM,eAAe,KAAK,gBAAgB;AAC1C,UAAM,SAAS,cAAc;AAC7B,WAAO,MAAM,KAAK,QAAQ,MAAM,WAAW,MAAM;AAAA,EACnD;AACF,CAAC;AAMM,IAAM,gBAAgB,OAAO,GAAgB;AAAA,EAClD,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,QAAQ,CAAC,SAAS,GAAG,KAAK,EAAE;AAC9B,CAAC;;;ACxBM,IAAM,eAAe,MAAM,GAAqB;AAAA,EACrD,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,iBAAiB;AAAA,EACjB,QAAQ,CAAC,SAAS,KAAK,MAAM,QAAQ,CAAC;AACxC,CAAC;AAMM,IAAM,cAAc,MAAM,GAAoB;AAAA,EACnD,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,iBAAiB;AAAA,EACjB,QAAQ,CAAC,SAAU,KAAK,SAAS,WAAW;AAC9C,CAAC;;;ACnBM,SAAS,yBACd,QACoB;AACpB,aAAW,QAAQ,QAAQ;AACzB,UAAM,MAAM,aAAa,KAAK,EAAE;AAChC,QAAI,OAAO,IAAI,oBAAoB,SAAS,OAAO,KAAK,SAAS,YAAY,KAAK,SAAS,QAAQ,WAAW,KAAK,MAAM;AACvH,YAAM,QAAS,KAAK,KAA4B;AAChD,UAAI,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,GAAG;AACvD,eAAO;AAAA,MACT;AAAA,IACF;AACA,UAAM,UAAU,eAAe,KAAK,IAAI;AACxC,QAAI,YAAY,QAAW;AACzB,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,eAAe,OAAoC;AACjE,MAAI,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,GAAG;AACvD,WAAO;AAAA,EACT;AACA,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AACA,QAAM,MAAM;AACZ,MACE,WAAW,OACX,OAAO,IAAI,UAAU,YACrB,OAAO,SAAS,IAAI,KAAK,GACzB;AACA,WAAO,IAAI;AAAA,EACb;AACA,QAAM,eAAe,OAAO,OAAO,KAAK,EAAE;AAAA,IACxC,CAAC,UACC,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAAA,EACtD;AACA,MAAI,aAAa,WAAW,GAAG;AAC7B,WAAO;AAAA,EACT;AACA,SACE,aAAa,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI,aAAa;AAEvE;;;ALnCA,SAAS,uBACP,WACA,QACA,QACS;AACT,QAAM,mBAAmB,OAAO,OAAO,CAAC,MAAM,YAAY,KAAK,EAAE,WAAW,MAAS;AACrF,MAAI,iBAAiB,SAAS,GAAG;AAC/B,WAAO,iBAAiB,MAAM,CAAC,MAAM,EAAE,WAAW,IAAI;AAAA,EACxD;AACA,QAAM,gBAAgB,UAAU,iBAAiB;AACjD,MAAI,eAAe;AACjB,WAAO,cAAc,MAAM;AAAA,EAC7B;AACA,QAAM,gBAAgB,UAAU,iBAAiB;AACjD,MAAI,kBAAkB,QAAW;AAC/B,UAAM,UAAU,yBAAyB,MAAM;AAC/C,WAAO,YAAY,UAAa,WAAW;AAAA,EAC7C;AACA,SAAO;AACT;AAEA,SAAS,gBACP,QAIA;AACA,MAAI,OAAO,WAAW,YAAY,WAAW,MAAM;AACjD,WAAO,EAAE,QAAQ,CAAC,EAAE;AAAA,EACtB;AACA,QAAM,MAAM;AACZ,QAAM,SAAS,MAAM,QAAQ,IAAI,MAAM,IAClC,IAAI,SACL,CAAC;AACL,QAAM,UAAU,MAAM,QAAQ,IAAI,OAAO,IACpC,IAAI,UACL;AACJ,SAAO,EAAE,QAAQ,QAAQ;AAC3B;AAEA,SAAS,WAAW,UAAkD;AACpE,QAAM,YAAY;AAClB,MAAI,OAAO,UAAU,cAAc,YAAY;AAC7C,WAAO;AAAA,EACT;AACA,SAAO,UAAU,UAAU;AAC7B;AAcA,SAAS,gBAAwB;AAC/B,UAAO,oBAAI,KAAK,GAAE,YAAY,EAAE,QAAQ,SAAS,GAAG;AACtD;AAEO,SAAS,mBACd,mBACA,WACA,OACQ;AACR,SAAO;AAAA,IACL;AAAA,IACA,GAAG,SAAS,IAAI,KAAK,IAAI,cAAc,CAAC;AAAA,EAC1C;AACF;AAEO,IAAM,iBAAiB,CAC5B,MACA,cACA,kBACA,mBAKA,OAAO,IAAI,aAAa;AACtB,QAAM,YAAY,KAAK,IAAI;AAC3B,iBAAe,KAAK,OAAO,CAAC,cAAc;AAAA,IACxC,GAAG;AAAA,IACH,QAAQ;AAAA,IACR;AAAA,EACF,EAAE;AACF,SAAO,aAAa;AAAA,IAClB,MAAM;AAAA,IACN,OAAO,KAAK;AAAA,IACZ;AAAA,EACF,CAAC;AAED,MAAI,qBAAqB;AACzB,MAAI,kBAAkB;AACtB,MAAI,kBAAkB;AAEtB,aAAW,gBAAgB,KAAK,WAAW;AACzC,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,kBAKD,CAAC;AACN,QAAI;AACJ,UAAM,SAAS,WAAW,aAAa,QAAQ;AAE/C,eAAW,EAAE,IAAI,aAAa,UAAU,KAAK,KAAK,YAAY;AAC5D,YAAM,aAAa,UAAU,cAAc;AAC3C,UAAI,CAAC,YAAY;AACf;AAAA,MACF;AAEA,UAAI;AACF,cAAM,MAAM,OAAO,OAAO;AAAA,UAAQ,MAChC,QAAQ,QAAQ,UAAU,eAAe,CAAC;AAAA,QAC5C;AACA,cAAM,SAAS,OAAO,OAAO;AAAA,UAAQ,MACnC,QAAQ;AAAA,YACN,WAAW;AAAA,cACT,OAAO,aAAa,SAAS,SAAS;AAAA,cACtC;AAAA,cACA;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AACA,cAAM,EAAE,QAAQ,QAAQ,IAAI,gBAAgB,MAAM;AAClD,cAAM,SAAS,uBAAuB,WAAW,QAAQ,MAAM;AAC/D,wBAAgB,KAAK,EAAE,aAAa,QAAQ,QAAQ,QAAQ,CAAC;AAAA,MAC/D,SAAS,OAAO;AACd,wBACE,iBAAiB,QACb,MAAM,UACN;AACN,wBAAgB,KAAK;AAAA,UACnB;AAAA,UACA,QAAQ,CAAC;AAAA,UACT,QAAQ;AAAA,QACV,CAAC;AAAA,MACH;AAAA,IACF;AAEA,UAAM,iBAAiB,gBAAgB,MAAM,CAAC,MAAM,EAAE,MAAM;AAC5D,0BAAsB;AACtB,QAAI,gBAAgB;AAClB,yBAAmB;AAAA,IACrB,OAAO;AACL,yBAAmB;AAAA,IACrB;AAEA,UAAM,gBAA6B;AAAA,MACjC,MAAM;AAAA,MACN,OAAO,KAAK;AAAA,MACZ,YAAY,aAAa;AAAA,MACzB,cAAc,aAAa,SAAS,QAAQ;AAAA,MAC5C;AAAA,MACA,gBAAgB,KAAK,UAAU;AAAA,MAC/B,QAAQ;AAAA,MACR,YAAY,KAAK,IAAI,IAAI;AAAA,MACzB;AAAA,MACA;AAAA,MACA,cAAc;AAAA,IAChB;AAEA,mBAAe,KAAK,OAAO,CAAC,cAAc;AAAA,MACxC,GAAG;AAAA,MACH;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE;AAEF,WAAO,aAAa,aAAa;AACjC,WAAO,MAAM,MAAM,kBAAkB;AAAA,MACnC,OAAO,KAAK;AAAA,MACZ,cAAc,KAAK,SAAS;AAAA,MAC5B,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,QAAM,aAAa,KAAK,IAAI;AAC5B,QAAM,iBAA8B;AAAA,IAClC,MAAM;AAAA,IACN,OAAO,KAAK;AAAA,IACZ;AAAA,IACA;AAAA,IACA;AAAA,IACA,gBAAgB,KAAK,UAAU;AAAA,IAC/B,cAAc,KAAK,SAAS;AAAA,EAC9B;AAEA,iBAAe,KAAK,OAAO,CAAC,cAAc;AAAA,IACxC,GAAG;AAAA,IACH,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE;AAEF,SAAO,aAAa,cAAc;AAClC,SAAO,MAAM,MAAM,kBAAkB;AAAA,IACnC,OAAO,KAAK;AAAA,IACZ,cAAc,KAAK,SAAS;AAAA,IAC5B,SAAS;AAAA,EACX,CAAC;AACD,SAAO,aAAa;AAAA,IAClB,MAAM;AAAA,IACN,OAAO,KAAK;AAAA,IACZ,cAAc,KAAK,SAAS;AAAA,EAC9B,CAAC;AACH,CAAC;;;AMnOH,SAAS,YAAY,aAAa;AAClC,SAAS,eAAe;AAExB,SAAS,UAAAN,SAAQ,SAAAC,cAAa;AAQ9B,eAAe,eACb,cACA,SACe;AACf,QAAM,MAAM,QAAQ,YAAY,GAAG,EAAE,WAAW,KAAK,CAAC;AACtD,QAAM,WAAW,cAAc,GAAG,KAAK,UAAU,OAAO,CAAC;AAAA,GAAM,MAAM;AACvE;AAEO,IAAM,0BAA0B,CACrC,UAEAD,QAAO;AAAA,EACLA,QAAO,IAAI,aAAa;AACtB,UAAM,UAAU,OAAOC,OAAM,KAAK,KAAK;AACvC,WAAOD,QAAO;AAAA,MAAQ,MACpB,eAAe,QAAQ,cAAc;AAAA,QACnC,OAAO,QAAQ;AAAA,QACf,IAAI,KAAK,IAAI;AAAA,QACb,GAAG,QAAQ;AAAA,MACb,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACH;;;AC/BF,SAAS,WACP,OACA,UACS;AACT,MAAI,CAAC,YAAY,SAAS,WAAW,GAAG;AACtC,WAAO;AAAA,EACT;AACA,SAAO,SAAS;AAAA,IAAK,CAAC,YACpB,OAAO,YAAY,WAAW,YAAY,QAAQ,QAAQ,KAAK,KAAK;AAAA,EACtE;AACF;AAEA,SAAS,YACP,OACA,UACS;AACT,MAAI,CAAC,YAAY,SAAS,WAAW,GAAG;AACtC,WAAO;AAAA,EACT;AACA,SAAO,SAAS,KAAK,CAAC,YAAY;AAChC,QAAI,OAAO,YAAY,UAAU;AAC/B,aAAO,MAAM,SAAS,OAAO;AAAA,IAC/B;AACA,WAAO,QAAQ,KAAK,KAAK;AAAA,EAC3B,CAAC;AACH;AAEO,SAAS,yBACd,KACA,OACkC;AAClC,MAAI,CAAC,OAAO;AACV,WAAO;AAAA,EACT;AAEA,SAAO,IAAI,OAAO,CAAC,SAAS;AAC1B,UAAM,OAAO,KAAK,SAAS,QAAQ;AAEnC,QACE,MAAM,gBACN,KAAK,KAAK,CAAC,QAAQ,WAAW,KAAK,MAAM,YAAY,CAAC,GACtD;AACA,aAAO;AAAA,IACT;AACA,QACE,MAAM,iBACN,YAAY,KAAK,UAAU,MAAM,aAAa,GAC9C;AACA,aAAO;AAAA,IACT;AAEA,UAAM,oBACJ,CAAC,MAAM,gBACP,MAAM,aAAa,WAAW,KAC9B,KAAK,KAAK,CAAC,QAAQ,WAAW,KAAK,MAAM,YAAY,CAAC;AAExD,UAAM,qBACJ,CAAC,MAAM,iBACP,MAAM,cAAc,WAAW,KAC/B,YAAY,KAAK,UAAU,MAAM,aAAa;AAEhD,WAAO,qBAAqB;AAAA,EAC9B,CAAC;AACH;;;AXpCA,SAAS,kBACP,SAC+C;AAC/C,MAAI,CAAC,QAAQ,WAAW,GAAG,GAAG;AAC5B,WAAO;AAAA,EACT;AACA,QAAM,YAAY,QAAQ,YAAY,GAAG;AACzC,MAAI,aAAa,GAAG;AAClB,WAAO;AAAA,EACT;AACA,SAAO;AAAA,IACL,QAAQ,QAAQ,MAAM,GAAG,SAAS;AAAA,IAClC,OAAO,QAAQ,MAAM,YAAY,CAAC;AAAA,EACpC;AACF;AAEA,SAAS,kBAAkB,SAA6C;AACtE,QAAM,oBAAoB,QAAQ,KAAK;AACvC,QAAM,eAAe,kBAAkB,iBAAiB;AACxD,MAAI,cAAc;AAChB,UAAM,QAAQ,IAAI,OAAO,aAAa,QAAQ,aAAa,KAAK;AAChE,WAAO,CAAC,UAAkB,MAAM,KAAK,KAAK;AAAA,EAC5C;AAEA,MAAI,kBAAkB,SAAS,GAAG,GAAG;AACnC,UAAM,UAAU,kBACb,QAAQ,qBAAqB,MAAM,EACnC,QAAQ,OAAO,IAAI;AACtB,UAAM,QAAQ,IAAI,OAAO,IAAI,OAAO,KAAK,GAAG;AAC5C,WAAO,CAAC,UAAkB,MAAM,KAAK,KAAK;AAAA,EAC5C;AAEA,SAAO,CAAC,UAAkB,MAAM,YAAY,MAAM,kBAAkB,YAAY;AAClF;AAuBA,SAAS,qBACP,MACA,MACmC;AACnC,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,QAAM,YAAY,KAAK,aAAa,KAAK,YACrC;AAAA,IACE,GAAI,KAAK,aAAa,CAAC;AAAA,IACvB,GAAI,KAAK,aAAa,CAAC;AAAA,EACzB,IACA;AACJ,SAAO;AAAA,IACL,GAAG;AAAA,IACH,GAAG;AAAA,IACH;AAAA,EACF;AACF;AAEO,SAAS,aAAa,WAA8C;AACzE,QAAM,gBAAgB,qBAAqB;AAC3C,QAAM,SAAS,qBAAqB,eAAe,SAAS;AAC5D,SAAO,IAAI,aAAa,iBAAiB,MAAM,CAAC;AAClD;AAEA,IAAM,eAAN,MAAwC;AAAA,EAiCtC,YAAY,QAAsB;AA9BlC,SAAiB,WAAWA,QAAO,QAAQ,OAAO,UAAuB,CAAC;AAE1E,SAAiB,WAAWA,QAAO,QAAQC,OAAM,UAAmB,CAAC;AAErE,SAAiB,mBAAmBD,QAAO;AAAA,MACzCC,OAAM,UAIH;AAAA,IACL;AAEA,SAAiB,YAAY,oBAAI,IAAyB;AAC1D,SAAiB,YAAY,oBAAI,IAG9B;AAEH,SAAiB,eAAe,oBAAI,IAA8B;AAElE,SAAiB,iBAAiB,oBAAI,IAAgC;AAEtE,SAAiB,iBAAiBD,QAAO;AAAA,MACvC,KAAK,sBAAsB;AAAA,IAC7B;AAEA,SAAiB,mBAAmBA,QAAO;AAAA,MACzC,wBAAwB,KAAK,gBAAgB;AAAA,IAC/C;AAGE,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,kBAA4D;AAChE,UAAM,WAAW,MAAM,yBAAyB,KAAK,OAAO,SAAS;AACrE,SAAK,aAAa,MAAM;AACxB,eAAW,WAAW,UAAU;AAC9B,WAAK,aAAa,IAAI,QAAQ,IAAI,OAAO;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAAgE;AACpE,UAAM,aAAa,MAAM,2BAA2B,KAAK,OAAO,SAAS;AACzE,SAAK,eAAe,MAAM;AAC1B,eAAW,aAAa,YAAY;AAClC,WAAK,eAAe,IAAI,UAAU,IAAI,SAAS;AAAA,IACjD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,qBAAqB,MAAqD;AAC9E,QAAI,KAAK,aAAa,SAAS,GAAG;AAChC,YAAM,KAAK,gBAAgB;AAAA,IAC7B;AACA,UAAM,aAAa,KAAK,KAAK,EAAE,YAAY;AAC3C,WAAO,MAAM,KAAK,KAAK,aAAa,OAAO,CAAC,EAAE;AAAA,MAC5C,CAAC,SAAS,KAAK,QAAQ,QAAQ,EAAE,YAAY,MAAM;AAAA,IACrD;AAAA,EACF;AAAA,EAEA,MAAM,+BACJ,SAC4C;AAC5C,QAAI,KAAK,eAAe,SAAS,GAAG;AAClC,YAAM,KAAK,kBAAkB;AAAA,IAC/B;AACA,UAAM,UAAU,kBAAkB,OAAO;AACzC,WAAO,MAAM,KAAK,KAAK,eAAe,OAAO,CAAC,EAAE;AAAA,MAAO,CAAC,SACtD,QAAQ,KAAK,UAAU,QAAQ,KAAK,EAAE;AAAA,IACxC;AAAA,EACF;AAAA,EAEA,MAAM,gBACJ,OAC2C;AAC3C,UAAM,YAAY,MAAM,0BAA0B,KAAK,OAAO,SAAS;AACvE,WAAO,yBAAyB,WAAW,KAAK;AAAA,EAClD;AAAA,EAEA,MAAM,wBACJ,WAC2C;AAC3C,QAAI,KAAK,aAAa,SAAS,GAAG;AAChC,YAAM,KAAK,gBAAgB;AAAA,IAC7B;AACA,UAAM,UAAU,KAAK,aAAa,IAAI,SAAS;AAC/C,QAAI,CAAC,SAAS;AACZ,YAAM,IAAI,MAAM,oBAAoB,SAAS,EAAE;AAAA,IACjD;AACA,UAAM,eAAe,MAAM,0BAA0B,KAAK,OAAO,SAAS;AAC1E,WAAO,aAAa;AAAA,MAAO,CAAC,aAC1B,QAAQ,QAAQ,gBAAgB,SAAS,UAAU,SAAS,QAAQ;AAAA,IACtE;AAAA,EACF;AAAA,EAEA,MAAM,eAAe,SAAkD;AACrE,QAAI,KAAK,aAAa,SAAS,GAAG;AAChC,YAAM,KAAK,gBAAgB;AAAA,IAC7B;AACA,QAAI,KAAK,eAAe,SAAS,GAAG;AAClC,YAAM,KAAK,kBAAkB;AAAA,IAC/B;AAEA,UAAM,UAAU,KAAK,aAAa,IAAI,QAAQ,SAAS;AACvD,QAAI,CAAC,SAAS;AACZ,YAAM,IAAI,MAAM,oBAAoB,QAAQ,SAAS,EAAE;AAAA,IACzD;AAEA,UAAM,qBAAqB,QAAQ,aAChC,IAAI,CAAC,OAAO,KAAK,eAAe,IAAI,EAAE,CAAC,EACvC,OAAO,CAAC,UAAuC,QAAQ,KAAK,CAAC,EAC7D,IAAI,CAAC,WAAW,EAAE,IAAI,MAAM,IAAI,WAAW,MAAM,UAAU,EAAE;AAEhE,QAAI,mBAAmB,WAAW,GAAG;AACnC,YAAM,IAAI,MAAM,gCAAgC;AAAA,IAClD;AAEA,UAAM,oBAAoB,MAAM,KAAK,wBAAwB,QAAQ,SAAS;AAE9E,UAAM,QAAQ,OAAO,WAAW,CAAC;AACjC,UAAM,eAAe;AAAA,MACnB,KAAK,OAAO;AAAA,MACZ,QAAQ;AAAA,MACR;AAAA,IACF;AACA,UAAM,WAAwB;AAAA,MAC5B;AAAA,MACA,WAAW,QAAQ;AAAA,MACnB,aAAa,QAAQ,QAAQ,QAAQ;AAAA,MACrC,cAAc,mBAAmB,IAAI,CAAC,SAAS,KAAK,EAAE;AAAA,MACtD,UAAU,KAAK,IAAI;AAAA,MACnB,gBAAgB,kBAAkB;AAAA,MAClC,oBAAoB;AAAA,MACpB,iBAAiB;AAAA,MACjB,iBAAiB;AAAA,MACjB,QAAQ;AAAA,MACR;AAAA,IACF;AAEA,SAAK,UAAU,IAAI,OAAO,QAAQ;AAClC,UAAM,cAA2B;AAAA,MAC/B,MAAM;AAAA,MACN;AAAA,MACA,WAAW,QAAQ;AAAA,MACnB,aAAa,QAAQ,QAAQ,QAAQ;AAAA,MACrC,cAAc,mBAAmB,IAAI,CAAC,SAAS,KAAK,EAAE;AAAA,MACtD,gBAAgB,kBAAkB;AAAA,MAClC;AAAA,IACF;AACA,UAAMA,QAAO,WAAW,KAAK,aAAa,WAAW,CAAC;AACtD,UAAMA,QAAO;AAAA,MACXC,OAAM,MAAM,KAAK,kBAAkB;AAAA,QACjC;AAAA,QACA;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAEA,UAAMD,QAAO;AAAA,MACXC,OAAM,MAAM,KAAK,UAAU;AAAA,QACzB;AAAA,QACA,WAAW,QAAQ;AAAA,QACnB,SAAS,QAAQ;AAAA,QACjB,YAAY;AAAA,QACZ,WAAW;AAAA,QACX;AAAA,MACF,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,mBACE,UACA,SACY;AACZ,UAAM,QAAQ,EAAE,OAAO,SAAS,OAAO,SAAS;AAChD,SAAK,UAAU,IAAI,KAAK;AACxB,WAAO,MAAM;AACX,WAAK,UAAU,OAAO,KAAK;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,eAAe,OAAwC;AACrD,WAAO,KAAK,UAAU,IAAI,KAAK;AAAA,EACjC;AAAA,EAEA,qBAAiD;AAC/C,WAAO,MAAM,KAAK,KAAK,UAAU,OAAO,CAAC,EAAE;AAAA,MACzC,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,MAAM,WAA0B;AAC9B,UAAMD,QAAO,WAAW,MAAM,UAAU,KAAK,cAAc,CAAC;AAC5D,UAAMA,QAAO,WAAW,MAAM,UAAU,KAAK,gBAAgB,CAAC;AAC9D,UAAMA,QAAO,WAAWC,OAAM,SAAS,KAAK,QAAQ,CAAC;AACrD,UAAMD,QAAO,WAAWC,OAAM,SAAS,KAAK,gBAAgB,CAAC;AAC7D,UAAMD,QAAO,WAAW,OAAO,SAAS,KAAK,QAAQ,CAAC;AAAA,EACxD;AAAA,EAEQ,wBAAwB;AAC9B,UAAM,OAAO;AACb,WAAOA,QAAO;AAAA,MACZA,QAAO,IAAI,aAAa;AACtB,cAAM,OAAO,OAAOC,OAAM,KAAK,KAAK,QAAQ;AAC5C,eAAOD,QAAO;AAAA,UACZ;AAAA,YACE;AAAA,YACA,KAAK,aAAa,KAAK,IAAI;AAAA,YAC3B,KAAK;AAAA,YACL,KAAK,eAAe,KAAK,IAAI;AAAA,UAC/B;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,eACN,OACA,SACM;AACN,UAAM,WAAW,KAAK,UAAU,IAAI,KAAK;AACzC,QAAI,CAAC,UAAU;AACb;AAAA,IACF;AACA,SAAK,UAAU,IAAI,OAAO,QAAQ,QAAQ,CAAC;AAAA,EAC7C;AAAA,EAEQ,aAAa,OAAuD;AAC1E,WAAOA,QAAO,KAAK,MAAM;AACvB,iBAAW,SAAS,KAAK,WAAW;AAClC,YAAI,MAAM,SAAS,MAAM,UAAU,MAAM,OAAO;AAC9C;AAAA,QACF;AACA,cAAM,SAAS,KAAK;AAAA,MACtB;AAAA,IACF,CAAC,EAAE;AAAA,MACDA,QAAO,QAAQ,MAAM,OAAO,QAAQ,KAAK,UAAU,KAAK,CAAC;AAAA,MACzDA,QAAO;AAAA,IACT;AAAA,EACF;AACF;;;AY/VO,SAAS,mBAAmB,MAA+B;AAChE,QAAM,OAAsB;AAAA,IAC1B,MAAM;AAAA,IACN,aAAa,CAAC;AAAA,EAChB;AACA,MAAI,QAAQ;AACZ,MAAI,KAAK,CAAC,MAAM,SAAS,KAAK,CAAC,MAAM,YAAY;AAC/C,SAAK,UAAU,KAAK,CAAC;AACrB,YAAQ;AAAA,EACV;AAEA,SAAO,QAAQ,KAAK,QAAQ,SAAS,GAAG;AACtC,UAAM,QAAQ,KAAK,KAAK;AACxB,QAAI,UAAU,YAAY,UAAU,MAAM;AACxC,WAAK,OAAO;AACZ;AAAA,IACF;AACA,SAAK,UAAU,eAAe,UAAU,oBAAoB,KAAK,QAAQ,CAAC,GAAG;AAC3E,WAAK,cAAc,KAAK,QAAQ,CAAC;AACjC,eAAS;AACT;AAAA,IACF;AACA,SAAK,UAAU,iBAAiB,UAAU,aAAa,KAAK,QAAQ,CAAC,GAAG;AACtE,WAAK,mBAAmB,KAAK,QAAQ,CAAC;AACtC,eAAS;AACT;AAAA,IACF;AACA,SAAK,YAAY,KAAK,KAAK;AAAA,EAC7B;AAEA,SAAO;AACT;AAEO,SAAS,oBAA4B;AAC1C,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;ACtDA,SAAS,iBAAiB;AAC1B,SAAS,QAAAO,OAAM,OAAO,WAAAJ,gBAAe;AAUrC,SAASK,YAAW,UAAkD;AACpE,MAAI,OAAO,SAAS,cAAc,YAAY;AAC5C,WAAO;AAAA,EACT;AACA,SAAO,SAAS,UAAU;AAC5B;AAEA,SAAS,iBAAiB,iBAAiC;AACzD,QAAM,SAAS,MAAM,eAAe;AACpC,SAAOD,MAAK,OAAO,KAAK,GAAG,OAAO,IAAI,aAAa;AACrD;AAEA,eAAsB,2BACpB,QACA,aACe;AACf,QAAM,UAAU,MAAM,OAAO,qBAAqB,WAAW;AAC7D,MAAI,CAAC,SAAS;AACZ,UAAM,IAAI,MAAM,YAAY,WAAW,cAAc;AAAA,EACvD;AAEA,QAAM,YAAY,MAAM,OAAO,wBAAwB,QAAQ,EAAE;AACjE,QAAM,UAAkC,UAAU,IAAI,CAAC,UAAU;AAAA,IAC/D,MAAM,KAAK,SAAS,QAAQ;AAAA,IAC5B,OAAO,KAAK,SAAS,SAAS;AAAA,IAC9B,QAAQC,YAAW,KAAK,QAAQ;AAAA,EAClC,EAAE;AAEF,QAAM,sBAAsBL,SAAQ,QAAQ,IAAI,GAAG,QAAQ,QAAQ;AACnE,QAAM,aAAa,iBAAiB,mBAAmB;AAEvD,QAAM,UAAU,YAAY,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AAAA,GAAM,MAAM;AAE3E,UAAQ,IAAI,aAAa,QAAQ,MAAM,4BAA4B,QAAQ,QAAQ,QAAQ,CAAC,IAAI;AAChG,UAAQ,IAAI,SAAS,UAAU,EAAE;AACnC;;;ACxBA,IAAM,OAAO;AAAA,EACX,OAAO;AAAA,EACP,MAAM;AAAA,EACN,KAAK;AAAA,EACL,OAAO;AAAA,EACP,QAAQ;AAAA,EACR,KAAK;AAAA,EACL,MAAM;AAAA,EACN,SAAS;AACX;AAEA,SAAS,SAAS,MAAc,OAAuB;AACrD,SAAO,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,KAAK;AACrC;AAEA,SAAS,aAAa,OAAuB;AAC3C,MAAI,SAAS,IAAI;AACf,WAAO,KAAK;AAAA,EACd;AACA,MAAI,SAAS,IAAI;AACf,WAAO,KAAK;AAAA,EACd;AACA,SAAO,KAAK;AACd;AAEA,SAAS,wBACP,eACA,WACQ;AACR,MAAI,CAAC,aAAa,UAAU,UAAU,GAAG;AACvC,WAAO,KAAK,cAAc,OAAO,EAAE,CAAC;AAAA,EACtC;AACA,QAAM,OAAO,UAAU,QAAQ,UAAU;AACzC,SAAO,KAAK,cAAc,OAAO,EAAE,CAAC,QAAQ,SAAS,KAAK,QAAQ,CAAC,GAAG,aAAa,IAAI,CAAC,CAAC,WAAW,UAAU,MAAM,WAAW,UAAU,MAAM;AACjJ;AAEA,SAAS,UAAU,OAAe,MAAM,KAAK,QAAQ,IAAY;AAC/D,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,KAAK,CAAC;AAC7C,QAAM,SAAS,KAAK,MAAO,OAAO,MAAO,KAAK;AAC9C,SAAO,GAAG,SAAI,OAAO,MAAM,CAAC,GAAG,SAAI,OAAO,QAAQ,MAAM,CAAC;AAC3D;AAEA,SAAS,yBACP,MACA,QACA,QACA,SACQ;AACR,QAAM,YAAY,SACd,SAAS,QAAQ,GAAG,KAAK,IAAI,GAAG,KAAK,KAAK,EAAE,IAC5C,SAAS,QAAQ,GAAG,KAAK,IAAI,GAAG,KAAK,GAAG,EAAE;AAC9C,QAAM,aAAuB,CAAC;AAC9B,aAAW,QAAQ,QAAQ;AACzB,UAAM,MAAM,aAAa,KAAK,EAAE;AAChC,QAAI,CAAC,KAAK;AACR,YAAM,UAAU,eAAe,KAAK,IAAI;AACxC,iBAAW;AAAA,QACT,YAAY,SACR,SAAS,QAAQ,QAAQ,CAAC,GAAG,aAAa,OAAO,CAAC,IAClD;AAAA,MACN;AACA;AAAA,IACF;AACA,UAAM,YAAY,IAAI,OAAO,KAAK,IAAI;AACtC,YAAQ,IAAI,iBAAiB;AAAA,MAC3B,KAAK,OAAO;AACV,cAAM,UACJ,OAAO,KAAK,SAAS,YACrB,KAAK,SAAS,QACd,WAAW,KAAK,OACX,KAAK,KAA4B,QAClC,eAAe,KAAK,IAAI;AAC9B,YAAI,OAAO,YAAY,YAAY,OAAO,SAAS,OAAO,GAAG;AAC3D,qBAAW;AAAA,YACT,GAAG,SAAS,WAAW,aAAa,OAAO,CAAC,CAAC,IAAI,SAAS,UAAU,OAAO,GAAG,KAAK,GAAG,CAAC;AAAA,UACzF;AAAA,QACF,OAAO;AACL,qBAAW,KAAK,SAAS;AAAA,QAC3B;AACA;AAAA,MACF;AAAA,MACA,KAAK;AACH,mBAAW,KAAK,SAAS;AACzB;AAAA,MACF,KAAK;AACH,mBAAW;AAAA,UACT;AAAA,YACE;AAAA,YACA,KAAK,WAAW,OACZ,GAAG,KAAK,IAAI,GAAG,KAAK,KAAK,KACzB,KAAK,WAAW,QACd,GAAG,KAAK,IAAI,GAAG,KAAK,GAAG,KACvB,KAAK;AAAA,UACb;AAAA,QACF;AACA;AAAA,IACJ;AAAA,EACF;AACA,QAAM,WAAW,WAAW,SAAS,IAAI,WAAW,KAAK,GAAG,IAAI;AAChE,MAAI,OAAO,MAAM,IAAI,KAAK,SAAS,IAAI,QAAQ;AAC/C,MAAI,WAAW,QAAQ,SAAS,GAAG;AACjC,UAAM,cAAwB,CAAC;AAC/B,eAAW,EAAE,IAAI,KAAK,KAAK,SAAS;AAClC,YAAM,MAAM,cAAc,EAAE;AAC5B,UAAI,KAAK;AACP,cAAM,YAAY,IAAI,OAAO,IAAI;AACjC,oBAAY;AAAA,UACV,IAAI,OAAO,IAAI,IAAI,IAAI,KAAK,SAAS,MAAM,IAAI,SAAS;AAAA,QAC1D;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY,SAAS,GAAG;AAC1B,cAAQ,IAAI,YAAY,KAAK,GAAG,CAAC;AAAA,IACnC;AAAA,EACF;AACA,SAAO;AACT;AAEA,eAAsB,qBACpB,QACA,aACA,kBACe;AACf,QAAM,UAAU,MAAM,OAAO,qBAAqB,WAAW;AAC7D,MAAI,CAAC,SAAS;AACZ,UAAM,QAAQ,MAAM,OAAO,gBAAgB;AAC3C,UAAM,YAAY,MAAM,IAAI,CAAC,SAAS,KAAK,QAAQ,QAAQ,CAAC,EAAE,KAAK;AACnE,UAAM,IAAI;AAAA,MACR,UAAU,SAAS,IACf,YAAY,WAAW,oCAAoC,UAAU,KAAK,IAAI,CAAC,KAC/E,YAAY,WAAW;AAAA,IAC7B;AAAA,EACF;AAEA,QAAM,aACJ,MAAM,OAAO,+BAA+B,gBAAgB;AAC9D,MAAI,WAAW,WAAW,GAAG;AAC3B,UAAM,QAAQ,MAAM,OAAO,kBAAkB;AAC7C,UAAM,YAAY,MACf,IAAI,CAAC,SAAS,KAAK,UAAU,QAAQ,CAAC,EACtC,OAAO,CAAC,SAAyB,OAAO,SAAS,QAAQ,EACzD,KAAK;AACR,UAAM,IAAI;AAAA,MACR,UAAU,SAAS,IACf,yBAAyB,gBAAgB,4BAA4B,UAAU,KAAK,IAAI,CAAC,KACzF,yBAAyB,gBAAgB;AAAA,IAC/C;AAAA,EACF;AAEA,QAAM,oBAAoB,IAAI;AAAA,IAC5B,WAAW,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,KAAK,UAAU,QAAQ,KAAK,KAAK,EAAE,CAAC;AAAA,EACzE;AACA,QAAM,aAAa,oBAAI,IAAgC;AACvD,QAAM,oBAA4C,CAAC;AACnD,MAAI,oBAAoB;AACxB,MAAI,oBAAoB;AACxB,MAAI,iBAAiB;AACrB,MAAI,aAAa;AACjB,MAAI,cAAc;AAClB,QAAM,gBAAgB,CAAC,UAAK,UAAK,UAAK,UAAK,UAAK,QAAG;AACnD,MAAI,eAAe;AAEnB,WAAS,YAAkB;AACzB,QAAI,CAAC,QAAQ,OAAO,OAAO;AACzB;AAAA,IACF;AACA,YAAQ,OAAO,MAAM,WAAW;AAAA,EAClC;AAEA,WAAS,cAAoB;AAC3B,QAAI,CAAC,QAAQ,OAAO,SAAS,aAAa;AACxC;AAAA,IACF;AACA,UAAM,QAAQ,cAAc,eAAe,cAAc,MAAM;AAC/D,oBAAgB;AAChB,YAAQ,OAAO;AAAA,MACb,KAAK,SAAS,OAAO,KAAK,IAAI,CAAC,wBAAwB;AAAA,QACrD,GAAG,cAAc,IAAI,UAAU;AAAA,QAC/B,KAAK;AAAA,MACP,CAAC,IAAI,SAAS,UAAU,KAAK,GAAG,CAAC;AAAA,IACnC;AAAA,EACF;AAEA,MAAI;AACJ,QAAM,OAAO,IAAI,QAAqB,CAACA,aAAY;AACjD,UAAM,cAAc,OAAO,mBAAmB,CAAC,UAAU;AACvD,UAAI,MAAM,SAAS,oBAAoB;AACrC,yBAAiB,MAAM;AACvB,cAAM,gBAAgB,MAAM,gBACzB,IAAI,CAAC,SAAS,yBAAyB,KAAK,MAAM,CAAC,EACnD,OAAO,CAAC,SAAyB,SAAS,MAAS;AACtD,cAAM,eACJ,cAAc,SAAS,IACnB,cAAc,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC,IACnD,cAAc,SACd;AAEN,kBAAU;AACV,gBAAQ;AAAA,UACN,GAAG,SAAS,IAAI,MAAM,kBAAkB,IAAI,MAAM,cAAc,KAAK,KAAK,IAAI,CAAC,IAAI,MAAM,YAAY,IAAI,SAAS,IAAI,MAAM,UAAU,OAAO,KAAK,GAAG,CAAC;AAAA,QACxJ;AACA,mBAAW,QAAQ,MAAM,iBAAiB;AACxC,gBAAM,OACJ,kBAAkB,IAAI,KAAK,WAAW,KAAK,KAAK;AAClD,kBAAQ;AAAA,YACN;AAAA,cACE;AAAA,cACA,KAAK;AAAA,cACL,KAAK;AAAA,cACL,KAAK;AAAA,YACP;AAAA,UACF;AAEA,gBAAM,UAAU,yBAAyB,KAAK,MAAM;AACpD,cAAI,YAAY,QAAW;AACzB,kBAAM,UAAU,WAAW,IAAI,KAAK,WAAW,KAAK;AAAA,cAClD,OAAO;AAAA,cACP,OAAO;AAAA,cACP,QAAQ;AAAA,cACR,QAAQ;AAAA,YACV;AACA,uBAAW,IAAI,KAAK,aAAa;AAAA,cAC/B,OAAO,QAAQ,QAAQ;AAAA,cACvB,OAAO,QAAQ,QAAQ;AAAA,cACvB,QAAQ,QAAQ,UAAU,KAAK,SAAS,IAAI;AAAA,cAC5C,QAAQ,QAAQ,UAAU,KAAK,SAAS,IAAI;AAAA,YAC9C,CAAC;AACD,iCAAqB;AACrB,iCAAqB;AAAA,UACvB;AAAA,QACF;AAEA,0BAAkB,KAAK;AAAA,UACrB,MAAM,MAAM;AAAA,UACZ;AAAA,UACA,YAAY,MAAM;AAAA,UAClB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAED,oBAAY;AAAA,MACd;AACA,UAAI,MAAM,SAAS,kBAAkB,MAAM,SAAS,aAAa;AAC/D,sBAAc;AACd,kBAAU;AACV,oBAAY;AACZ,QAAAA,SAAQ,KAAK;AAAA,MACf;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,QAAM,WAAW,MAAM,OAAO,eAAe;AAAA,IAC3C,WAAW,QAAQ;AAAA,IACnB,cAAc,WAAW,IAAI,CAAC,SAAS,KAAK,EAAE;AAAA,EAChD,CAAC;AACD,eAAa,SAAS;AAEtB,UAAQ,IAAI,SAAS,4BAA4B,GAAG,KAAK,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;AAC5E,UAAQ,IAAI,QAAQ,SAAS,SAAS,OAAO,KAAK,IAAI,CAAC,EAAE;AACzD,UAAQ,IAAI,YAAY,SAAS,SAAS,aAAa,KAAK,IAAI,CAAC,EAAE;AACnE,UAAQ;AAAA,IACN,eAAe,WACZ,IAAI,CAAC,SAAS,KAAK,UAAU,QAAQ,KAAK,KAAK,EAAE,EACjD,KAAK,IAAI,CAAC;AAAA,EACf;AACA,UAAQ;AAAA,IACN,qBAAqB,SAAS,OAAO,SAAS,cAAc,GAAG,KAAK,IAAI,CAAC;AAAA,EAC3E;AACA,UAAQ,IAAI,EAAE;AACd,cAAY;AACZ,iBAAe,YAAY,aAAa,GAAG;AAE3C,QAAM,aAAa,MAAM;AACzB,MAAI,cAAc;AAChB,kBAAc,YAAY;AAAA,EAC5B;AAEA,MAAI,WAAW,SAAS,aAAa;AACnC,UAAM,IAAI,MAAM,eAAe,WAAW,YAAY,EAAE;AAAA,EAC1D;AAEA,UAAQ,IAAI,EAAE;AACd,UAAQ,IAAI,SAAS,uBAAuB,GAAG,KAAK,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;AACvE,UAAQ;AAAA,IACN,aAAa;AAAA,MACX,GAAG,WAAW,eAAe,IAAI,WAAW,cAAc;AAAA,MAC1D,KAAK;AAAA,IACP,CAAC;AAAA,EACH;AACA,UAAQ;AAAA,IACN,aAAa;AAAA,MACX,GAAG,WAAW,eAAe,IAAI,WAAW,cAAc;AAAA,MAC1D,WAAW,kBAAkB,IAAI,KAAK,MAAM,KAAK;AAAA,IACnD,CAAC;AAAA,EACH;AACA,MAAI,oBAAoB,GAAG;AACzB,UAAM,iBAAiB,oBAAoB;AAC3C,YAAQ;AAAA,MACN,wBAAwB;AAAA,QACtB,eAAe,QAAQ,CAAC;AAAA,QACxB,aAAa,cAAc;AAAA,MAC7B,CAAC,IAAI,SAAS,UAAU,cAAc,GAAG,KAAK,GAAG,CAAC;AAAA,IACpD;AAAA,EACF;AACA,UAAQ,IAAI,SAAS,yBAAyB,KAAK,OAAO,CAAC;AAC3D,aAAW,CAAC,aAAa,aAAa,KAAK,kBAAkB,QAAQ,GAAG;AACtE,YAAQ;AAAA,MACN,wBAAwB,eAAe,WAAW,IAAI,WAAW,CAAC;AAAA,IACpE;AAAA,EACF;AACA,MAAI,kBAAkB,SAAS,GAAG;AAChC,YAAQ,IAAI,SAAS,uBAAuB,KAAK,OAAO,CAAC;AACzD,eAAW,WAAW,mBAAmB;AACvC,YAAM,SAAS,QAAQ,SACnB,SAAS,QAAQ,KAAK,KAAK,IAC3B,SAAS,QAAQ,KAAK,GAAG;AAC7B,UAAI,QAAQ,iBAAiB,QAAW;AACtC,gBAAQ;AAAA,UACN,KAAK,MAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC,cAAc,SAAS,IAAI,QAAQ,UAAU,OAAO,KAAK,GAAG,CAAC;AAAA,QACrG;AACA;AAAA,MACF;AACA,cAAQ;AAAA,QACN,KAAK,MAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC,UAAU;AAAA,UAC9C,QAAQ,aAAa,QAAQ,CAAC;AAAA,UAC9B,aAAa,QAAQ,YAAY;AAAA,QACnC,CAAC,IAAI,SAAS,UAAU,QAAQ,cAAc,KAAK,EAAE,GAAG,KAAK,GAAG,CAAC,IAAI,SAAS,IAAI,QAAQ,UAAU,OAAO,KAAK,GAAG,CAAC;AAAA,MACtH;AAAA,IACF;AAAA,EACF;AACA,UAAQ,IAAI,eAAe,SAAS,WAAW,cAAc,KAAK,GAAG,CAAC,EAAE;AAC1E;;;ACzVA,SAAS,kBAAkB,UAAyB;AAClD,QAAM,UAAU,aAAa,IAAI,QAAQ,MAAM,QAAQ;AACvD,UAAQ,kBAAkB,CAAC;AAC3B,UAAQ,KAAK,QAAQ;AACvB;AAEA,eAAe,OAAsB;AACnC,QAAM,OAAO,mBAAmB,QAAQ,KAAK,MAAM,CAAC,CAAC;AAErD,MAAI,KAAK,MAAM;AACb,sBAAkB,CAAC;AAAA,EACrB;AACA,MAAI,KAAK,YAAY,SAAS,GAAG;AAC/B,YAAQ,MAAM,sBAAsB,KAAK,YAAY,KAAK,IAAI,CAAC,EAAE;AACjE,sBAAkB,CAAC;AAAA,EACrB;AACA,MAAI,CAAC,KAAK,SAAS;AACjB,sBAAkB,CAAC;AAAA,EACrB;AACA,MAAI,CAAC,KAAK,aAAa;AACrB,YAAQ,MAAM,oDAAoD;AAClE,sBAAkB,CAAC;AAAA,EACrB;AAEA,QAAM,SAAS,aAAa;AAC5B,MAAI;AACF,QAAI,KAAK,YAAY,OAAO;AAC1B,UAAI,CAAC,KAAK,kBAAkB;AAC1B,gBAAQ,MAAM,0DAA0D;AACxE,0BAAkB,CAAC;AAAA,MACrB;AACA,YAAM,qBAAqB,QAAQ,KAAK,aAAa,KAAK,gBAAgB;AAC1E;AAAA,IACF;AAEA,UAAM,2BAA2B,QAAQ,KAAK,WAAW;AAAA,EAC3D,UAAE;AACA,UAAM,OAAO,SAAS;AAAA,EACxB;AACF;AAEA,KAAK,KAAK,EAAE,MAAM,CAAC,UAAmB;AACpC,UAAQ,MAAM,iBAAiB,QAAQ,MAAM,UAAU,gBAAgB;AACvE,UAAQ,KAAK,CAAC;AAChB,CAAC","sourcesContent":["import { randomUUID } from 'node:crypto';\n\nimport { Effect, Fiber, PubSub, Queue } from 'effect';\n\nimport type { RunnerConfig, RunnerConfigOverrides } from './config';\nimport { withRunnerConfig } from './config';\nimport { loadRunnerConfigFile } from './config-loader';\nimport {\n  collectDatasetsFromFiles,\n  collectEvaluatorsFromFiles,\n  collectTestCasesFromFiles,\n} from './discovery';\nimport { createArtifactPath, executeRunTask, type RunTask } from './execution';\nimport type {\n  CollectedDataset,\n  CollectedEvaluator,\n  CollectedTestCase,\n  RunDatasetRequest,\n  RunSnapshot,\n  RunnerEvent,\n  SearchTestCasesQuery,\n} from './events';\nimport { createPersistenceWorker } from './persistence';\nimport { searchCollectedTestCases } from './search';\n\ninterface SubscribeOptions {\n  runId?: string;\n}\n\nfunction parseRegexLiteral(\n  pattern: string,\n): { source: string; flags: string } | undefined {\n  if (!pattern.startsWith('/')) {\n    return undefined;\n  }\n  const lastSlash = pattern.lastIndexOf('/');\n  if (lastSlash <= 0) {\n    return undefined;\n  }\n  return {\n    source: pattern.slice(1, lastSlash),\n    flags: pattern.slice(lastSlash + 1),\n  };\n}\n\nfunction createNameMatcher(pattern: string): (value: string) => boolean {\n  const normalizedPattern = pattern.trim();\n  const regexLiteral = parseRegexLiteral(normalizedPattern);\n  if (regexLiteral) {\n    const regex = new RegExp(regexLiteral.source, regexLiteral.flags);\n    return (value: string) => regex.test(value);\n  }\n\n  if (normalizedPattern.includes('*')) {\n    const escaped = normalizedPattern\n      .replace(/[.+^${}()|[\\]\\\\]/g, '\\\\$&')\n      .replace(/\\*/g, '.*');\n    const regex = new RegExp(`^${escaped}$`, 'i');\n    return (value: string) => regex.test(value);\n  }\n\n  return (value: string) => value.toLowerCase() === normalizedPattern.toLowerCase();\n}\n\nexport interface RunnerApi {\n  collectDatasets(): Promise<ReadonlyArray<CollectedDataset>>;\n  collectEvaluators(): Promise<ReadonlyArray<CollectedEvaluator>>;\n  resolveDatasetByName(name: string): Promise<CollectedDataset | undefined>;\n  resolveEvaluatorsByNamePattern(\n    pattern: string,\n  ): Promise<ReadonlyArray<CollectedEvaluator>>;\n  searchTestCases(\n    query?: SearchTestCasesQuery,\n  ): Promise<ReadonlyArray<CollectedTestCase>>;\n  collectDatasetTestCases(datasetId: string): Promise<ReadonlyArray<CollectedTestCase>>;\n  runDatasetWith(request: RunDatasetRequest): Promise<RunSnapshot>;\n  subscribeRunEvents(\n    listener: (event: RunnerEvent) => void,\n    options?: SubscribeOptions,\n  ): () => void;\n  getRunSnapshot(runId: string): RunSnapshot | undefined;\n  getAllRunSnapshots(): ReadonlyArray<RunSnapshot>;\n  shutdown(): Promise<void>;\n}\n\nfunction mergeRunnerOverrides(\n  base?: RunnerConfigOverrides,\n  next?: RunnerConfigOverrides,\n): RunnerConfigOverrides | undefined {\n  if (!base) {\n    return next;\n  }\n  if (!next) {\n    return base;\n  }\n  const discovery = base.discovery || next.discovery\n    ? {\n        ...(base.discovery ?? {}),\n        ...(next.discovery ?? {}),\n      }\n    : undefined;\n  return {\n    ...base,\n    ...next,\n    discovery,\n  };\n}\n\nexport function createRunner(overrides?: RunnerConfigOverrides): RunnerApi {\n  const fileOverrides = loadRunnerConfigFile();\n  const merged = mergeRunnerOverrides(fileOverrides, overrides);\n  return new EffectRunner(withRunnerConfig(merged));\n}\n\nclass EffectRunner implements RunnerApi {\n  private readonly config: RunnerConfig;\n\n  private readonly eventBus = Effect.runSync(PubSub.unbounded<RunnerEvent>());\n\n  private readonly runQueue = Effect.runSync(Queue.unbounded<RunTask>());\n\n  private readonly persistenceQueue = Effect.runSync(\n    Queue.unbounded<{\n      runId: string;\n      artifactPath: string;\n      payload: unknown;\n    }>(),\n  );\n\n  private readonly snapshots = new Map<string, RunSnapshot>();\n  private readonly listeners = new Set<{\n    runId?: string;\n    listener: (event: RunnerEvent) => void;\n  }>();\n\n  private readonly datasetsById = new Map<string, CollectedDataset>();\n\n  private readonly evaluatorsById = new Map<string, CollectedEvaluator>();\n\n  private readonly schedulerFiber = Effect.runFork(\n    this.createSchedulerEffect(),\n  );\n\n  private readonly persistenceFiber = Effect.runFork(\n    createPersistenceWorker(this.persistenceQueue),\n  );\n\n  constructor(config: RunnerConfig) {\n    this.config = config;\n  }\n\n  async collectDatasets(): Promise<ReadonlyArray<CollectedDataset>> {\n    const datasets = await collectDatasetsFromFiles(this.config.discovery);\n    this.datasetsById.clear();\n    for (const dataset of datasets) {\n      this.datasetsById.set(dataset.id, dataset);\n    }\n    return datasets;\n  }\n\n  async collectEvaluators(): Promise<ReadonlyArray<CollectedEvaluator>> {\n    const evaluators = await collectEvaluatorsFromFiles(this.config.discovery);\n    this.evaluatorsById.clear();\n    for (const evaluator of evaluators) {\n      this.evaluatorsById.set(evaluator.id, evaluator);\n    }\n    return evaluators;\n  }\n\n  async resolveDatasetByName(name: string): Promise<CollectedDataset | undefined> {\n    if (this.datasetsById.size === 0) {\n      await this.collectDatasets();\n    }\n    const normalized = name.trim().toLowerCase();\n    return Array.from(this.datasetsById.values()).find(\n      (item) => item.dataset.getName().toLowerCase() === normalized,\n    );\n  }\n\n  async resolveEvaluatorsByNamePattern(\n    pattern: string,\n  ): Promise<ReadonlyArray<CollectedEvaluator>> {\n    if (this.evaluatorsById.size === 0) {\n      await this.collectEvaluators();\n    }\n    const matcher = createNameMatcher(pattern);\n    return Array.from(this.evaluatorsById.values()).filter((item) =>\n      matcher(item.evaluator.getName() ?? ''),\n    );\n  }\n\n  async searchTestCases(\n    query?: SearchTestCasesQuery,\n  ): Promise<ReadonlyArray<CollectedTestCase>> {\n    const testCases = await collectTestCasesFromFiles(this.config.discovery);\n    return searchCollectedTestCases(testCases, query);\n  }\n\n  async collectDatasetTestCases(\n    datasetId: string,\n  ): Promise<ReadonlyArray<CollectedTestCase>> {\n    if (this.datasetsById.size === 0) {\n      await this.collectDatasets();\n    }\n    const dataset = this.datasetsById.get(datasetId);\n    if (!dataset) {\n      throw new Error(`Unknown dataset: ${datasetId}`);\n    }\n    const allTestCases = await collectTestCasesFromFiles(this.config.discovery);\n    return allTestCases.filter((testCase) =>\n      dataset.dataset.matchesTestCase(testCase.testCase, testCase.filePath),\n    );\n  }\n\n  async runDatasetWith(request: RunDatasetRequest): Promise<RunSnapshot> {\n    if (this.datasetsById.size === 0) {\n      await this.collectDatasets();\n    }\n    if (this.evaluatorsById.size === 0) {\n      await this.collectEvaluators();\n    }\n\n    const dataset = this.datasetsById.get(request.datasetId);\n    if (!dataset) {\n      throw new Error(`Unknown dataset: ${request.datasetId}`);\n    }\n\n    const selectedEvaluators = request.evaluatorIds\n      .map((id) => this.evaluatorsById.get(id))\n      .filter((value): value is CollectedEvaluator => Boolean(value))\n      .map((value) => ({ id: value.id, evaluator: value.evaluator }));\n\n    if (selectedEvaluators.length === 0) {\n      throw new Error('No evaluators selected for run');\n    }\n\n    const selectedTestCases = await this.collectDatasetTestCases(request.datasetId);\n\n    const runId = `run-${randomUUID()}`;\n    const artifactPath = createArtifactPath(\n      this.config.artifactDirectory,\n      request.datasetId,\n      runId,\n    );\n    const snapshot: RunSnapshot = {\n      runId,\n      datasetId: request.datasetId,\n      datasetName: dataset.dataset.getName(),\n      evaluatorIds: selectedEvaluators.map((item) => item.id),\n      queuedAt: Date.now(),\n      totalTestCases: selectedTestCases.length,\n      completedTestCases: 0,\n      passedTestCases: 0,\n      failedTestCases: 0,\n      status: 'queued',\n      artifactPath,\n    };\n\n    this.snapshots.set(runId, snapshot);\n    const queuedEvent: RunnerEvent = {\n      type: 'RunQueued',\n      runId,\n      datasetId: request.datasetId,\n      datasetName: dataset.dataset.getName(),\n      evaluatorIds: selectedEvaluators.map((item) => item.id),\n      totalTestCases: selectedTestCases.length,\n      artifactPath,\n    };\n    await Effect.runPromise(this.publishEvent(queuedEvent));\n    await Effect.runPromise(\n      Queue.offer(this.persistenceQueue, {\n        runId,\n        artifactPath,\n        payload: queuedEvent,\n      }),\n    );\n\n    await Effect.runPromise(\n      Queue.offer(this.runQueue, {\n        runId,\n        datasetId: request.datasetId,\n        dataset: dataset.dataset,\n        evaluators: selectedEvaluators,\n        testCases: selectedTestCases,\n        snapshot,\n      }),\n    );\n\n    return snapshot;\n  }\n\n  subscribeRunEvents(\n    listener: (event: RunnerEvent) => void,\n    options?: SubscribeOptions,\n  ): () => void {\n    const entry = { runId: options?.runId, listener };\n    this.listeners.add(entry);\n    return () => {\n      this.listeners.delete(entry);\n    };\n  }\n\n  getRunSnapshot(runId: string): RunSnapshot | undefined {\n    return this.snapshots.get(runId);\n  }\n\n  getAllRunSnapshots(): ReadonlyArray<RunSnapshot> {\n    return Array.from(this.snapshots.values()).sort(\n      (a, b) => b.queuedAt - a.queuedAt,\n    );\n  }\n\n  async shutdown(): Promise<void> {\n    await Effect.runPromise(Fiber.interrupt(this.schedulerFiber));\n    await Effect.runPromise(Fiber.interrupt(this.persistenceFiber));\n    await Effect.runPromise(Queue.shutdown(this.runQueue));\n    await Effect.runPromise(Queue.shutdown(this.persistenceQueue));\n    await Effect.runPromise(PubSub.shutdown(this.eventBus));\n  }\n\n  private createSchedulerEffect() {\n    const self = this;\n    return Effect.forever(\n      Effect.gen(function* () {\n        const task = yield* Queue.take(self.runQueue);\n        yield* Effect.fork(\n          executeRunTask(\n            task,\n            self.publishEvent.bind(self),\n            self.persistenceQueue,\n            self.updateSnapshot.bind(self),\n          ),\n        );\n      }),\n    );\n  }\n\n  private updateSnapshot(\n    runId: string,\n    updater: (snapshot: RunSnapshot) => RunSnapshot,\n  ): void {\n    const existing = this.snapshots.get(runId);\n    if (!existing) {\n      return;\n    }\n    this.snapshots.set(runId, updater(existing));\n  }\n\n  private publishEvent(event: RunnerEvent): Effect.Effect<void, never, never> {\n    return Effect.sync(() => {\n      for (const entry of this.listeners) {\n        if (entry.runId && entry.runId !== event.runId) {\n          continue;\n        }\n        entry.listener(event);\n      }\n    }).pipe(\n      Effect.flatMap(() => PubSub.publish(this.eventBus, event)),\n      Effect.asVoid,\n    );\n  }\n}\n","export interface RunnerDiscoveryConfig {\n  rootDir: string;\n  datasetSuffixes: ReadonlyArray<string>;\n  evaluatorSuffixes: ReadonlyArray<string>;\n  testCaseSuffixes: ReadonlyArray<string>;\n  excludeDirectories: ReadonlyArray<string>;\n}\n\nexport interface RunnerConfig {\n  discovery: RunnerDiscoveryConfig;\n  artifactDirectory: string;\n}\n\nexport type RunnerConfigOverrides = Omit<Partial<RunnerConfig>, 'discovery'> & {\n  discovery?: Partial<RunnerDiscoveryConfig>;\n};\n\nexport interface M4trixEvalConfigDiscovery {\n  rootDir?: string;\n  datasetFilePatterns?: ReadonlyArray<string>;\n  evaluatorFilePatterns?: ReadonlyArray<string>;\n  testCaseFilePatterns?: ReadonlyArray<string>;\n  datasetSuffixes?: ReadonlyArray<string>;\n  evaluatorSuffixes?: ReadonlyArray<string>;\n  testCaseSuffixes?: ReadonlyArray<string>;\n  excludeDirectories?: ReadonlyArray<string>;\n}\n\nexport interface M4trixEvalConfig {\n  discovery?: M4trixEvalConfigDiscovery;\n  artifactDirectory?: string;\n}\n\nexport type ConfigType = M4trixEvalConfig;\n\nexport type M4trixEvalConfigFactory<TConfig extends ConfigType = ConfigType> = () => TConfig;\n\nexport function defineConfig<TConfig extends ConfigType>(\n  factory: M4trixEvalConfigFactory<TConfig>,\n): M4trixEvalConfigFactory<TConfig> {\n  return factory;\n}\n\nexport const defaultRunnerConfig: RunnerConfig = {\n  discovery: {\n    rootDir: process.cwd(),\n    datasetSuffixes: ['.dataset.ts', '.dataset.tsx', '.dataset.js', '.dataset.mjs'],\n    evaluatorSuffixes: [\n      '.evaluator.ts',\n      '.evaluator.tsx',\n      '.evaluator.js',\n      '.evaluator.mjs',\n    ],\n    testCaseSuffixes: [\n      '.test-case.ts',\n      '.test-case.tsx',\n      '.test-case.js',\n      '.test-case.mjs',\n    ],\n    excludeDirectories: ['node_modules', 'dist', '.next', '.git', '.pnpm-store'],\n  },\n  artifactDirectory: '.eval-results',\n};\n\nexport function toRunnerConfigOverrides(\n  config?: ConfigType,\n): RunnerConfigOverrides | undefined {\n  if (!config) {\n    return undefined;\n  }\n\n  const rawDiscovery = config.discovery;\n  const discovery: Partial<RunnerDiscoveryConfig> = {};\n  if (rawDiscovery?.rootDir !== undefined) {\n    discovery.rootDir = rawDiscovery.rootDir;\n  }\n  if (rawDiscovery?.datasetFilePatterns !== undefined) {\n    discovery.datasetSuffixes = rawDiscovery.datasetFilePatterns;\n  } else if (rawDiscovery?.datasetSuffixes !== undefined) {\n    discovery.datasetSuffixes = rawDiscovery.datasetSuffixes;\n  }\n  if (rawDiscovery?.evaluatorFilePatterns !== undefined) {\n    discovery.evaluatorSuffixes = rawDiscovery.evaluatorFilePatterns;\n  } else if (rawDiscovery?.evaluatorSuffixes !== undefined) {\n    discovery.evaluatorSuffixes = rawDiscovery.evaluatorSuffixes;\n  }\n  if (rawDiscovery?.testCaseFilePatterns !== undefined) {\n    discovery.testCaseSuffixes = rawDiscovery.testCaseFilePatterns;\n  } else if (rawDiscovery?.testCaseSuffixes !== undefined) {\n    discovery.testCaseSuffixes = rawDiscovery.testCaseSuffixes;\n  }\n  if (rawDiscovery?.excludeDirectories !== undefined) {\n    discovery.excludeDirectories = rawDiscovery.excludeDirectories;\n  }\n\n  const overrides: RunnerConfigOverrides = {};\n  if (config.artifactDirectory !== undefined) {\n    overrides.artifactDirectory = config.artifactDirectory;\n  }\n  if (Object.keys(discovery).length > 0) {\n    overrides.discovery = discovery;\n  }\n  return overrides;\n}\n\nexport function withRunnerConfig(overrides?: RunnerConfigOverrides): RunnerConfig {\n  if (!overrides) {\n    return defaultRunnerConfig;\n  }\n  const discovery = overrides.discovery\n    ? {\n        ...defaultRunnerConfig.discovery,\n        ...overrides.discovery,\n      }\n    : defaultRunnerConfig.discovery;\n\n  return {\n    ...defaultRunnerConfig,\n    ...overrides,\n    discovery,\n  };\n}\n","import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\n\nimport * as jitiModule from 'jiti';\n\nimport type { RunnerConfigOverrides } from './config';\nimport {\n  toRunnerConfigOverrides,\n  type ConfigType,\n  type M4trixEvalConfigFactory,\n} from './config';\n\nconst CONFIG_FILE_NAME = 'm4trix-eval.config.ts';\n\ntype JitiLoader = {\n  (id: string): unknown;\n  import?: (id: string) => Promise<unknown> | unknown;\n};\n\nlet cachedLoader: JitiLoader | undefined;\n\nfunction getJitiLoader(): JitiLoader {\n  if (cachedLoader) {\n    return cachedLoader;\n  }\n  const createJiti =\n    (jitiModule as { createJiti?: unknown; default?: unknown }).createJiti ??\n    (jitiModule as { default?: unknown }).default;\n  if (typeof createJiti !== 'function') {\n    throw new Error(\n      'Failed to initialize jiti for m4trix eval config loading.',\n    );\n  }\n  cachedLoader = (\n    createJiti as (id: string, options?: Record<string, unknown>) => JitiLoader\n  )(import.meta.url, {\n    interopDefault: true,\n    moduleCache: true,\n  });\n  return cachedLoader;\n}\n\nfunction resolveConfigModuleExport(loadedModule: unknown): unknown {\n  if (\n    loadedModule &&\n    typeof loadedModule === 'object' &&\n    'default' in loadedModule\n  ) {\n    return (loadedModule as { default: unknown }).default;\n  }\n  return loadedModule;\n}\n\nfunction resolveConfigValue(value: unknown): ConfigType | undefined {\n  if (value === undefined || value === null) {\n    return undefined;\n  }\n  if (typeof value === 'function') {\n    return (value as M4trixEvalConfigFactory<ConfigType>)();\n  }\n  if (typeof value !== 'object') {\n    throw new Error(\n      'Invalid m4trix eval config export. Expected an object or defineConfig(() => config).',\n    );\n  }\n  return value as ConfigType;\n}\n\nexport function loadRunnerConfigFile(\n  cwd = process.cwd(),\n): RunnerConfigOverrides | undefined {\n  const configPath = resolve(cwd, CONFIG_FILE_NAME);\n  if (!existsSync(configPath)) {\n    return undefined;\n  }\n  const loader = getJitiLoader();\n  const loaded = loader(configPath);\n  const exportedValue = resolveConfigModuleExport(loaded);\n  const config = resolveConfigValue(exportedValue);\n  return toRunnerConfigOverrides(config);\n}\n","import { Dirent } from 'node:fs';\nimport { readdir } from 'node:fs/promises';\nimport { resolve, relative } from 'node:path';\nimport { pathToFileURL } from 'node:url';\n\nimport type { Dataset } from '../evals/dataset';\nimport type { Evaluator } from '../evals/evaluator';\nimport type { TestCase } from '../evals/test-case';\nimport type {\n  CollectedDataset,\n  CollectedEvaluator,\n  CollectedTestCase,\n} from './events';\nimport type { RunnerDiscoveryConfig } from './config';\n\ntype JitiModuleLoader = {\n  (id: string): unknown;\n  import?: (id: string) => Promise<unknown> | unknown;\n};\n\nlet jitiLoader: JitiModuleLoader | undefined;\n\nfunction toId(prefix: string, filePath: string, name?: string): string {\n  const stable = name && name.trim().length > 0 ? name : filePath;\n  return `${prefix}:${stable}`\n    .toLowerCase()\n    .replace(/[^a-z0-9]+/g, '-')\n    .replace(/^-+|-+$/g, '');\n}\n\nfunction hasMethod(value: unknown, methodName: string): boolean {\n  return (\n    typeof value === 'object' &&\n    value !== null &&\n    methodName in value &&\n    typeof (value as Record<string, unknown>)[methodName] === 'function'\n  );\n}\n\nfunction isDatasetLike(value: unknown): value is Dataset {\n  return hasMethod(value, 'getName') && hasMethod(value, 'matchesTestCase');\n}\n\nfunction isEvaluatorLike(\n  value: unknown,\n): value is Evaluator<unknown, unknown, unknown, unknown> {\n  return (\n    hasMethod(value, 'getName') &&\n    hasMethod(value, 'resolveContext') &&\n    hasMethod(value, 'getEvaluateFn')\n  );\n}\n\nfunction isTestCaseLike(value: unknown): value is TestCase<unknown> {\n  return (\n    hasMethod(value, 'getName') &&\n    hasMethod(value, 'getTags') &&\n    hasMethod(value, 'getInput')\n  );\n}\n\nasync function walkDirectory(\n  rootDir: string,\n  excludeDirectories: ReadonlyArray<string>,\n): Promise<string[]> {\n  const out: string[] = [];\n\n  async function walk(currentDir: string): Promise<void> {\n    let entries: Dirent[];\n    try {\n      entries = await readdir(currentDir, { withFileTypes: true });\n    } catch {\n      return;\n    }\n\n    await Promise.all(\n      entries.map(async (entry) => {\n        const absolute = resolve(currentDir, entry.name);\n        if (entry.isDirectory()) {\n          if (excludeDirectories.includes(entry.name)) {\n            return;\n          }\n          await walk(absolute);\n          return;\n        }\n\n        if (entry.isFile()) {\n          out.push(absolute);\n        }\n      }),\n    );\n  }\n\n  await walk(rootDir);\n  return out;\n}\n\nfunction hasOneSuffix(\n  filePath: string,\n  suffixes: ReadonlyArray<string>,\n): boolean {\n  return suffixes.some((suffix) => filePath.endsWith(suffix));\n}\n\nasync function loadModuleExports(filePath: string): Promise<unknown[]> {\n  if (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) {\n    if (!jitiLoader) {\n      const jitiModule = (await import('jiti')) as {\n        createJiti?: (filename: string, opts?: Record<string, unknown>) => JitiModuleLoader;\n        default?: (filename: string, opts?: Record<string, unknown>) => JitiModuleLoader;\n      };\n      const createJiti = jitiModule.createJiti ?? jitiModule.default;\n      if (!createJiti) {\n        throw new Error('Failed to initialize jiti TypeScript loader');\n      }\n      jitiLoader = createJiti(import.meta.url, {\n        interopDefault: true,\n        moduleCache: true,\n      }) as JitiModuleLoader;\n    }\n    const loaded = jitiLoader.import\n      ? await jitiLoader.import(filePath)\n      : await Promise.resolve(jitiLoader(filePath));\n    return Object.values(loaded as Record<string, unknown>);\n  }\n\n  const moduleUrl = pathToFileURL(filePath).href;\n  const loaded = (await import(moduleUrl)) as Record<string, unknown>;\n  return Object.values(loaded);\n}\n\nexport async function collectDatasetsFromFiles(\n  config: RunnerDiscoveryConfig,\n): Promise<ReadonlyArray<CollectedDataset>> {\n  const files = await walkDirectory(config.rootDir, config.excludeDirectories);\n  const matched = files.filter((filePath) =>\n    hasOneSuffix(filePath, config.datasetSuffixes),\n  );\n\n  const found = await Promise.all(\n    matched.map(async (absolutePath) => {\n      const exports = await loadModuleExports(absolutePath);\n      const datasets = exports.filter(isDatasetLike);\n      const relPath = relative(config.rootDir, absolutePath);\n      return datasets.map((dataset) => ({\n        id: toId('dataset', relPath, dataset.getName()),\n        filePath: relPath,\n        dataset,\n      }));\n    }),\n  );\n\n  return found.flat();\n}\n\nexport async function collectEvaluatorsFromFiles(\n  config: RunnerDiscoveryConfig,\n): Promise<ReadonlyArray<CollectedEvaluator>> {\n  const files = await walkDirectory(config.rootDir, config.excludeDirectories);\n  const matched = files.filter((filePath) =>\n    hasOneSuffix(filePath, config.evaluatorSuffixes),\n  );\n\n  const found = await Promise.all(\n    matched.map(async (absolutePath) => {\n      const exports = await loadModuleExports(absolutePath);\n      const evaluators = exports.filter(isEvaluatorLike);\n      const relPath = relative(config.rootDir, absolutePath);\n      return evaluators.map((evaluator) => ({\n        id: toId('evaluator', relPath, evaluator.getName()),\n        filePath: relPath,\n        evaluator,\n      }));\n    }),\n  );\n\n  return found.flat();\n}\n\nexport async function collectTestCasesFromFiles(\n  config: RunnerDiscoveryConfig,\n): Promise<ReadonlyArray<CollectedTestCase>> {\n  const files = await walkDirectory(config.rootDir, config.excludeDirectories);\n  const matched = files.filter((filePath) =>\n    hasOneSuffix(filePath, config.testCaseSuffixes),\n  );\n\n  const found = await Promise.all(\n    matched.map(async (absolutePath) => {\n      const exports = await loadModuleExports(absolutePath);\n      const testCases = exports.filter(isTestCaseLike);\n      const relPath = relative(config.rootDir, absolutePath);\n      return testCases.map((testCase) => ({\n        id: toId('test-case', relPath, testCase.getName()),\n        filePath: relPath,\n        testCase,\n      }));\n    }),\n  );\n\n  return found.flat();\n}\n","import { join } from 'node:path';\n\nimport { Effect, Queue } from 'effect';\n\nimport type { Dataset } from '../evals/dataset';\nimport type { Evaluator } from '../evals/evaluator';\nimport type { MetricItem } from '../evals/metric';\nimport type { ScoreItem } from '../evals/score';\nimport type { CollectedTestCase, RunSnapshot, RunnerEvent } from './events';\nimport type { PersistenceMessage } from './persistence';\nimport { toNumericScoreFromScores } from './score-utils';\n\nfunction computeEvaluatorPassed(\n  evaluator: Evaluator<unknown, unknown, unknown, unknown>,\n  result: unknown,\n  scores: ReadonlyArray<ScoreItem>,\n): boolean {\n  const scoresWithPassed = scores.filter((s) => 'passed' in s && s.passed !== undefined);\n  if (scoresWithPassed.length > 0) {\n    return scoresWithPassed.every((s) => s.passed === true);\n  }\n  const passCriterion = evaluator.getPassCriterion();\n  if (passCriterion) {\n    return passCriterion(result);\n  }\n  const passThreshold = evaluator.getPassThreshold();\n  if (passThreshold !== undefined) {\n    const numeric = toNumericScoreFromScores(scores);\n    return numeric !== undefined && numeric >= passThreshold;\n  }\n  return true;\n}\n\nfunction normalizeResult(\n  result: unknown,\n): {\n  scores: ReadonlyArray<ScoreItem>;\n  metrics?: ReadonlyArray<MetricItem>;\n} {\n  if (typeof result !== 'object' || result === null) {\n    return { scores: [] };\n  }\n  const obj = result as Record<string, unknown>;\n  const scores = Array.isArray(obj.scores)\n    ? (obj.scores as ReadonlyArray<ScoreItem>)\n    : [];\n  const metrics = Array.isArray(obj.metrics)\n    ? (obj.metrics as ReadonlyArray<MetricItem>)\n    : undefined;\n  return { scores, metrics };\n}\n\nfunction readOutput(testCase: CollectedTestCase['testCase']): unknown {\n  const candidate = testCase as unknown as { getOutput?: () => unknown };\n  if (typeof candidate.getOutput !== 'function') {\n    return undefined;\n  }\n  return candidate.getOutput();\n}\n\nexport interface RunTask {\n  runId: string;\n  datasetId: string;\n  dataset: Dataset;\n  evaluators: ReadonlyArray<{\n    id: string;\n    evaluator: Evaluator<unknown, unknown, unknown, unknown>;\n  }>;\n  testCases: ReadonlyArray<CollectedTestCase>;\n  snapshot: RunSnapshot;\n}\n\nfunction nowIsoForFile(): string {\n  return new Date().toISOString().replace(/[:.]/g, '-');\n}\n\nexport function createArtifactPath(\n  artifactDirectory: string,\n  datasetId: string,\n  runId: string,\n): string {\n  return join(\n    artifactDirectory,\n    `${datasetId}_${runId}_${nowIsoForFile()}.jsonl`,\n  );\n}\n\nexport const executeRunTask = (\n  task: RunTask,\n  publishEvent: (event: RunnerEvent) => Effect.Effect<void, never, never>,\n  persistenceQueue: Queue.Queue<PersistenceMessage>,\n  updateSnapshot: (\n    runId: string,\n    updater: (snapshot: RunSnapshot) => RunSnapshot,\n  ) => void,\n): Effect.Effect<void, never, never> =>\n  Effect.gen(function* () {\n    const startedAt = Date.now();\n    updateSnapshot(task.runId, (snapshot) => ({\n      ...snapshot,\n      status: 'running',\n      startedAt,\n    }));\n    yield* publishEvent({\n      type: 'RunStarted',\n      runId: task.runId,\n      startedAt,\n    });\n\n    let completedTestCases = 0;\n    let passedTestCases = 0;\n    let failedTestCases = 0;\n\n    for (const testCaseItem of task.testCases) {\n      const started = Date.now();\n      const evaluatorScores: Array<{\n        evaluatorId: string;\n        scores: ReadonlyArray<ScoreItem>;\n        passed: boolean;\n        metrics?: ReadonlyArray<MetricItem>;\n      }> = [];\n      let testCaseError: string | undefined;\n      const output = readOutput(testCaseItem.testCase);\n\n      for (const { id: evaluatorId, evaluator } of task.evaluators) {\n        const evaluateFn = evaluator.getEvaluateFn();\n        if (!evaluateFn) {\n          continue;\n        }\n\n        try {\n          const ctx = yield* Effect.promise(() =>\n            Promise.resolve(evaluator.resolveContext()),\n          );\n          const result = yield* Effect.promise(() =>\n            Promise.resolve(\n              evaluateFn({\n                input: testCaseItem.testCase.getInput(),\n                ctx,\n                output,\n              }),\n            ),\n          );\n          const { scores, metrics } = normalizeResult(result);\n          const passed = computeEvaluatorPassed(evaluator, result, scores);\n          evaluatorScores.push({ evaluatorId, scores, passed, metrics });\n        } catch (error) {\n          testCaseError =\n            error instanceof Error\n              ? error.message\n              : 'Evaluator execution failed';\n          evaluatorScores.push({\n            evaluatorId,\n            scores: [],\n            passed: false,\n          });\n        }\n      }\n\n      const testCasePassed = evaluatorScores.every((s) => s.passed);\n      completedTestCases += 1;\n      if (testCasePassed) {\n        passedTestCases += 1;\n      } else {\n        failedTestCases += 1;\n      }\n\n      const progressEvent: RunnerEvent = {\n        type: 'TestCaseProgress',\n        runId: task.runId,\n        testCaseId: testCaseItem.id,\n        testCaseName: testCaseItem.testCase.getName(),\n        completedTestCases,\n        totalTestCases: task.testCases.length,\n        passed: testCasePassed,\n        durationMs: Date.now() - started,\n        evaluatorScores,\n        output,\n        errorMessage: testCaseError,\n      };\n\n      updateSnapshot(task.runId, (snapshot) => ({\n        ...snapshot,\n        completedTestCases,\n        passedTestCases,\n        failedTestCases,\n      }));\n\n      yield* publishEvent(progressEvent);\n      yield* Queue.offer(persistenceQueue, {\n        runId: task.runId,\n        artifactPath: task.snapshot.artifactPath,\n        payload: progressEvent,\n      });\n    }\n\n    const finishedAt = Date.now();\n    const completedEvent: RunnerEvent = {\n      type: 'RunCompleted',\n      runId: task.runId,\n      finishedAt,\n      passedTestCases,\n      failedTestCases,\n      totalTestCases: task.testCases.length,\n      artifactPath: task.snapshot.artifactPath,\n    };\n\n    updateSnapshot(task.runId, (snapshot) => ({\n      ...snapshot,\n      status: 'completed',\n      completedTestCases,\n      passedTestCases,\n      failedTestCases,\n      finishedAt,\n    }));\n\n    yield* publishEvent(completedEvent);\n    yield* Queue.offer(persistenceQueue, {\n      runId: task.runId,\n      artifactPath: task.snapshot.artifactPath,\n      payload: completedEvent,\n    });\n    yield* publishEvent({\n      type: 'ArtifactFlushed',\n      runId: task.runId,\n      artifactPath: task.snapshot.artifactPath,\n    });\n  });\n","const registry = new Map<string, MetricDef<unknown>>();\n\nexport interface MetricItem<TData = unknown> {\n  readonly id: string;\n  readonly data: TData;\n}\n\nexport interface MetricDef<TData = unknown> {\n  readonly id: string;\n  readonly name?: string;\n  format(data: TData): string;\n  make(data: TData): MetricItem<TData>;\n}\n\nexport const Metric = {\n  of<TData>(config: {\n    id: string;\n    name?: string;\n    format: (data: TData) => string;\n  }): MetricDef<TData> {\n    const def: MetricDef<TData> = {\n      id: config.id,\n      name: config.name,\n      format: config.format,\n      make: (data: TData) => ({ id: config.id, data }),\n    };\n    registry.set(config.id, def as MetricDef<unknown>);\n    return def;\n  },\n};\n\nexport function getMetricById(id: string): MetricDef<unknown> | undefined {\n  return registry.get(id);\n}\n","const registry = new Map<string, ScoreDef<unknown>>();\n\nexport type ScoreDisplayStrategy = 'bar' | 'number' | 'passFail';\n\nexport interface ScoreItem<TData = unknown> {\n  readonly id: string;\n  readonly data: TData;\n  readonly passed?: boolean;\n}\n\nexport interface ScoreDef<TData = unknown> {\n  readonly id: string;\n  readonly name?: string;\n  readonly displayStrategy: ScoreDisplayStrategy;\n  format(data: TData): string;\n  make(\n    data: TData,\n    options?: { definePassed?: (data: TData) => boolean },\n  ): ScoreItem<TData>;\n}\n\nexport const Score = {\n  of<TData>(config: {\n    id: string;\n    name?: string;\n    displayStrategy: ScoreDisplayStrategy;\n    format: (data: TData) => string;\n  }): ScoreDef<TData> {\n    const def: ScoreDef<TData> = {\n      id: config.id,\n      name: config.name,\n      displayStrategy: config.displayStrategy,\n      format: config.format,\n      make: (data: TData, options?: { definePassed?: (data: TData) => boolean }) => {\n        const passed =\n          options?.definePassed !== undefined\n            ? options.definePassed(data)\n            : undefined;\n        return {\n          id: config.id,\n          data,\n          ...(passed !== undefined && { passed }),\n        };\n      },\n    };\n    registry.set(config.id, def as ScoreDef<unknown>);\n    return def;\n  },\n};\n\nexport function getScoreById(id: string): ScoreDef<unknown> | undefined {\n  return registry.get(id);\n}\n","import { Metric } from '../metric';\n\nexport interface TokenCountData {\n  input?: number;\n  output?: number;\n  inputCached?: number;\n  outputCached?: number;\n}\n\nexport const tokenCountMetric = Metric.of<TokenCountData>({\n  id: 'token-count',\n  name: 'Tokens',\n  format: (data) => {\n    const input = data.input ?? 0;\n    const output = data.output ?? 0;\n    const inputCached = data.inputCached ?? 0;\n    const outputCached = data.outputCached ?? 0;\n    const cached = inputCached + outputCached;\n    return `in:${input} out:${output} cached:${cached}`;\n  },\n});\n\nexport interface LatencyData {\n  ms: number;\n}\n\nexport const latencyMetric = Metric.of<LatencyData>({\n  id: 'latency',\n  name: 'Latency',\n  format: (data) => `${data.ms}ms`,\n});\n","import { Score } from '../score';\n\nexport interface PercentScoreData {\n  value: number;\n}\n\nexport const percentScore = Score.of<PercentScoreData>({\n  id: 'percent',\n  name: 'Score',\n  displayStrategy: 'bar',\n  format: (data) => data.value.toFixed(2),\n});\n\nexport interface BinaryScoreData {\n  passed: boolean;\n}\n\nexport const binaryScore = Score.of<BinaryScoreData>({\n  id: 'binary',\n  name: 'Result',\n  displayStrategy: 'passFail',\n  format: (data) => (data.passed ? 'PASSED' : 'NOT PASSED'),\n});\n","import type { ScoreItem } from '../evals/score';\nimport { getScoreById } from '../evals';\n\nexport function toNumericScoreFromScores(\n  scores: ReadonlyArray<ScoreItem>,\n): number | undefined {\n  for (const item of scores) {\n    const def = getScoreById(item.id);\n    if (def && def.displayStrategy === 'bar' && typeof item.data === 'object' && item.data !== null && 'value' in item.data) {\n      const value = (item.data as { value: unknown }).value;\n      if (typeof value === 'number' && Number.isFinite(value)) {\n        return value;\n      }\n    }\n    const numeric = toNumericScore(item.data);\n    if (numeric !== undefined) {\n      return numeric;\n    }\n  }\n  return undefined;\n}\n\nexport function toNumericScore(value: unknown): number | undefined {\n  if (typeof value === 'number' && Number.isFinite(value)) {\n    return value;\n  }\n  if (typeof value !== 'object' || value === null) {\n    return undefined;\n  }\n  const obj = value as Record<string, unknown>;\n  if (\n    'score' in obj &&\n    typeof obj.score === 'number' &&\n    Number.isFinite(obj.score)\n  ) {\n    return obj.score;\n  }\n  const numberValues = Object.values(value).filter(\n    (entry): entry is number =>\n      typeof entry === 'number' && Number.isFinite(entry),\n  );\n  if (numberValues.length === 0) {\n    return undefined;\n  }\n  return (\n    numberValues.reduce((sum, entry) => sum + entry, 0) / numberValues.length\n  );\n}\n","import { appendFile, mkdir } from 'node:fs/promises';\nimport { dirname } from 'node:path';\n\nimport { Effect, Queue } from 'effect';\n\nexport interface PersistenceMessage {\n  runId: string;\n  artifactPath: string;\n  payload: unknown;\n}\n\nasync function appendJsonLine(\n  artifactPath: string,\n  payload: unknown,\n): Promise<void> {\n  await mkdir(dirname(artifactPath), { recursive: true });\n  await appendFile(artifactPath, `${JSON.stringify(payload)}\\n`, 'utf8');\n}\n\nexport const createPersistenceWorker = (\n  queue: Queue.Queue<PersistenceMessage>,\n): Effect.Effect<never, never, never> =>\n  Effect.forever(\n    Effect.gen(function* () {\n      const message = yield* Queue.take(queue);\n      yield* Effect.promise(() =>\n        appendJsonLine(message.artifactPath, {\n          runId: message.runId,\n          ts: Date.now(),\n          ...message.payload,\n        }),\n      );\n    }),\n  );\n","import type { CollectedTestCase, SearchTestCasesQuery } from './events';\n\nfunction matchesAny(\n  value: string,\n  matchers: ReadonlyArray<string | RegExp> | undefined,\n): boolean {\n  if (!matchers || matchers.length === 0) {\n    return true;\n  }\n  return matchers.some((matcher) =>\n    typeof matcher === 'string' ? matcher === value : matcher.test(value),\n  );\n}\n\nfunction matchesPath(\n  value: string,\n  matchers: ReadonlyArray<string | RegExp> | undefined,\n): boolean {\n  if (!matchers || matchers.length === 0) {\n    return true;\n  }\n  return matchers.some((matcher) => {\n    if (typeof matcher === 'string') {\n      return value.includes(matcher);\n    }\n    return matcher.test(value);\n  });\n}\n\nexport function searchCollectedTestCases(\n  all: ReadonlyArray<CollectedTestCase>,\n  query?: SearchTestCasesQuery,\n): ReadonlyArray<CollectedTestCase> {\n  if (!query) {\n    return all;\n  }\n\n  return all.filter((item) => {\n    const tags = item.testCase.getTags();\n\n    if (\n      query.excludedTags &&\n      tags.some((tag) => matchesAny(tag, query.excludedTags))\n    ) {\n      return false;\n    }\n    if (\n      query.excludedPaths &&\n      matchesPath(item.filePath, query.excludedPaths)\n    ) {\n      return false;\n    }\n\n    const includedTagsMatch =\n      !query.includedTags ||\n      query.includedTags.length === 0 ||\n      tags.some((tag) => matchesAny(tag, query.includedTags));\n\n    const includedPathsMatch =\n      !query.includedPaths ||\n      query.includedPaths.length === 0 ||\n      matchesPath(item.filePath, query.includedPaths);\n\n    return includedTagsMatch && includedPathsMatch;\n  });\n}\n","export type SimpleCliCommand = 'run' | 'generate';\n\nexport interface SimpleCliArgs {\n  command?: SimpleCliCommand;\n  datasetName?: string;\n  evaluatorPattern?: string;\n  help: boolean;\n  unknownArgs: string[];\n}\n\nexport function parseSimpleCliArgs(argv: string[]): SimpleCliArgs {\n  const args: SimpleCliArgs = {\n    help: false,\n    unknownArgs: [],\n  };\n  let index = 0;\n  if (argv[0] === 'run' || argv[0] === 'generate') {\n    args.command = argv[0];\n    index = 1;\n  }\n\n  for (; index < argv.length; index += 1) {\n    const token = argv[index];\n    if (token === '--help' || token === '-h') {\n      args.help = true;\n      continue;\n    }\n    if ((token === '--dataset' || token === '--datasetName') && argv[index + 1]) {\n      args.datasetName = argv[index + 1];\n      index += 1;\n      continue;\n    }\n    if ((token === '--evaluator' || token === '--name') && argv[index + 1]) {\n      args.evaluatorPattern = argv[index + 1];\n      index += 1;\n      continue;\n    }\n    args.unknownArgs.push(token);\n  }\n\n  return args;\n}\n\nexport function getSimpleCliUsage(): string {\n  return [\n    'Usage:',\n    '  eval-agents-simple run --dataset <datasetName> --evaluator <name-or-pattern>',\n    '  eval-agents-simple generate --dataset <datasetName>',\n    '',\n    'Pattern examples for --evaluator:',\n    '  score-evaluator       exact name (case-insensitive)',\n    '  \"*score*\"             wildcard pattern',\n    '  \"/score/i\"            regex literal',\n  ].join('\\n');\n}\n","import { writeFile } from 'node:fs/promises';\nimport { join, parse, resolve } from 'node:path';\n\nimport type { RunnerApi } from '../runner';\n\ninterface GeneratedDatasetCase {\n  name: string;\n  input: unknown;\n  output?: unknown;\n}\n\nfunction readOutput(testCase: { getOutput?: () => unknown }): unknown {\n  if (typeof testCase.getOutput !== 'function') {\n    return undefined;\n  }\n  return testCase.getOutput();\n}\n\nfunction createOutputPath(datasetFilePath: string): string {\n  const parsed = parse(datasetFilePath);\n  return join(parsed.dir, `${parsed.name}.cases.json`);\n}\n\nexport async function generateDatasetJsonCommand(\n  runner: RunnerApi,\n  datasetName: string,\n): Promise<void> {\n  const dataset = await runner.resolveDatasetByName(datasetName);\n  if (!dataset) {\n    throw new Error(`Dataset \"${datasetName}\" not found.`);\n  }\n\n  const testCases = await runner.collectDatasetTestCases(dataset.id);\n  const payload: GeneratedDatasetCase[] = testCases.map((item) => ({\n    name: item.testCase.getName(),\n    input: item.testCase.getInput(),\n    output: readOutput(item.testCase),\n  }));\n\n  const absoluteDatasetPath = resolve(process.cwd(), dataset.filePath);\n  const outputPath = createOutputPath(absoluteDatasetPath);\n\n  await writeFile(outputPath, `${JSON.stringify(payload, null, 2)}\\n`, 'utf8');\n\n  console.log(`Generated ${payload.length} test cases for dataset \"${dataset.dataset.getName()}\".`);\n  console.log(`Wrote ${outputPath}`);\n}\n","import { getMetricById, getScoreById } from '../evals';\nimport type { ScoreItem } from '../evals/score';\nimport type { RunnerApi, RunnerEvent } from '../runner';\nimport {\n  toNumericScore,\n  toNumericScoreFromScores,\n} from '../runner/score-utils';\n\ninterface EvaluatorAggregate {\n  total: number;\n  count: number;\n  passed: number;\n  failed: number;\n}\n\ninterface TestCaseScoreSummary {\n  name: string;\n  averageScore?: number;\n  durationMs: number;\n  passed: boolean;\n}\n\nconst ansi = {\n  reset: '\\x1b[0m',\n  bold: '\\x1b[1m',\n  dim: '\\x1b[2m',\n  green: '\\x1b[32m',\n  yellow: '\\x1b[33m',\n  red: '\\x1b[31m',\n  cyan: '\\x1b[36m',\n  magenta: '\\x1b[35m',\n} as const;\n\nfunction colorize(text: string, color: string): string {\n  return `${color}${text}${ansi.reset}`;\n}\n\nfunction scoreToColor(score: number): string {\n  if (score >= 80) {\n    return ansi.green;\n  }\n  if (score >= 50) {\n    return ansi.yellow;\n  }\n  return ansi.red;\n}\n\nfunction getEvaluatorSummaryLine(\n  evaluatorName: string,\n  aggregate: EvaluatorAggregate | undefined,\n): string {\n  if (!aggregate || aggregate.count === 0) {\n    return `- ${evaluatorName.padEnd(28)} no numeric scores`;\n  }\n  const mean = aggregate.total / aggregate.count;\n  return `- ${evaluatorName.padEnd(28)} avg=${colorize(mean.toFixed(2), scoreToColor(mean))} passed=${aggregate.passed} failed=${aggregate.failed}`;\n}\n\nfunction createBar(value: number, max = 100, width = 20): string {\n  const safe = Math.max(0, Math.min(max, value));\n  const filled = Math.round((safe / max) * width);\n  return `${'█'.repeat(filled)}${'░'.repeat(width - filled)}`;\n}\n\nfunction formatEvaluatorScoreLine(\n  name: string,\n  scores: ReadonlyArray<ScoreItem>,\n  passed: boolean,\n  metrics?: ReadonlyArray<{ id: string; data: unknown }>,\n): string {\n  const passLabel = passed\n    ? colorize('PASS', `${ansi.bold}${ansi.green}`)\n    : colorize('FAIL', `${ansi.bold}${ansi.red}`);\n  const scoreParts: string[] = [];\n  for (const item of scores) {\n    const def = getScoreById(item.id);\n    if (!def) {\n      const numeric = toNumericScore(item.data);\n      scoreParts.push(\n        numeric !== undefined\n          ? colorize(numeric.toFixed(2), scoreToColor(numeric))\n          : 'n/a',\n      );\n      continue;\n    }\n    const formatted = def.format(item.data);\n    switch (def.displayStrategy) {\n      case 'bar': {\n        const numeric =\n          typeof item.data === 'object' &&\n          item.data !== null &&\n          'value' in item.data\n            ? (item.data as { value: unknown }).value\n            : toNumericScore(item.data);\n        if (typeof numeric === 'number' && Number.isFinite(numeric)) {\n          scoreParts.push(\n            `${colorize(formatted, scoreToColor(numeric))} ${colorize(createBar(numeric), ansi.dim)}`,\n          );\n        } else {\n          scoreParts.push(formatted);\n        }\n        break;\n      }\n      case 'number':\n        scoreParts.push(formatted);\n        break;\n      case 'passFail':\n        scoreParts.push(\n          colorize(\n            formatted,\n            item.passed === true\n              ? `${ansi.bold}${ansi.green}`\n              : item.passed === false\n                ? `${ansi.bold}${ansi.red}`\n                : ansi.dim,\n          ),\n        );\n        break;\n    }\n  }\n  const scoreStr = scoreParts.length > 0 ? scoreParts.join(' ') : 'n/a';\n  let line = `   ${name}: ${passLabel} ${scoreStr}`;\n  if (metrics && metrics.length > 0) {\n    const metricParts: string[] = [];\n    for (const { id, data } of metrics) {\n      const def = getMetricById(id);\n      if (def) {\n        const formatted = def.format(data);\n        metricParts.push(\n          def.name ? `[${def.name}: ${formatted}]` : `[${formatted}]`,\n        );\n      }\n    }\n    if (metricParts.length > 0) {\n      line += ` ${metricParts.join(' ')}`;\n    }\n  }\n  return line;\n}\n\nexport async function runSimpleEvalCommand(\n  runner: RunnerApi,\n  datasetName: string,\n  evaluatorPattern: string,\n): Promise<void> {\n  const dataset = await runner.resolveDatasetByName(datasetName);\n  if (!dataset) {\n    const known = await runner.collectDatasets();\n    const available = known.map((item) => item.dataset.getName()).sort();\n    throw new Error(\n      available.length > 0\n        ? `Dataset \"${datasetName}\" not found. Available datasets: ${available.join(', ')}`\n        : `Dataset \"${datasetName}\" not found and no datasets were discovered.`,\n    );\n  }\n\n  const evaluators =\n    await runner.resolveEvaluatorsByNamePattern(evaluatorPattern);\n  if (evaluators.length === 0) {\n    const known = await runner.collectEvaluators();\n    const available = known\n      .map((item) => item.evaluator.getName())\n      .filter((name): name is string => typeof name === 'string')\n      .sort();\n    throw new Error(\n      available.length > 0\n        ? `No evaluator matched \"${evaluatorPattern}\". Available evaluators: ${available.join(', ')}`\n        : `No evaluator matched \"${evaluatorPattern}\" and no evaluators were discovered.`,\n    );\n  }\n\n  const evaluatorNameById = new Map(\n    evaluators.map((item) => [item.id, item.evaluator.getName() ?? item.id]),\n  );\n  const aggregates = new Map<string, EvaluatorAggregate>();\n  const testCaseSummaries: TestCaseScoreSummary[] = [];\n  let overallScoreTotal = 0;\n  let overallScoreCount = 0;\n  let completedCount = 0;\n  let totalCount = 0;\n  let runFinished = false;\n  const spinnerFrames = ['⠋', '⠙', '⠸', '⠴', '⠦', '⠇'];\n  let spinnerIndex = 0;\n\n  function clearLine(): void {\n    if (!process.stdout.isTTY) {\n      return;\n    }\n    process.stdout.write('\\r\\x1b[2K');\n  }\n\n  function drawSpinner(): void {\n    if (!process.stdout.isTTY || runFinished) {\n      return;\n    }\n    const frame = spinnerFrames[spinnerIndex % spinnerFrames.length];\n    spinnerIndex += 1;\n    process.stdout.write(\n      `\\r${colorize(frame, ansi.cyan)} Running evaluations ${colorize(\n        `${completedCount}/${totalCount}`,\n        ansi.bold,\n      )} ${colorize('(live)', ansi.dim)}`,\n    );\n  }\n\n  let spinnerTimer: NodeJS.Timeout | undefined;\n  const done = new Promise<RunnerEvent>((resolve) => {\n    const unsubscribe = runner.subscribeRunEvents((event) => {\n      if (event.type === 'TestCaseProgress') {\n        completedCount = event.completedTestCases;\n        const numericScores = event.evaluatorScores\n          .map((item) => toNumericScoreFromScores(item.scores))\n          .filter((item): item is number => item !== undefined);\n        const averageScore =\n          numericScores.length > 0\n            ? numericScores.reduce((sum, value) => sum + value, 0) /\n              numericScores.length\n            : undefined;\n\n        clearLine();\n        console.log(\n          `${colorize(`[${event.completedTestCases}/${event.totalTestCases}]`, ansi.cyan)} ${event.testCaseName} ${colorize(`(${event.durationMs}ms)`, ansi.dim)}`,\n        );\n        for (const item of event.evaluatorScores) {\n          const name =\n            evaluatorNameById.get(item.evaluatorId) ?? item.evaluatorId;\n          console.log(\n            formatEvaluatorScoreLine(\n              name,\n              item.scores,\n              item.passed,\n              item.metrics,\n            ),\n          );\n\n          const numeric = toNumericScoreFromScores(item.scores);\n          if (numeric !== undefined) {\n            const current = aggregates.get(item.evaluatorId) ?? {\n              total: 0,\n              count: 0,\n              passed: 0,\n              failed: 0,\n            };\n            aggregates.set(item.evaluatorId, {\n              total: current.total + numeric,\n              count: current.count + 1,\n              passed: current.passed + (item.passed ? 1 : 0),\n              failed: current.failed + (item.passed ? 0 : 1),\n            });\n            overallScoreTotal += numeric;\n            overallScoreCount += 1;\n          }\n        }\n\n        testCaseSummaries.push({\n          name: event.testCaseName,\n          averageScore,\n          durationMs: event.durationMs,\n          passed: event.passed,\n        });\n\n        drawSpinner();\n      }\n      if (event.type === 'RunCompleted' || event.type === 'RunFailed') {\n        runFinished = true;\n        clearLine();\n        unsubscribe();\n        resolve(event);\n      }\n    });\n  });\n\n  const snapshot = await runner.runDatasetWith({\n    datasetId: dataset.id,\n    evaluatorIds: evaluators.map((item) => item.id),\n  });\n  totalCount = snapshot.totalTestCases;\n\n  console.log(colorize('=== Eval Run Started ===', `${ansi.bold}${ansi.cyan}`));\n  console.log(`Run: ${colorize(snapshot.runId, ansi.cyan)}`);\n  console.log(`Dataset: ${colorize(snapshot.datasetName, ansi.bold)}`);\n  console.log(\n    `Evaluators: ${evaluators\n      .map((item) => item.evaluator.getName() ?? item.id)\n      .join(', ')}`,\n  );\n  console.log(\n    `Total test cases: ${colorize(String(snapshot.totalTestCases), ansi.bold)}`,\n  );\n  console.log('');\n  drawSpinner();\n  spinnerTimer = setInterval(drawSpinner, 100);\n\n  const finalEvent = await done;\n  if (spinnerTimer) {\n    clearInterval(spinnerTimer);\n  }\n\n  if (finalEvent.type === 'RunFailed') {\n    throw new Error(`Run failed: ${finalEvent.errorMessage}`);\n  }\n\n  console.log('');\n  console.log(colorize('=== Run Summary ===', `${ansi.bold}${ansi.cyan}`));\n  console.log(\n    `- passed: ${colorize(\n      `${finalEvent.passedTestCases}/${finalEvent.totalTestCases}`,\n      ansi.green,\n    )}`,\n  );\n  console.log(\n    `- failed: ${colorize(\n      `${finalEvent.failedTestCases}/${finalEvent.totalTestCases}`,\n      finalEvent.failedTestCases > 0 ? ansi.red : ansi.dim,\n    )}`,\n  );\n  if (overallScoreCount > 0) {\n    const overallAverage = overallScoreTotal / overallScoreCount;\n    console.log(\n      `- overall avg score: ${colorize(\n        overallAverage.toFixed(2),\n        scoreToColor(overallAverage),\n      )} ${colorize(createBar(overallAverage), ansi.dim)}`,\n    );\n  }\n  console.log(colorize('- evaluator averages:', ansi.magenta));\n  for (const [evaluatorId, evaluatorName] of evaluatorNameById.entries()) {\n    console.log(\n      getEvaluatorSummaryLine(evaluatorName, aggregates.get(evaluatorId)),\n    );\n  }\n  if (testCaseSummaries.length > 0) {\n    console.log(colorize('- test case scores:', ansi.magenta));\n    for (const summary of testCaseSummaries) {\n      const status = summary.passed\n        ? colorize('PASS', ansi.green)\n        : colorize('FAIL', ansi.red);\n      if (summary.averageScore === undefined) {\n        console.log(\n          `  ${status} ${summary.name.padEnd(24)} score=n/a ${colorize(`(${summary.durationMs}ms)`, ansi.dim)}`,\n        );\n        continue;\n      }\n      console.log(\n        `  ${status} ${summary.name.padEnd(24)} score=${colorize(\n          summary.averageScore.toFixed(2),\n          scoreToColor(summary.averageScore),\n        )} ${colorize(createBar(summary.averageScore, 100, 14), ansi.dim)} ${colorize(`(${summary.durationMs}ms)`, ansi.dim)}`,\n      );\n    }\n  }\n  console.log(`- artifact: ${colorize(finalEvent.artifactPath, ansi.dim)}`);\n}\n","#!/usr/bin/env node\n\nimport { createRunner } from '../runner';\nimport { getSimpleCliUsage, parseSimpleCliArgs } from './args';\nimport { generateDatasetJsonCommand } from './generate';\nimport { runSimpleEvalCommand } from './run';\n\nfunction printUsageAndExit(exitCode: number): never {\n  const printer = exitCode === 0 ? console.log : console.error;\n  printer(getSimpleCliUsage());\n  process.exit(exitCode);\n}\n\nasync function main(): Promise<void> {\n  const args = parseSimpleCliArgs(process.argv.slice(2));\n\n  if (args.help) {\n    printUsageAndExit(0);\n  }\n  if (args.unknownArgs.length > 0) {\n    console.error(`Unknown arguments: ${args.unknownArgs.join(', ')}`);\n    printUsageAndExit(1);\n  }\n  if (!args.command) {\n    printUsageAndExit(1);\n  }\n  if (!args.datasetName) {\n    console.error('Missing required --dataset <datasetName> argument.');\n    printUsageAndExit(1);\n  }\n\n  const runner = createRunner();\n  try {\n    if (args.command === 'run') {\n      if (!args.evaluatorPattern) {\n        console.error('Missing required --evaluator <name-or-pattern> argument.');\n        printUsageAndExit(1);\n      }\n      await runSimpleEvalCommand(runner, args.datasetName, args.evaluatorPattern);\n      return;\n    }\n\n    await generateDatasetJsonCommand(runner, args.datasetName);\n  } finally {\n    await runner.shutdown();\n  }\n}\n\nvoid main().catch((error: unknown) => {\n  console.error(error instanceof Error ? error.message : 'Command failed');\n  process.exit(1);\n});\n"]}

package/dist/cli-simple.js

@@ -2,7 +2,7 @@
 import { randomUUID } from 'crypto';
 import { Effect, PubSub, Queue, Fiber } from 'effect';
 import { existsSync } from 'fs';
-import { resolve, parse, join, relative, dirname } from 'path';
+import { resolve, relative, join, parse, dirname } from 'path';
 import * as jitiModule from 'jiti';
 import { writeFile, mkdir, appendFile, readdir } from 'fs/promises';
 import { pathToFileURL } from 'url';
@@ -393,12 +393,12 @@ function normalizeResult(result) {
   const metrics = Array.isArray(obj.metrics) ? obj.metrics : void 0;
   return { scores, metrics };
 }
-function readOutputDefinition(testCase) {
+function readOutput(testCase) {
   const candidate = testCase;
-  if (typeof candidate.getOutputDefinition !== "function") {
+  if (typeof candidate.getOutput !== "function") {
     return void 0;
   }
-  return candidate.getOutputDefinition();
+  return candidate.getOutput();
 }
 function nowIsoForFile() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -428,7 +428,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     const started = Date.now();
     const evaluatorScores = [];
     let testCaseError;
-    const outputDefinition = readOutputDefinition(testCaseItem.testCase);
+    const output = readOutput(testCaseItem.testCase);
     for (const { id: evaluatorId, evaluator } of task.evaluators) {
       const evaluateFn = evaluator.getEvaluateFn();
       if (!evaluateFn) {
@@ -443,7 +443,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
             evaluateFn({
               input: testCaseItem.testCase.getInput(),
               ctx,
-              output: outputDefinition
+              output
             })
           )
         );
@@ -476,7 +476,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
       passed: testCasePassed,
       durationMs: Date.now() - started,
       evaluatorScores,
-      outputDefinition,
+      output,
       errorMessage: testCaseError
     };
     updateSnapshot(task.runId, (snapshot) => ({
@@ -610,9 +610,18 @@ function mergeRunnerOverrides(base, next) {
   if (!base) {
     return next;
   }
-  {
+  if (!next) {
     return base;
   }
+  const discovery = base.discovery || next.discovery ? {
+    ...base.discovery ?? {},
+    ...next.discovery ?? {}
+  } : void 0;
+  return {
+    ...base,
+    ...next,
+    discovery
+  };
 }
 function createRunner(overrides) {
   const fileOverrides = loadRunnerConfigFile();
@@ -857,11 +866,11 @@ function getSimpleCliUsage() {
     '  "/score/i"            regex literal'
   ].join("\n");
 }
-function readOutputDefinition2(testCase) {
-  if (typeof testCase.getOutputDefinition !== "function") {
+function readOutput2(testCase) {
+  if (typeof testCase.getOutput !== "function") {
     return void 0;
   }
-  return testCase.getOutputDefinition();
+  return testCase.getOutput();
 }
 function createOutputPath(datasetFilePath) {
   const parsed = parse(datasetFilePath);
@@ -876,7 +885,7 @@ async function generateDatasetJsonCommand(runner, datasetName) {
   const payload = testCases.map((item) => ({
     name: item.testCase.getName(),
     input: item.testCase.getInput(),
-    outputDefinition: readOutputDefinition2(item.testCase)
+    output: readOutput2(item.testCase)
   }));
   const absoluteDatasetPath = resolve(process.cwd(), dataset.filePath);
   const outputPath = createOutputPath(absoluteDatasetPath);