oh-my-knowledge 0.24.0 → 0.25.0

package/dist/src/eval-workflows/batch-evaluation-workflow.js

@@ -52,6 +52,9 @@ function buildExecutorRuntimesBySkill({ skillDir, skillResults, executorName, mo
     return executorRuntimes;
 }
 export function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries, skillResults, model, judgeModel, noJudge, executorName, judgeExecutorName, project, owner, tags, concurrency, timeoutMs, totalCostUSD, repeat, judgeModels, noCache, }) {
+    const effectiveBatchJudges = judgeModels && judgeModels.length > 0
+        ? judgeModels
+        : [{ executor: judgeExecutorName || executorName, model: judgeModel }];
     const request = buildEvaluationRequest({
         samplesPath: '',
         skillDir,
@@ -63,9 +66,7 @@ export function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries,
             locator: entry.skillPath,
         })),
         model,
-        judgeModel: noJudge ? null : judgeModel,
         executor: executorName,
-        judgeExecutor: judgeExecutorName || executorName,
         noJudge,
         concurrency,
         timeoutMs,
@@ -76,7 +77,7 @@ export function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries,
         owner,
         tags,
         repeat,
-        judgeModels,
+        judgeModels: effectiveBatchJudges,
         batch: true,
     });
     const createdAt = new Date().toISOString();
@@ -97,9 +98,13 @@ export function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries,
     const executorRuntime = commonRuntime(executorRuntimes)
         ?? Object.values(executorRuntimes)[0]
         ?? getExecutorRuntimeFingerprint(executorName, model, { skillDir });
-    const isJudgeEnsemble = Boolean(judgeModels && judgeModels.length >= 2);
     const totalCostReported = skillResults.every((skill) => skill.report.meta.totalCostReported !== false
         && Object.values(skill.report.summary || {}).every((variant) => variant.execCostReported !== false && variant.judgeCostReported !== false));
+    const judgeModelsMeta = effectiveBatchJudges.map((jc) => ({
+        executor: jc.executor,
+        model: jc.model,
+        ...(noJudge ? {} : { runtime: getExecutorRuntimeFingerprint(jc.executor, jc.model, { skillDir }) }),
+    }));
     const report = {
         kind: 'batch-evaluation',
         id: batchRunId,
@@ -107,7 +112,6 @@ export function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries,
         meta: {
             mode: 'skill',
             model,
-            judgeModel: noJudge || isJudgeEnsemble ? null : judgeModel,
             executor: executorName,
             skillDir,
             sampleCount,
@@ -120,11 +124,8 @@ export function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries,
             nodeVersion: process.version,
             executorRuntime,
             executorRuntimes,
-            judgeRuntime: noJudge || isJudgeEnsemble ? null : getExecutorRuntimeFingerprint(judgeExecutorName || executorName, judgeModel, { skillDir }),
-            ...(judgeModels && judgeModels.length >= 2 ? {
-                judgeModels: judgeModels.map((jc) => `${jc.executor}:${jc.model}`),
-                judgeRuntimes: Object.fromEntries(judgeModels.map((jc) => [`${jc.executor}:${jc.model}`, getExecutorRuntimeFingerprint(jc.executor, jc.model, { skillDir })])),
-            } : {}),
+            judgeModels: judgeModelsMeta,
+            ...(noJudge ? { noJudge: true } : {}),
             request,
             run,
             job,
@@ -134,7 +135,7 @@ export function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries,
     };
     return { report, job };
 }
-export async function executeBatchEvaluationRuns({ skillDir, skillEntries, model, judgeModel, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, concurrency = 1, timeoutMs, executorName, judgeExecutorName, jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipPreflight = false, mcpConfig, verbose = false, repeat, judgeRepeat, judgeModels, lengthDebias, noCache = false, strictBaseline, variantAllowedSkills, runSingleEvaluation, }) {
+export async function executeBatchEvaluationRuns({ skillDir, skillEntries, model, judgeModel, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, concurrency = 1, timeoutMs, executorName, judgeExecutorName, jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipConnectivity = false, lang = 'zh', mcpConfig, verbose = false, repeat, judgeRepeat, judgeModels, lengthDebias, noCache = false, strictBaseline, variantAllowedSkills, runSingleEvaluation, }) {
     const batchRunId = generateRunId(['batch']);
     const skillResults = [];
     let totalCostUSD = 0;
@@ -168,7 +169,8 @@ export async function executeBatchEvaluationRuns({ skillDir, skillEntries, model
             jobStore: null,
             persistJob: false,
             onProgress,
-            skipPreflight: skipPreflight || i > 0,
+            skipConnectivity: skipConnectivity || i > 0,
+            lang,
             mcpConfig,
             verbose,
             runId: childRunId,

package/dist/src/eval-workflows/batch-evaluation-workflow.js.map

@@ -1 +1 @@
-{"version":3,"file":"batch-evaluation-workflow.js","sourceRoot":"","sources":["../../../src/eval-workflows/batch-evaluation-workflow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACnI,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AACxI,OAAO,EAAE,6BAA6B,EAAE,MAAM,qCAAqC,CAAC;AACpF,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC9E,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAgD7D,SAAS,aAAa,CAAC,QAAoD;IACzE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACxB,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAClG,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK;SACT,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,mBAAmB,EAAE,GAAG,CAAC;SACpC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;WACrB,OAAO,CAAC;AACf,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAwB,EAAE,gBAAwB;IAC/E,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAK,EAAqB;QAC3D,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAK,EAAqB;KAC/E,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,YAAsC;IAC7D,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/E,IAAI;QACJ,SAAS;QACT,WAAW;QACX,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,WAAW;QACnB,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW;QACpC,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY;QACtC,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI;QACxD,OAAO,EAAE,qBAAqB,CAAC,MAAM,EAAE,IAAI,CAAC;QAC5C,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,4BAA4B,CAAC,EACpC,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,KAAK,GAMN;IACC,MAAM,gBAAgB,GAA+C,EAAE,CAAC;IACxE,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC;YAC1B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;mBAC7C,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe;mBACjC,6BAA6B,CAAC,YAAY,EAAE,KAAK,EAAE;oBACpD,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ;iBAChE,CAAC,CAAC;IACP,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,EACzC,UAAU,EACV,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,KAAK,EACL,IAAI,EACJ,WAAW,EACX,SAAS,EACT,YAAY,EACZ,MAAM,EACN,WAAW,EACX,OAAO,GAoBR;IACC,MAAM,OAAO,GAAG,sBAAsB,CAAC;QACrC,WAAW,EAAE,EAAE;QACf,QAAQ;QACR,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACtC,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,WAAW;YACnB,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,KAAK,CAAC,SAAS;SACzB,CAAC,CAAC;QACH,KAAK;QACL,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU;QACvC,QAAQ,EAAE,YAAY;QACtB,aAAa,EAAE,iBAAiB,IAAI,YAAY;QAChD,OAAO;QACP,WAAW;QACX,SAAS;QACT,OAAO;QACP,MAAM,EAAE,KAAK;QACb,KAAK,EAAE,KAAK;QACZ,OAAO;QACP,KAAK;QACL,IAAI;QACJ,MAAM;QACN,WAAW;QACX,KAAK,EAAE,IAAI;KACZ,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,mBAAmB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,GAAG,GAAG,qBAAqB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAG,kBAAkB,CAAC;QAC7B,KAAK,EAAE,OAAO,UAAU,EAAE;QAC1B,KAAK,EAAE,UAAU;QACjB,QAAQ,EAAE,UAAU;QACpB,OAAO;QACP,SAAS;QACT,SAAS;QACT,UAAU;KACX,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAChG,MAAM,gBAAgB,GAAG,4BAA4B,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IACvG,MAAM,eAAe,GAAG,aAAa,CAAC,gBAAgB,CAAC;WAClD,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;WAClC,6BAA6B,CAAC,YAAY,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;IACtE,MAAM,eAAe,GAAG,OAAO,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IACxE,MAAM,iBAAiB,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACrD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,KAAK,KAAK;WAC1C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAC7D,OAAO,CAAC,gBAAgB,KAAK,KAAK,IAAI,OAAO,CAAC,iBAAiB,KAAK,KAAK,CAAC,CAAC,CAAC;IAEhF,MAAM,MAAM,GAA0B;QACpC,IAAI,EAAE,kBAAkB;QACxB,EAAE,EAAE,UAAU;QACd,IAAI,EAAE,OAAO;QACb,IAAI,EAAE;YACJ,IAAI,EAAE,OAAO;YACb,KAAK;YACL,UAAU,EAAE,OAAO,IAAI,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU;YAC1D,QAAQ,EAAE,YAAY;YACtB,QAAQ;YACR,WAAW;YACX,SAAS,EAAE,WAAW,GAAG,CAAC;YAC1B,cAAc,EAAE,YAAY,CAAC,MAAM;YACnC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC7C,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC;YAC1D,SAAS,EAAE,UAAU;YACrB,UAAU,EAAE,aAAa,EAAE;YAC3B,WAAW,EAAE,OAAO,CAAC,OAAO;YAC5B,eAAe;YACf,gBAAgB;YAChB,YAAY,EAAE,OAAO,IAAI,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,6BAA6B,CAAC,iBAAiB,IAAI,YAAY,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,CAAC;YAC5I,GAAG,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3C,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC;gBAClE,aAAa,EAAE,MAAM,CAAC,WAAW,CAC/B,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,6BAA6B,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAC5H;aACF,CAAC,CAAC,CAAC,EAAE,CAAC;YACP,OAAO;YACP,GAAG;YACH,GAAG;YACH,OAAO,EAAE,UAAU,EAAE;SACtB;QACD,KAAK,EAAE,eAAe,CAAC,YAAY,CAAC;KACrC,CAAC;IAEF,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AACzB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,EAC/C,QAAQ,EACR,YAAY,EACZ,KAAK,EACL,UAAU,EACV,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,KAAK,EACf,WAAW,GAAG,CAAC,EACf,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,eAAe,GAAG,IAAI,EACtB,aAAa,GAAG,KAAK,EACrB,SAAS,EACT,OAAO,GAAG,KAAK,EACf,MAAM,EACN,WAAW,EACX,WAAW,EACX,YAAY,EACZ,OAAO,GAAG,KAAK,EACf,cAAc,EACd,oBAAoB,EACpB,mBAAmB,GAgCpB;IACC,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC5C,MAAM,YAAY,GAA6B,EAAE,CAAC;IAClD,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAC9B,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;QAErG,8DAA8D;QAC9D,MAAM,qBAAqB,GAAG,oBAAoB,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,SAAS;YAC5E,CAAC,CAAC,EAAE,GAAG,oBAAoB,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;YAClF,CAAC,CAAC,oBAAoB,CAAC;QACzB,MAAM,cAAc,GAAG,gBAAgB,CACrC,OAAO,CAAC,QAAQ,CAAC,EACjB,CAAC,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC,EAC7B,EAAE,cAAc,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,CAChE,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YACjB,IAAI,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;gBACtC,OAAO,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,WAAoB,EAAE,CAAC;YACjF,CAAC;YACD,OAAO,EAAE,GAAG,QAAQ,EAAE,cAAc,EAAE,SAAkB,EAAE,CAAC;QAC7D,CAAC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,GAAG,UAAU,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAClG,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,mBAAmB,CAAC;YACrD,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ;YACR,SAAS,EAAE,cAAc;YACzB,KAAK;YACL,UAAU;YACV,SAAS;YACT,OAAO;YACP,WAAW;YACX,SAAS;YACT,OAAO;YACP,YAAY;YACZ,iBAAiB;YACjB,QAAQ,EAAE,IAAI;YACd,UAAU,EAAE,KAAK;YACjB,UAAU;YACV,aAAa,EAAE,aAAa,IAAI,CAAC,GAAG,CAAC;YACrC,SAAS;YACT,OAAO;YACP,KAAK,EAAE,UAAU;YACjB,WAAW;YACX,WAAW;YACX,YAAY;SACb,CAAC,CAAC;QAEH,YAAY,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM;YACN,QAAQ;SACT,CAAC,CAAC;QAEH,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;QACzC,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;IACtG,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,GAAG,0BAA0B,CAAC;QAC9D,UAAU;QACV,QAAQ;QACR,YAAY;QACZ,YAAY;QACZ,KAAK;QACL,UAAU;QACV,OAAO;QACP,YAAY;QACZ,iBAAiB;QACjB,OAAO;QACP,KAAK;QACL,IAAI;QACJ,WAAW;QACX,SAAS;QACT,YAAY;QACZ,MAAM;QACN,WAAW;QACX,OAAO;KACR,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,aAAa,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACvD,MAAM,gBAAgB,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChG,IAAI,gBAAgB;QAAE,MAAM,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAClE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC3C,CAAC"}
+{"version":3,"file":"batch-evaluation-workflow.js","sourceRoot":"","sources":["../../../src/eval-workflows/batch-evaluation-workflow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACnI,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AACxI,OAAO,EAAE,6BAA6B,EAAE,MAAM,qCAAqC,CAAC;AACpF,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC9E,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAiD7D,SAAS,aAAa,CAAC,QAAoD;IACzE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACxB,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAClG,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK;SACT,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,mBAAmB,EAAE,GAAG,CAAC;SACpC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;WACrB,OAAO,CAAC;AACf,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAwB,EAAE,gBAAwB;IAC/E,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAK,EAAqB;QAC3D,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAK,EAAqB;KAC/E,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,YAAsC;IAC7D,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/E,IAAI;QACJ,SAAS;QACT,WAAW;QACX,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,WAAW;QACnB,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW;QACpC,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY;QACtC,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI;QACxD,OAAO,EAAE,qBAAqB,CAAC,MAAM,EAAE,IAAI,CAAC;QAC5C,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,4BAA4B,CAAC,EACpC,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,KAAK,GAMN;IACC,MAAM,gBAAgB,GAA+C,EAAE,CAAC;IACxE,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC;YAC1B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;mBAC7C,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe;mBACjC,6BAA6B,CAAC,YAAY,EAAE,KAAK,EAAE;oBACpD,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ;iBAChE,CAAC,CAAC;IACP,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,EACzC,UAAU,EACV,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,KAAK,EACL,IAAI,EACJ,WAAW,EACX,SAAS,EACT,YAAY,EACZ,MAAM,EACN,WAAW,EACX,OAAO,GAoBR;IACC,MAAM,oBAAoB,GAAG,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;QAChE,CAAC,CAAC,WAAW;QACb,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,iBAAiB,IAAI,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;IACzE,MAAM,OAAO,GAAG,sBAAsB,CAAC;QACrC,WAAW,EAAE,EAAE;QACf,QAAQ;QACR,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACtC,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,WAAW;YACnB,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,KAAK,CAAC,SAAS;SACzB,CAAC,CAAC;QACH,KAAK;QACL,QAAQ,EAAE,YAAY;QACtB,OAAO;QACP,WAAW;QACX,SAAS;QACT,OAAO;QACP,MAAM,EAAE,KAAK;QACb,KAAK,EAAE,KAAK;QACZ,OAAO;QACP,KAAK;QACL,IAAI;QACJ,MAAM;QACN,WAAW,EAAE,oBAAoB;QACjC,KAAK,EAAE,IAAI;KACZ,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,mBAAmB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,GAAG,GAAG,qBAAqB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAG,kBAAkB,CAAC;QAC7B,KAAK,EAAE,OAAO,UAAU,EAAE;QAC1B,KAAK,EAAE,UAAU;QACjB,QAAQ,EAAE,UAAU;QACpB,OAAO;QACP,SAAS;QACT,SAAS;QACT,UAAU;KACX,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAChG,MAAM,gBAAgB,GAAG,4BAA4B,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IACvG,MAAM,eAAe,GAAG,aAAa,CAAC,gBAAgB,CAAC;WAClD,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;WAClC,6BAA6B,CAAC,YAAY,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;IACtE,MAAM,iBAAiB,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACrD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,KAAK,KAAK;WAC1C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAC7D,OAAO,CAAC,gBAAgB,KAAK,KAAK,IAAI,OAAO,CAAC,iBAAiB,KAAK,KAAK,CAAC,CAAC,CAAC;IAEhF,MAAM,eAAe,GAAoD,oBAAoB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACzG,QAAQ,EAAE,EAAE,CAAC,QAAQ;QACrB,KAAK,EAAE,EAAE,CAAC,KAAK;QACf,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,6BAA6B,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;KACpG,CAAC,CAAC,CAAC;IAEJ,MAAM,MAAM,GAA0B;QACpC,IAAI,EAAE,kBAAkB;QACxB,EAAE,EAAE,UAAU;QACd,IAAI,EAAE,OAAO;QACb,IAAI,EAAE;YACJ,IAAI,EAAE,OAAO;YACb,KAAK;YACL,QAAQ,EAAE,YAAY;YACtB,QAAQ;YACR,WAAW;YACX,SAAS,EAAE,WAAW,GAAG,CAAC;YAC1B,cAAc,EAAE,YAAY,CAAC,MAAM;YACnC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC7C,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC;YAC1D,SAAS,EAAE,UAAU;YACrB,UAAU,EAAE,aAAa,EAAE;YAC3B,WAAW,EAAE,OAAO,CAAC,OAAO;YAC5B,eAAe;YACf,gBAAgB;YAChB,WAAW,EAAE,eAAe;YAC5B,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,OAAO;YACP,GAAG;YACH,GAAG;YACH,OAAO,EAAE,UAAU,EAAE;SACtB;QACD,KAAK,EAAE,eAAe,CAAC,YAAY,CAAC;KACrC,CAAC;IAEF,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AACzB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,EAC/C,QAAQ,EACR,YAAY,EACZ,KAAK,EACL,UAAU,EACV,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,KAAK,EACf,WAAW,GAAG,CAAC,EACf,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,eAAe,GAAG,IAAI,EACtB,gBAAgB,GAAG,KAAK,EACxB,IAAI,GAAG,IAAI,EACX,SAAS,EACT,OAAO,GAAG,KAAK,EACf,MAAM,EACN,WAAW,EACX,WAAW,EACX,YAAY,EACZ,OAAO,GAAG,KAAK,EACf,cAAc,EACd,oBAAoB,EACpB,mBAAmB,GAiCpB;IACC,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC5C,MAAM,YAAY,GAA6B,EAAE,CAAC;IAClD,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAC9B,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;QAErG,8DAA8D;QAC9D,MAAM,qBAAqB,GAAG,oBAAoB,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,SAAS;YAC5E,CAAC,CAAC,EAAE,GAAG,oBAAoB,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;YAClF,CAAC,CAAC,oBAAoB,CAAC;QACzB,MAAM,cAAc,GAAG,gBAAgB,CACrC,OAAO,CAAC,QAAQ,CAAC,EACjB,CAAC,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC,EAC7B,EAAE,cAAc,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,CAChE,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YACjB,IAAI,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;gBACtC,OAAO,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,WAAoB,EAAE,CAAC;YACjF,CAAC;YACD,OAAO,EAAE,GAAG,QAAQ,EAAE,cAAc,EAAE,SAAkB,EAAE,CAAC;QAC7D,CAAC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,GAAG,UAAU,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAClG,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,mBAAmB,CAAC;YACrD,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ;YACR,SAAS,EAAE,cAAc;YACzB,KAAK;YACL,UAAU;YACV,SAAS;YACT,OAAO;YACP,WAAW;YACX,SAAS;YACT,OAAO;YACP,YAAY;YACZ,iBAAiB;YACjB,QAAQ,EAAE,IAAI;YACd,UAAU,EAAE,KAAK;YACjB,UAAU;YACV,gBAAgB,EAAE,gBAAgB,IAAI,CAAC,GAAG,CAAC;YAC3C,IAAI;YACJ,SAAS;YACT,OAAO;YACP,KAAK,EAAE,UAAU;YACjB,WAAW;YACX,WAAW;YACX,YAAY;SACb,CAAC,CAAC;QAEH,YAAY,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM;YACN,QAAQ;SACT,CAAC,CAAC;QAEH,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;QACzC,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;IACtG,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,GAAG,0BAA0B,CAAC;QAC9D,UAAU;QACV,QAAQ;QACR,YAAY;QACZ,YAAY;QACZ,KAAK;QACL,UAAU;QACV,OAAO;QACP,YAAY;QACZ,iBAAiB;QACjB,OAAO;QACP,KAAK;QACL,IAAI;QACJ,WAAW;QACX,SAAS;QACT,YAAY;QACZ,MAAM;QACN,WAAW;QACX,OAAO;KACR,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,aAAa,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACvD,MAAM,gBAAgB,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChG,IAAI,gBAAgB;QAAE,MAAM,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAClE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC3C,CAAC"}

package/dist/src/eval-workflows/evaluation-pipeline.d.ts

@@ -49,7 +49,7 @@ export interface EvaluationPipelineOptions {
     jobStore?: JobStore | null;
     persistJob?: boolean;
     onProgress?: ProgressCallback | null;
-    skipPreflight?: boolean;
+    skipConnectivity?: boolean;
     verbose?: boolean;
     retry?: number;
     existingResults?: Record<string, Record<string, VariantResult>>;
@@ -78,7 +78,7 @@ export interface EvaluationPipelineOptions {
     /** Explicit persisted run id. Used by batch workflows that need stable child ids. */
     runId?: string;
 }
-export declare function executeEvaluationPipeline({ samplesPath, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, blind, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipPreflight, verbose, retry, existingResults, requires, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, }: EvaluationPipelineOptions): Promise<{
+export declare function executeEvaluationPipeline({ samplesPath, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, blind, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, }: EvaluationPipelineOptions): Promise<{
     report: Report;
     filePath: string | null;
 }>;

package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation-pipeline.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-pipeline.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AAgBjF,OAAO,KAAK,EACV,QAAQ,EAIR,UAAU,EACV,QAAQ,EACR,gBAAgB,EAChB,MAAM,EACN,MAAM,EACN,IAAI,EACJ,aAAa,EACd,MAAM,mBAAmB,CAAC;AA4J3B;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAiBhF;AAQD;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,QAAQ,EAAE,EACrB,cAAc,EAAE,OAAO,GAAG,SAAS,GAClC,MAAM,EAAE,CAsBV;AAgED,MAAM,WAAW,yBAAyB;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,UAAU,CAAC;IACrB,aAAa,EAAE,UAAU,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAChE,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,8BAA8B;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6BAA6B;IAC7B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yEAAyE;IACzE,WAAW,CAAC,EAAE,OAAO,mBAAmB,EAAE,WAAW,EAAE,CAAC;IACxD,mBAAmB;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,2CAA2C;IAC3C,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mFAAmF;IACnF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,wBAAwB;IACxB,MAAM,CAAC,EAAE,OAAO,mBAAmB,EAAE,UAAU,CAAC;IAChD;;iGAE6F;IAC7F,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,qFAAqF;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,WAAW,EACX,QAAQ,EACR,OAAO,EACP,KAAK,EACL,SAAS,EACT,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,QAAQ,EACR,aAAa,EACb,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,KAAa,EACb,WAAe,EACf,SAAS,EACT,OAAe,EACf,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,aAAqB,EACrB,OAAe,EACf,KAAS,EACT,eAAe,EACf,QAAQ,EACR,YAAoB,EACpB,MAAM,EACN,KAAK,EACL,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAmB,EACnB,MAAM,EACN,cAAc,EACd,KAAK,GACN,EAAE,yBAAyB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAyIlF"}
+{"version":3,"file":"evaluation-pipeline.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-pipeline.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AAgBjF,OAAO,KAAK,EACV,QAAQ,EAIR,UAAU,EACV,QAAQ,EACR,gBAAgB,EAChB,MAAM,EACN,MAAM,EACN,IAAI,EACJ,aAAa,EACd,MAAM,mBAAmB,CAAC;AA6J3B;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAiBhF;AAQD;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,QAAQ,EAAE,EACrB,cAAc,EAAE,OAAO,GAAG,SAAS,GAClC,MAAM,EAAE,CAsBV;AAgED,MAAM,WAAW,yBAAyB;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,UAAU,CAAC;IACrB,aAAa,EAAE,UAAU,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAChE,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,8BAA8B;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6BAA6B;IAC7B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yEAAyE;IACzE,WAAW,CAAC,EAAE,OAAO,mBAAmB,EAAE,WAAW,EAAE,CAAC;IACxD,mBAAmB;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,2CAA2C;IAC3C,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mFAAmF;IACnF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,wBAAwB;IACxB,MAAM,CAAC,EAAE,OAAO,mBAAmB,EAAE,UAAU,CAAC;IAChD;;iGAE6F;IAC7F,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,qFAAqF;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,WAAW,EACX,QAAQ,EACR,OAAO,EACP,KAAK,EACL,SAAS,EACT,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,QAAQ,EACR,aAAa,EACb,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,KAAa,EACb,WAAe,EACf,SAAS,EACT,OAAe,EACf,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,gBAAwB,EACxB,OAAe,EACf,KAAS,EACT,eAAe,EAGf,QAAQ,EAAE,SAAS,EACnB,YAAoB,EACpB,MAAM,EACN,KAAK,EACL,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAmB,EACnB,MAAM,EACN,cAAc,EACd,KAAK,GACN,EAAE,yBAAyB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CA0IlF"}

package/dist/src/eval-workflows/evaluation-pipeline.js

@@ -6,20 +6,20 @@ import { analyzeResults } from '../analysis/report-diagnostics.js';
 import { computeReportCoverage } from '../analysis/coverage-analyzer.js';
 import { computeReportGapRates } from '../analysis/gap-analyzer.js';
 import { aggregateReport, applyBlindMode, DEFAULT_OUTPUT_DIR, generateRunId, persistReport } from '../eval-core/evaluation-reporting.js';
-import { executeTasks, preflight } from '../eval-core/evaluation-execution.js';
-import { preflightDependencies, formatDependencyErrors } from '../eval-core/dependency-checker.js';
+import { executeTasks, preflight, preflightAllJudges } from '../eval-core/evaluation-execution.js';
 import { createFileJobStore, DEFAULT_JOBS_DIR, } from '../server/job-store.js';
 import { stopAllServers } from '../inputs/mcp-resolver.js';
 import { buildEvaluationRequest, createFailedJob, createEvaluationRun, createQueuedJob, createSucceededJob, finalizeEvaluationRun, markJobRunning, failEvaluationRun, } from '../eval-core/evaluation-job.js';
 async function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, blind, project, owner, tags, runId, jobStore, persistJob, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, }) {
+    const effectiveJudges = judgeModels && judgeModels.length > 0
+        ? judgeModels
+        : [{ executor: judgeExecutorName, model: judgeModel }];
     const request = buildEvaluationRequest({
         samplesPath,
         skillDir,
         artifacts,
         model,
-        judgeModel: noJudge ? null : judgeModel,
         executor: executorName,
-        judgeExecutor: judgeExecutorName,
         noJudge,
         concurrency,
         timeoutMs,
@@ -32,7 +32,7 @@ async function initializeEvaluationRunState({ samplesPath, skillDir, artifacts,
         repeat,
         batch,
         judgeRepeat,
-        judgeModels,
+        judgeModels: effectiveJudges,
         bootstrap,
         bootstrapSamples,
         lengthDebias,
@@ -201,7 +201,10 @@ function finalizeEvaluationReport({ report, results, artifacts, variantNames, bl
     }
     return report;
 }
-export async function executeEvaluationPipeline({ samplesPath, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, blind = false, concurrency = 1, timeoutMs, noCache = false, jobStore = null, persistJob = true, onProgress = null, skipPreflight = false, verbose = false, retry = 0, existingResults, requires, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, }) {
+export async function executeEvaluationPipeline({ samplesPath, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, blind = false, concurrency = 1, timeoutMs, noCache = false, jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, verbose = false, retry = 0, existingResults, 
+// requires 现在由 runEvaluation 上游传给 doctor 处理; eval-pipeline 不再用
+// 但保留接口字段,避免破 programmatic API (类型层面接收, 内部忽略)
+requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, }) {
     const variantNames = artifacts.map((artifact) => artifact.name);
     const runState = await initializeEvaluationRunState({
         samplesPath,
@@ -232,18 +235,37 @@ export async function executeEvaluationPipeline({ samplesPath, skillDir, samples
         budget,
     });
     try {
-        if (!skipPreflight) {
+        // 解析 judge config + 配套 executor map 一次,后续 preflight 与 executeTasks 共用。
+        // production 路径 judgeModels 始终非空(RunConfig 必填);fallback 仅兼容 internal
+        // legacy caller 还在传 single judgeModel + judgeExecutorName 的场景。
+        //
+        // executor map 必须先于 preflight build,否则 ensemble 中第二、第三个 judge 配错
+        // (404 model / executor 不存在 / auth fail)只会在 grading 半途才暴露 — 那时已经
+        // 浪费 task execution 成本 + 报告里也不会有 judgeAgreement。
+        const resolvedJudgeModels = judgeModels && judgeModels.length > 0
+            ? judgeModels
+            : [{ executor: executorName, model: judgeModel }];
+        let resolvedJudgeExecutors;
+        if (judgeModels && judgeModels.length > 0) {
+            const { createExecutor } = await import('../executors/index.js');
+            resolvedJudgeExecutors = {};
+            for (const jc of resolvedJudgeModels) {
+                if (!resolvedJudgeExecutors[jc.executor]) {
+                    resolvedJudgeExecutors[jc.executor] = createExecutor(jc.executor);
+                }
+            }
+        }
+        else {
+            resolvedJudgeExecutors = { [executorName]: judgeExecutor };
+        }
+        if (!skipConnectivity) {
             if (onProgress)
                 onProgress({ phase: 'preflight', jobId: runState.jobId });
+            // LLM 连通性: eval 唯一职责。doctor 在 runEvaluation 上游已跑,
+            // 包含 dep / 结构 / 元数据 / 契约检查。这里只剩 executor + 所有 judge。
             await preflight(executor, model);
-            if (!noJudge)
-                await preflight(judgeExecutor, judgeModel);
-            // Dependency check: auto-extract from skill contents + merge explicit requires
-            const skillContents = artifacts.map((a) => a.content).filter((c) => typeof c === 'string');
-            const cwd = artifacts.find((a) => a.cwd)?.cwd || skillDir || process.cwd();
-            const depResult = await preflightDependencies(skillContents, samples, cwd, requires, artifacts);
-            if (!depResult.ok) {
-                throw new Error(formatDependencyErrors(depResult.missing));
+            if (!noJudge) {
+                await preflightAllJudges(resolvedJudgeModels, resolvedJudgeExecutors);
             }
         }
         // Structural power warnings — print to stderr after preflight passes, before
@@ -253,28 +275,11 @@ export async function executeEvaluationPipeline({ samplesPath, skillDir, samples
         emitPowerWarnings(samples.length, repeat ?? 1);
         // Isolation pre-flight warning (--no-strict-baseline + ~/.claude/skills/ non-empty)
         emitIsolationWarnings(artifacts, strictBaseline);
-        // Pre-build a per-executor map for ensemble judges. Each unique executor name
-        // gets one ExecutorFn, shared across all judges using that executor. The default
-        // judge's executor is also included so single-judge fallbacks have it available.
-        let judgeExecutors;
-        if (judgeModels && judgeModels.length >= 2) {
-            const { createExecutor } = await import('../executors/index.js');
-            judgeExecutors = {};
-            const seen = new Set();
-            for (const jc of judgeModels) {
-                if (!seen.has(jc.executor)) {
-                    seen.add(jc.executor);
-                    judgeExecutors[jc.executor] = createExecutor(jc.executor);
-                }
-            }
-        }
         const { results, totalCostUSD, skipped, budgetExhausted } = await executeTasks({
             tasks,
             executor,
             executorName,
-            judgeExecutor,
             model,
-            judgeModel,
             noJudge,
             samplesPath,
             concurrency,
@@ -285,8 +290,8 @@ export async function executeEvaluationPipeline({ samplesPath, skillDir, samples
             retry,
             existingResults,
             judgeRepeat,
-            judgeModels,
-            judgeExecutors,
+            judgeModels: resolvedJudgeModels,
+            judgeExecutors: resolvedJudgeExecutors,
             lengthDebias,
             budget,
         });

package/dist/src/eval-workflows/evaluation-pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation-pipeline.js","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACzI,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AAEnG,OAAO,EACL,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EACL,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,iBAAiB,GAClB,MAAM,gCAAgC,CAAC;AA4BxC,KAAK,UAAU,4BAA4B,CAAC,EAC1C,WAAW,EACX,QAAQ,EACR,SAAS,EACT,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,WAAW,EACX,SAAS,EACT,OAAO,EACP,KAAK,EACL,OAAO,EACP,KAAK,EACL,IAAI,EACJ,KAAK,EACL,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,MAAM,GA4BP;IACC,MAAM,OAAO,GAAG,sBAAsB,CAAC;QACrC,WAAW;QACX,QAAQ;QACR,SAAS;QACT,KAAK;QACL,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU;QACvC,QAAQ,EAAE,YAAY;QACtB,aAAa,EAAE,iBAAiB;QAChC,OAAO;QACP,WAAW;QACX,SAAS;QACT,OAAO;QACP,MAAM,EAAE,KAAK;QACb,KAAK;QACL,OAAO;QACP,KAAK;QACL,IAAI;QACJ,MAAM;QACN,KAAK;QACL,WAAW;QACX,WAAW;QACX,SAAS;QACT,gBAAgB;QAChB,YAAY;QACZ,MAAM;KACP,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,mBAAmB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7E,MAAM,KAAK,GAAG,OAAO,KAAK,EAAE,CAAC;IAC7B,MAAM,gBAAgB,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChG,MAAM,SAAS,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IACjE,IAAI,gBAAgB;QAAE,MAAM,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,cAAc,CAAC,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IAC/D,IAAI,gBAAgB;QAAE,MAAM,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IACrE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC;AACnG,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACtD,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,GAAG,GAAG,qBAAqB,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAChE,MAAM,GAAG,GAAG,kBAAkB,CAAC;QAC7B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,QAAQ,EAAE,KAAK,CAAC,KAAK;QACrB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,UAAU;KACX,CAAC,CAAC;IACH,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,KAAyB,EAAE,GAAkB;IAC/E,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC3B,MAAM,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAyB,EAAE,GAAY;IACrE,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,SAAS,GAAG,eAAe,CAAC;QAChC,GAAG,EAAE,EAAE,GAAG,KAAK,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE;QACnG,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;QACvD,UAAU;KACX,CAAC,CAAC;IACH,KAAK,iBAAiB,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IACrD,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC3B,MAAM,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,WAAmB;IAC7C,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACnD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,kBAAkB,CAAC,WAAmB,EAAE,MAAc;IACpE,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,QAAQ,CAAC,IAAI,CACX,OAAO,WAAW,6GAA6G,CAChI,CAAC;IACJ,CAAC;SAAM,IAAI,WAAW,GAAG,EAAE,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CACX,OAAO,WAAW,kIAAkI,CACrJ,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CACX,0IAA0I,CAC3I,CAAC;IACJ,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,WAAmB,EAAE,MAAc;IAC5D,KAAK,MAAM,CAAC,IAAI,kBAAkB,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC;QACxD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CACpC,SAAqB,EACrB,cAAmC;IAEnC,qDAAqD;IACrD,IAAI,cAAc,KAAK,KAAK;QAAE,OAAO,EAAE,CAAC;IAExC,MAAM,eAAe,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACrE,IAAI,CAAC,eAAe;QAAE,OAAO,EAAE,CAAC;IAEhC,kFAAkF;IAClF,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IACvD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IAEtC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,CAAC;QACH,UAAU,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IACvF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,UAAU,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhC,OAAO;QACL,mEAAmE,UAAU,+EAA+E;KAC7J,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,SAAqB,EAAE,cAAmC;IACvF,KAAK,MAAM,CAAC,IAAI,sBAAsB,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,CAAC;QAClE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,EAChC,MAAM,EACN,OAAO,EACP,SAAS,EACT,YAAY,EACZ,KAAK,EACL,WAAW,EACX,OAAO,GASR;IACC,qEAAqE;IACrE,2EAA2E;IAC3E,0DAA0D;IAC1D,MAAM,CAAC,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAEtD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,CACjE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,SAAS,IAAI,aAAa,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CACpH,CAAC,CAAC;IACH,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5G,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5G,MAAM,QAAQ,GAAG,qBAAqB,CAAC,MAAM,EAAE,gBAAgB,EAAE,YAAY,CAAC,CAAC;QAC/E,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,QAAS,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACvC,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,2EAA2E;IAC3E,gFAAgF;IAChF,MAAM,UAAU,GAAG,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACvE,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,WAAW,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;QACpD,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;YAC/B,IAAI,CAAC,EAAE;gBAAE,SAAS;YAClB,EAAE,CAAC,WAAW,GAAG,WAAW,CAAC;YAC7B,EAAE,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,CAAC;QACD,MAAM,CAAC,QAAS,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,CAAC;IAED,IAAI,KAAK,EAAE,CAAC;QACV,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC,CAAC;IACnF,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAwDD,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,EAC9C,WAAW,EACX,QAAQ,EACR,OAAO,EACP,KAAK,EACL,SAAS,EACT,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,QAAQ,EACR,aAAa,EACb,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,KAAK,GAAG,KAAK,EACb,WAAW,GAAG,CAAC,EACf,SAAS,EACT,OAAO,GAAG,KAAK,EACf,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,aAAa,GAAG,KAAK,EACrB,OAAO,GAAG,KAAK,EACf,KAAK,GAAG,CAAC,EACT,eAAe,EACf,QAAQ,EACR,YAAY,GAAG,KAAK,EACpB,MAAM,EACN,KAAK,EACL,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAY,GAAG,IAAI,EACnB,MAAM,EACN,cAAc,EACd,KAAK,GACqB;IAC1B,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,MAAM,4BAA4B,CAAC;QAClD,WAAW;QACX,QAAQ;QACR,SAAS;QACT,KAAK;QACL,UAAU;QACV,OAAO;QACP,YAAY;QACZ,iBAAiB;QACjB,WAAW;QACX,SAAS;QACT,OAAO;QACP,KAAK;QACL,OAAO;QACP,KAAK;QACL,IAAI;QACJ,KAAK,EAAE,KAAK,IAAI,aAAa,CAAC,YAAY,CAAC;QAC3C,QAAQ;QACR,UAAU;QACV,MAAM;QACN,KAAK;QACL,WAAW;QACX,WAAW;QACX,SAAS;QACT,gBAAgB;QAChB,YAAY;QACZ,MAAM;KACP,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,IAAI,UAAU;gBAAE,UAAU,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1E,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACjC,IAAI,CAAC,OAAO;gBAAE,MAAM,SAAS,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;YAEzD,+EAA+E;YAC/E,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;YACxG,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,QAAQ,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YAC3E,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,aAAa,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,6EAA6E;QAC7E,+EAA+E;QAC/E,+EAA+E;QAC/E,4DAA4D;QAC5D,iBAAiB,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;QAC/C,oFAAoF;QACpF,qBAAqB,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QAEjD,8EAA8E;QAC9E,iFAAiF;QACjF,iFAAiF;QACjF,IAAI,cAAsD,CAAC;QAC3D,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3C,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;YACjE,cAAc,GAAG,EAAE,CAAC;YACpB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC3B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;oBACtB,cAAc,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,cAAc,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;gBAC5D,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAM,YAAY,CAAC;YAC7E,KAAK;YACL,QAAQ;YACR,YAAY;YACZ,aAAa;YACb,KAAK;YACL,UAAU;YACV,OAAO;YACP,WAAW;YACX,WAAW;YACX,SAAS;YACT,OAAO;YACP,OAAO;YACP,UAAU;YACV,KAAK;YACL,eAAe;YACf,WAAW;YACX,WAAW;YACX,cAAc;YACd,YAAY;YACZ,MAAM;SACP,CAAC,CAAC;QACH,IAAI,OAAO,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;YAC9B,UAAU,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACzH,CAAC;QAED,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,wBAAwB,CAAC;YACtC,MAAM,EAAE,eAAe,CAAC;gBACtB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,QAAQ,EAAE,YAAY;gBACtB,KAAK;gBACL,UAAU;gBACV,OAAO;gBACP,YAAY;gBACZ,OAAO;gBACP,KAAK;gBACL,OAAO;gBACP,YAAY;gBACZ,SAAS;gBACT,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,GAAG;gBACH,GAAG;gBACH,YAAY;aACb,CAAC;YACF,OAAO;YACP,SAAS;YACT,YAAY;YACZ,KAAK;YACL,WAAW;YACX,OAAO;SACR,CAAC,CAAC;QACH,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QACrC,CAAC;QACD,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,oBAAoB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC1C,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAC9B,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,gBAAgB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,MAAM,cAAc,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}
+{"version":3,"file":"evaluation-pipeline.js","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACzI,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AAEnG,OAAO,EACL,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EACL,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,iBAAiB,GAClB,MAAM,gCAAgC,CAAC;AA4BxC,KAAK,UAAU,4BAA4B,CAAC,EAC1C,WAAW,EACX,QAAQ,EACR,SAAS,EACT,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,WAAW,EACX,SAAS,EACT,OAAO,EACP,KAAK,EACL,OAAO,EACP,KAAK,EACL,IAAI,EACJ,KAAK,EACL,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,MAAM,GA4BP;IACC,MAAM,eAAe,GAA8C,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;QACtG,CAAC,CAAC,WAAW;QACb,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,iBAAiB,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,sBAAsB,CAAC;QACrC,WAAW;QACX,QAAQ;QACR,SAAS;QACT,KAAK;QACL,QAAQ,EAAE,YAAY;QACtB,OAAO;QACP,WAAW;QACX,SAAS;QACT,OAAO;QACP,MAAM,EAAE,KAAK;QACb,KAAK;QACL,OAAO;QACP,KAAK;QACL,IAAI;QACJ,MAAM;QACN,KAAK;QACL,WAAW;QACX,WAAW,EAAE,eAAe;QAC5B,SAAS;QACT,gBAAgB;QAChB,YAAY;QACZ,MAAM;KACP,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,mBAAmB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7E,MAAM,KAAK,GAAG,OAAO,KAAK,EAAE,CAAC;IAC7B,MAAM,gBAAgB,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChG,MAAM,SAAS,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IACjE,IAAI,gBAAgB;QAAE,MAAM,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,cAAc,CAAC,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IAC/D,IAAI,gBAAgB;QAAE,MAAM,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IACrE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC;AACnG,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACtD,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,GAAG,GAAG,qBAAqB,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAChE,MAAM,GAAG,GAAG,kBAAkB,CAAC;QAC7B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,QAAQ,EAAE,KAAK,CAAC,KAAK;QACrB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,UAAU;KACX,CAAC,CAAC;IACH,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,KAAyB,EAAE,GAAkB;IAC/E,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC3B,MAAM,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAyB,EAAE,GAAY;IACrE,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,SAAS,GAAG,eAAe,CAAC;QAChC,GAAG,EAAE,EAAE,GAAG,KAAK,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE;QACnG,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;QACvD,UAAU;KACX,CAAC,CAAC;IACH,KAAK,iBAAiB,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IACrD,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC3B,MAAM,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,WAAmB;IAC7C,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACnD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,kBAAkB,CAAC,WAAmB,EAAE,MAAc;IACpE,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,QAAQ,CAAC,IAAI,CACX,OAAO,WAAW,6GAA6G,CAChI,CAAC;IACJ,CAAC;SAAM,IAAI,WAAW,GAAG,EAAE,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CACX,OAAO,WAAW,kIAAkI,CACrJ,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CACX,0IAA0I,CAC3I,CAAC;IACJ,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,WAAmB,EAAE,MAAc;IAC5D,KAAK,MAAM,CAAC,IAAI,kBAAkB,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC;QACxD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CACpC,SAAqB,EACrB,cAAmC;IAEnC,qDAAqD;IACrD,IAAI,cAAc,KAAK,KAAK;QAAE,OAAO,EAAE,CAAC;IAExC,MAAM,eAAe,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACrE,IAAI,CAAC,eAAe;QAAE,OAAO,EAAE,CAAC;IAEhC,kFAAkF;IAClF,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IACvD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IAEtC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,CAAC;QACH,UAAU,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IACvF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,UAAU,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhC,OAAO;QACL,mEAAmE,UAAU,+EAA+E;KAC7J,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,SAAqB,EAAE,cAAmC;IACvF,KAAK,MAAM,CAAC,IAAI,sBAAsB,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,CAAC;QAClE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,EAChC,MAAM,EACN,OAAO,EACP,SAAS,EACT,YAAY,EACZ,KAAK,EACL,WAAW,EACX,OAAO,GASR;IACC,qEAAqE;IACrE,2EAA2E;IAC3E,0DAA0D;IAC1D,MAAM,CAAC,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAEtD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,CACjE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,SAAS,IAAI,aAAa,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CACpH,CAAC,CAAC;IACH,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5G,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5G,MAAM,QAAQ,GAAG,qBAAqB,CAAC,MAAM,EAAE,gBAAgB,EAAE,YAAY,CAAC,CAAC;QAC/E,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,QAAS,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACvC,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,2EAA2E;IAC3E,gFAAgF;IAChF,MAAM,UAAU,GAAG,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACvE,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,WAAW,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;QACpD,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;YAC/B,IAAI,CAAC,EAAE;gBAAE,SAAS;YAClB,EAAE,CAAC,WAAW,GAAG,WAAW,CAAC;YAC7B,EAAE,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,CAAC;QACD,MAAM,CAAC,QAAS,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,CAAC;IAED,IAAI,KAAK,EAAE,CAAC;QACV,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC,CAAC;IACnF,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAwDD,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,EAC9C,WAAW,EACX,QAAQ,EACR,OAAO,EACP,KAAK,EACL,SAAS,EACT,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,QAAQ,EACR,aAAa,EACb,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,KAAK,GAAG,KAAK,EACb,WAAW,GAAG,CAAC,EACf,SAAS,EACT,OAAO,GAAG,KAAK,EACf,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,gBAAgB,GAAG,KAAK,EACxB,OAAO,GAAG,KAAK,EACf,KAAK,GAAG,CAAC,EACT,eAAe;AACf,+DAA+D;AAC/D,8CAA8C;AAC9C,QAAQ,EAAE,SAAS,EACnB,YAAY,GAAG,KAAK,EACpB,MAAM,EACN,KAAK,EACL,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAY,GAAG,IAAI,EACnB,MAAM,EACN,cAAc,EACd,KAAK,GACqB;IAC1B,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,MAAM,4BAA4B,CAAC;QAClD,WAAW;QACX,QAAQ;QACR,SAAS;QACT,KAAK;QACL,UAAU;QACV,OAAO;QACP,YAAY;QACZ,iBAAiB;QACjB,WAAW;QACX,SAAS;QACT,OAAO;QACP,KAAK;QACL,OAAO;QACP,KAAK;QACL,IAAI;QACJ,KAAK,EAAE,KAAK,IAAI,aAAa,CAAC,YAAY,CAAC;QAC3C,QAAQ;QACR,UAAU;QACV,MAAM;QACN,KAAK;QACL,WAAW;QACX,WAAW;QACX,SAAS;QACT,gBAAgB;QAChB,YAAY;QACZ,MAAM;KACP,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,uEAAuE;QACvE,qEAAqE;QACrE,+DAA+D;QAC/D,EAAE;QACF,iEAAiE;QACjE,iEAAiE;QACjE,iDAAiD;QACjD,MAAM,mBAAmB,GAAG,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YAC/D,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACpD,IAAI,sBAAkD,CAAC;QACvD,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;YACjE,sBAAsB,GAAG,EAAE,CAAC;YAC5B,KAAK,MAAM,EAAE,IAAI,mBAAmB,EAAE,CAAC;gBACrC,IAAI,CAAC,sBAAsB,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACzC,sBAAsB,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,cAAc,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,sBAAsB,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,aAAa,EAAE,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,IAAI,UAAU;gBAAE,UAAU,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1E,kDAAkD;YAClD,qDAAqD;YACrD,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACjC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,kBAAkB,CAAC,mBAAmB,EAAE,sBAAsB,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;QAED,6EAA6E;QAC7E,+EAA+E;QAC/E,+EAA+E;QAC/E,4DAA4D;QAC5D,iBAAiB,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;QAC/C,oFAAoF;QACpF,qBAAqB,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QAEjD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAM,YAAY,CAAC;YAC7E,KAAK;YACL,QAAQ;YACR,YAAY;YACZ,KAAK;YACL,OAAO;YACP,WAAW;YACX,WAAW;YACX,SAAS;YACT,OAAO;YACP,OAAO;YACP,UAAU;YACV,KAAK;YACL,eAAe;YACf,WAAW;YACX,WAAW,EAAE,mBAAmB;YAChC,cAAc,EAAE,sBAAsB;YACtC,YAAY;YACZ,MAAM;SACP,CAAC,CAAC;QACH,IAAI,OAAO,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;YAC9B,UAAU,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACzH,CAAC;QAED,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,wBAAwB,CAAC;YACtC,MAAM,EAAE,eAAe,CAAC;gBACtB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,QAAQ,EAAE,YAAY;gBACtB,KAAK;gBACL,UAAU;gBACV,OAAO;gBACP,YAAY;gBACZ,OAAO;gBACP,KAAK;gBACL,OAAO;gBACP,YAAY;gBACZ,SAAS;gBACT,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,GAAG;gBACH,GAAG;gBACH,YAAY;aACb,CAAC;YACF,OAAO;YACP,SAAS;YACT,YAAY;YACZ,KAAK;YACL,WAAW;YACX,OAAO;SACR,CAAC,CAAC;QACH,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QACrC,CAAC;QACD,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,oBAAoB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC1C,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAC9B,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,gBAAgB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,MAAM,cAAc,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}

package/dist/src/eval-workflows/evaluation-preparation.d.ts

@@ -19,16 +19,18 @@ export declare function prepareEvaluationRun({ samplesPath, skillDir, variantSpe
     /**  — eval.yaml 显式 allowedSkills per variant (overrides strictBaseline default). */
     variantAllowedSkills?: Record<string, string[]>;
 }): Promise<PreparedEvaluationRun>;
-export declare function buildDryRunTaskReport({ model, judgeModel, judgeModels, executorName, samplesPath, skillDir, tasks, variantNames, }: {
+export declare function buildDryRunTaskReport({ model, judgeModels, executorName, samplesPath, skillDir, tasks, variantNames, }: {
     model: string;
-    judgeModel: string;
-    judgeModels?: JudgeConfig[];
+    judgeModels: JudgeConfig[];
     executorName: string;
     samplesPath: string;
     skillDir: string;
     tasks: Task[];
     variantNames: string[];
 }): {
+    dryRun: true;
+    model: string;
+    judgeModels: JudgeConfig[];
     variants: string[];
     executor: string;
     samplesPath: string;
@@ -49,22 +51,5 @@ export declare function buildDryRunTaskReport({ model, judgeModel, judgeModels,
         hasDimensions: boolean;
         hasSystem: boolean;
     }[];
-    judgeModels?: string[] | undefined;
-    dryRun: true;
-    model: string;
-    judgeModel: string | null;
-};
-export declare function buildDryRunBatchArtifacts(skillEntries: Array<{
-    name: string;
-    skillPath: string;
-    samplesPath: string;
-}>): {
-    artifacts: {
-        name: string;
-        samplesPath: string;
-        sampleCount: number;
-        taskCount: number;
-    }[];
-    totalTasks: number;
 };
 //# sourceMappingURL=evaluation-preparation.d.ts.map

package/dist/src/eval-workflows/evaluation-preparation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation-preparation.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-preparation.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AACjF,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAc,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEtG,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,CAAC,EAAE,sBAAsB,CAAC;CACnC;AAED,wBAAsB,oBAAoB,CAAC,EACzC,WAAW,EACX,QAAQ,EACR,YAAY,EACZ,SAAS,EACT,MAAM,EACN,SAAS,EACT,cAAc,EACd,oBAAoB,GACrB,EAAE;IACD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,oFAAoF;IACpF,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACjD,GAAG,OAAO,CAAC,qBAAqB,CAAC,CA6CjC;AAED,wBAAgB,qBAAqB,CAAC,EACpC,KAAK,EACL,UAAU,EACV,WAAW,EACX,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,KAAK,EACL,YAAY,GACb,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,WAAW,EAAE,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;;;;;;;;;;;;;;;;;;;;;;;;;EAiCA;AAED,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC;;;;;;;;EAetH"}
+{"version":3,"file":"evaluation-preparation.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-preparation.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AACjF,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAc,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEtG,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,CAAC,EAAE,sBAAsB,CAAC;CACnC;AAED,wBAAsB,oBAAoB,CAAC,EACzC,WAAW,EACX,QAAQ,EACR,YAAY,EACZ,SAAS,EACT,MAAM,EACN,SAAS,EACT,cAAc,EACd,oBAAoB,GACrB,EAAE;IACD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,oFAAoF;IACpF,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACjD,GAAG,OAAO,CAAC,qBAAqB,CAAC,CA6CjC;AAED,wBAAgB,qBAAqB,CAAC,EACpC,KAAK,EACL,WAAW,EACX,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,KAAK,EACL,YAAY,GACb,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,WAAW,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;;;;;;;;;;;;;;;;;;;;;;;;EA6BA"}

package/dist/src/eval-workflows/evaluation-preparation.js

@@ -42,15 +42,11 @@ export async function prepareEvaluationRun({ samplesPath, skillDir, variantSpecs
         requires,
     };
 }
-export function buildDryRunTaskReport({ model, judgeModel, judgeModels, executorName, samplesPath, skillDir, tasks, variantNames, }) {
-    const judgeModelList = judgeModels && judgeModels.length >= 2
-        ? judgeModels.map((jc) => `${jc.executor}:${jc.model}`)
-        : undefined;
+export function buildDryRunTaskReport({ model, judgeModels, executorName, samplesPath, skillDir, tasks, variantNames, }) {
     return {
         dryRun: true,
         model,
-        judgeModel: judgeModelList ? null : judgeModel,
-        ...(judgeModelList ? { judgeModels: judgeModelList } : {}),
+        judgeModels,
         variants: variantNames,
         executor: executorName,
         samplesPath,
@@ -76,19 +72,4 @@ export function buildDryRunTaskReport({ model, judgeModel, judgeModels, executor
         }),
     };
 }
-export function buildDryRunBatchArtifacts(skillEntries) {
-    const artifacts = skillEntries.map((entry) => {
-        const { samples } = loadSamples(entry.samplesPath);
-        return {
-            name: entry.name,
-            samplesPath: entry.samplesPath,
-            sampleCount: samples.length,
-            taskCount: samples.length * 2,
-        };
-    });
-    return {
-        artifacts,
-        totalTasks: artifacts.reduce((sum, artifact) => sum + artifact.taskCount, 0),
-    };
-}
 //# sourceMappingURL=evaluation-preparation.js.map

package/dist/src/eval-workflows/evaluation-preparation.js.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation-preparation.js","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-preparation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAYvD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EACzC,WAAW,EACX,QAAQ,EACR,YAAY,EACZ,SAAS,EACT,MAAM,EACN,SAAS,EACT,cAAc,EACd,oBAAoB,GAYrB;IACC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IAEvD,4EAA4E;IAC5E,MAAM,kBAAkB,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjE,MAAM,iBAAiB,GAAG,SAAS,IAAI,gBAAgB,CACrD,OAAO,CAAC,QAAQ,CAAC,EACjB,kBAAkB,EAClB,EAAE,cAAc,EAAE,oBAAoB,EAAE,CACzC,CAAC;IAEF,iFAAiF;IACjF,MAAM,UAAU,GAAwC,EAAE,CAAC;IAC3D,KAAK,MAAM,IAAI,IAAI,YAAY;QAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;IACnE,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;QACzC,IAAI,QAAQ,CAAC,cAAc;YAAE,SAAS,CAAE,gDAAgD;QACxF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,IAAI;YAAE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC;IAC3C,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,UAAU,GAAsB,aAAa,CAAC,SAAS,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,cAAc,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAU,CAAC;QAC/F,MAAM,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,sBAAsB;cACpB,qBAAqB,OAAO,CAAC,QAAQ,CAAC,IAAI;cAC1C,sCAAsC;cACtC,wEAAwE,CAC3E,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,uBAAuB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;IAClE,MAAM,YAAY,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAExE,OAAO;QACL,OAAO;QACP,SAAS,EAAE,iBAAiB;QAC5B,KAAK;QACL,YAAY;QACZ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,EACpC,KAAK,EACL,UAAU,EACV,WAAW,EACX,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,KAAK,EACL,YAAY,GAUb;IACC,MAAM,cAAc,GAAG,WAAW,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC;QAC3D,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC;QACvD,CAAC,CAAC,SAAS,CAAC;IACd,OAAO;QACL,MAAM,EAAE,IAAa;QACrB,KAAK;QACL,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU;QAC9C,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,YAAY;QACtB,WAAW;QACX,QAAQ;QACR,UAAU,EAAE,KAAK,CAAC,MAAM;QACxB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxB,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjD,OAAO;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;gBAChC,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBACpC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;gBAC3C,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBACxC,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC/B,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;gBAC/C,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;gBAC9E,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC;aACzC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,YAA6E;IACrH,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC3C,MAAM,EAAE,OAAO,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACnD,OAAO;YACL,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,WAAW,EAAE,OAAO,CAAC,MAAM;YAC3B,SAAS,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;SAC9B,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,SAAS;QACT,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;KAC7E,CAAC;AACJ,CAAC"}
+{"version":3,"file":"evaluation-preparation.js","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-preparation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAYvD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EACzC,WAAW,EACX,QAAQ,EACR,YAAY,EACZ,SAAS,EACT,MAAM,EACN,SAAS,EACT,cAAc,EACd,oBAAoB,GAYrB;IACC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IAEvD,4EAA4E;IAC5E,MAAM,kBAAkB,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjE,MAAM,iBAAiB,GAAG,SAAS,IAAI,gBAAgB,CACrD,OAAO,CAAC,QAAQ,CAAC,EACjB,kBAAkB,EAClB,EAAE,cAAc,EAAE,oBAAoB,EAAE,CACzC,CAAC;IAEF,iFAAiF;IACjF,MAAM,UAAU,GAAwC,EAAE,CAAC;IAC3D,KAAK,MAAM,IAAI,IAAI,YAAY;QAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;IACnE,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;QACzC,IAAI,QAAQ,CAAC,cAAc;YAAE,SAAS,CAAE,gDAAgD;QACxF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,IAAI;YAAE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC;IAC3C,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,UAAU,GAAsB,aAAa,CAAC,SAAS,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,cAAc,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAU,CAAC;QAC/F,MAAM,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,sBAAsB;cACpB,qBAAqB,OAAO,CAAC,QAAQ,CAAC,IAAI;cAC1C,sCAAsC;cACtC,wEAAwE,CAC3E,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,uBAAuB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;IAClE,MAAM,YAAY,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAExE,OAAO;QACL,OAAO;QACP,SAAS,EAAE,iBAAiB;QAC5B,KAAK;QACL,YAAY;QACZ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,EACpC,KAAK,EACL,WAAW,EACX,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,KAAK,EACL,YAAY,GASb;IACC,OAAO;QACL,MAAM,EAAE,IAAa;QACrB,KAAK;QACL,WAAW;QACX,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,YAAY;QACtB,WAAW;QACX,QAAQ;QACR,UAAU,EAAE,KAAK,CAAC,MAAM;QACxB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxB,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjD,OAAO;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;gBAChC,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBACpC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;gBAC3C,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBACxC,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC/B,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;gBAC/C,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;gBAC9E,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC;aACzC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC"}

package/dist/src/eval-workflows/run-evaluation.d.ts

@@ -7,7 +7,6 @@ export interface SkillProgressInfo {
 }
 interface CommonEvaluationOptions {
     model?: string;
-    judgeModel?: string;
     outputDir?: string | null;
     project?: string;
     owner?: string;
@@ -16,11 +15,13 @@ interface CommonEvaluationOptions {
     concurrency?: number;
     timeoutMs?: number;
     executorName?: string;
-    judgeExecutorName?: string;
     jobStore?: JobStore | null;
     persistJob?: boolean;
     onProgress?: ProgressCallback | null;
-    skipPreflight?: boolean;
+    /** 跳过 LLM 模型连通性检测。--resume 时自动 true(已经验过)。 */
+    skipConnectivity?: boolean;
+    /** 用户语言, 透传给 doctor 报告渲染。 */
+    lang?: 'zh' | 'en';
     mcpConfig?: string;
     verbose?: boolean;
     layeredStats?: boolean;
@@ -33,14 +34,15 @@ interface CommonEvaluationOptions {
     /** --judge-repeat N. 每条 sample × dimension 调 LLM judge N 次, 输出 stddev (judge 自一致性).
      *  默认 1 (单次). 用于量化 LLM judge 在该 rubric 上的稳定性 — stddev 高 = 评分噪声大. */
     judgeRepeat?: number;
-    /** --judge-models executor:model,executor:model,... — multi-judge ensemble. ≥ 2 个
-     *  judge 时每条 sample × dimension 由所有 judge 各自打分, 输出 inter-judge agreement. */
+    /** Unified judge config. 1 entry = single judge, ≥ 2 = ensemble + inter-judge agreement.
+     *  Optional at the API surface; runEvaluation defaults to `[{executor, model: 'haiku'}]`
+     *  when omitted. RunConfig (parseRunConfig 出口) 保证非空。 */
     judgeModels?: import('../types/index.js').JudgeConfig[];
     /** --bootstrap. Distribution-free CI on each variant mean + pairwise diff. */
     bootstrap?: boolean;
     /** --bootstrap-samples N. Default 1000. */
     bootstrapSamples?: number;
-    /** v0.21 Phase 3a length-debias toggle. Default true (judge prompt v3-cot-length).
+    /** length-debias toggle. Default true (judge prompt v3-cot-length).
      *  CLI passes false when --no-debias-length is set. */
     lengthDebias?: boolean;
     /** hard budget caps. */
@@ -100,8 +102,7 @@ interface DryRunTask {
 interface DryRunBase {
     dryRun: true;
     model: string;
-    judgeModel: string | null;
-    judgeModels?: string[];
+    judgeModels: import('../types/index.js').JudgeConfig[];
     executor: string;
     skillDir: string;
     totalTasks: number;
@@ -111,7 +112,7 @@ export interface DryRunReport extends DryRunBase {
     samplesPath: string;
     tasks: DryRunTask[];
 }
-export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, artifacts, model, judgeModel, outputDir, project, owner, tags, noJudge, dryRun, blind, concurrency, timeoutMs, noCache, executorName, judgeExecutorName, jobStore, persistJob, onProgress, skipPreflight, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, variantAllowedSkills, }: RunEvaluationOptions): Promise<{
+export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, artifacts, model, outputDir, project, owner, tags, noJudge, dryRun, blind, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, variantAllowedSkills, }: RunEvaluationOptions): Promise<{
     report: Report | DryRunReport;
     filePath: string | null;
 }>;
@@ -127,7 +128,7 @@ export interface DryRunBatchReport extends DryRunBase {
     artifacts: DryRunBatchSkill[];
 }
 export declare function buildVarianceData(runs: Report[]): VarianceData | null;
-export declare function runBatchEvaluation({ skillDir, model, judgeModel, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, executorName, judgeExecutorName, jobStore, persistJob, onProgress, onSkillProgress, skipPreflight, mcpConfig, verbose, repeat, judgeRepeat, judgeModels, lengthDebias, noCache, strictBaseline, variantAllowedSkills, }: RunBatchEvaluationOptions): Promise<{
+export declare function runBatchEvaluation({ skillDir, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, executorName, jobStore, persistJob, onProgress, onSkillProgress, skipConnectivity, lang, mcpConfig, verbose, repeat, judgeRepeat, judgeModels, lengthDebias, noCache, strictBaseline, variantAllowedSkills, }: RunBatchEvaluationOptions): Promise<{
     report: BatchEvaluationReport | DryRunBatchReport;
     filePath: string | null;
 }>;

package/dist/src/eval-workflows/run-evaluation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-evaluation.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/run-evaluation.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EACV,QAAQ,EACR,qBAAqB,EACrB,gBAAgB,EAEhB,QAAQ,EACR,gBAAgB,EAChB,MAAM,EAIN,YAAY,EAGZ,WAAW,EAGZ,MAAM,mBAAmB,CAAC;AAI3B,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,UAAU,uBAAuB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAIlB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;yDACqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;wCACoC;IACpC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;wEACoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;iFAC6E;IAC7E,WAAW,CAAC,EAAE,OAAO,mBAAmB,EAAE,WAAW,EAAE,CAAC;IACxD,8EAA8E;IAC9E,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,2CAA2C;IAC3C,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;2DACuD;IACvD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,wBAAwB;IACxB,MAAM,CAAC,EAAE,OAAO,mBAAmB,EAAE,UAAU,CAAC;IAChD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,oBAAqB,SAAQ,uBAAuB;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qFAAqF;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;6DAEyD;IACzD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;0EACsE;IACtE,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,yBAA0B,SAAQ,uBAAuB;IACxE,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,eAAe,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,iBAAiB,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7D,qDAAqD;IACrD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,6EAA6E;IAC7E,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;CAC5E;AAED,UAAU,UAAU;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/B,cAAc,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,MAAM,EAAE,IAAI,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,wBAAsB,aAAa,CAAC,EAClC,WAAW,EACX,QAAQ,EACR,YAAiB,EACjB,SAAS,EACT,KAAqB,EACrB,UAAwB,EACxB,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAe,EACf,MAAc,EACd,KAAa,EACb,WAAe,EACf,SAAS,EACT,OAAe,EACf,YAAuB,EACvB,iBAAiB,EACjB,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,aAAqB,EACrB,SAAS,EACT,OAAe,EACf,KAAS,EACT,MAAM,EACN,KAAK,EACL,YAAoB,EACpB,MAAM,EACN,KAAK,EACL,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,MAAM,EACN,cAAc,EACd,oBAAoB,GACrB,EAAE,oBAAoB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAuH5F;AAED,UAAU,gBAAgB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,iBAAkB,SAAQ,UAAU;IACnD,KAAK,EAAE,IAAI,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAyHD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,IAAI,CA6ErE;AAED,wBAAsB,kBAAkB,CAAC,EACvC,QAAQ,EACR,KAAqB,EACrB,UAAwB,EACxB,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAe,EACf,MAAc,EACd,WAAe,EACf,SAAS,EACT,YAAuB,EACvB,iBAAiB,EACjB,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,eAAsB,EACtB,aAAqB,EACrB,SAAS,EACT,OAAe,EACf,MAAM,EACN,WAAW,EACX,WAAW,EACX,YAAY,EACZ,OAAe,EACf,cAAc,EACd,oBAAoB,GACrB,EAAE,yBAAyB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,qBAAqB,GAAG,iBAAiB,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAiErH;AAED,wBAAsB,WAAW,CAAC,EAAE,MAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,EAAE,EAAE,kBAAkB;;;;GAyBhG"}
+{"version":3,"file":"run-evaluation.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/run-evaluation.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EACV,QAAQ,EACR,qBAAqB,EACrB,gBAAgB,EAEhB,QAAQ,EACR,gBAAgB,EAChB,MAAM,EAIN,YAAY,EAGZ,WAAW,EAGZ,MAAM,mBAAmB,CAAC;AAI3B,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,UAAU,uBAAuB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,8CAA8C;IAC9C,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,6BAA6B;IAC7B,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAIlB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;yDACqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;wCACoC;IACpC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;wEACoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;4DAEwD;IACxD,WAAW,CAAC,EAAE,OAAO,mBAAmB,EAAE,WAAW,EAAE,CAAC;IACxD,8EAA8E;IAC9E,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,2CAA2C;IAC3C,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;2DACuD;IACvD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,wBAAwB;IACxB,MAAM,CAAC,EAAE,OAAO,mBAAmB,EAAE,UAAU,CAAC;IAChD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,oBAAqB,SAAQ,uBAAuB;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qFAAqF;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;6DAEyD;IACzD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;0EACsE;IACtE,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,yBAA0B,SAAQ,uBAAuB;IACxE,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,eAAe,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,iBAAiB,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7D,qDAAqD;IACrD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,6EAA6E;IAC7E,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;CAC5E;AAED,UAAU,UAAU;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/B,cAAc,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,MAAM,EAAE,IAAI,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,OAAO,mBAAmB,EAAE,WAAW,EAAE,CAAC;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,wBAAsB,aAAa,CAAC,EAClC,WAAW,EACX,QAAQ,EACR,YAAiB,EACjB,SAAS,EACT,KAAqB,EACrB,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAe,EACf,MAAc,EACd,KAAa,EACb,WAAe,EACf,SAAS,EACT,OAAe,EACf,YAAuB,EACvB,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,gBAAwB,EACxB,IAAW,EACX,SAAS,EACT,OAAe,EACf,KAAS,EACT,MAAM,EACN,KAAK,EACL,YAAoB,EACpB,MAAM,EACN,KAAK,EACL,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,MAAM,EACN,cAAc,EACd,oBAAoB,GACrB,EAAE,oBAAoB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAqK5F;AAED,UAAU,gBAAgB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,iBAAkB,SAAQ,UAAU;IACnD,KAAK,EAAE,IAAI,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAyHD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,IAAI,CA6ErE;AAED,wBAAsB,kBAAkB,CAAC,EACvC,QAAQ,EACR,KAAqB,EACrB,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAe,EACf,MAAc,EACd,WAAe,EACf,SAAS,EACT,YAAuB,EACvB,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,eAAsB,EACtB,gBAAwB,EACxB,IAAW,EACX,SAAS,EACT,OAAe,EACf,MAAM,EACN,WAAW,EACX,WAAW,EACX,YAAY,EACZ,OAAe,EACf,cAAc,EACd,oBAAoB,GACrB,EAAE,yBAAyB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,qBAAqB,GAAG,iBAAiB,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAoIrH;AAED,wBAAsB,WAAW,CAAC,EAAE,MAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,EAAE,EAAE,kBAAkB;;;;GAyBhG"}