oh-my-knowledge 0.22.0 → 0.24.0

package/dist/src/eval-core/cache.js

@@ -2,12 +2,14 @@
  * Executor result cache.
  *
  * Caches successful executor results to disk to avoid redundant API calls.
- * Cache key v2 = "v2:" + sha256(model + system + prompt + cwd + allowedSkills).
+ * Cache key v4 = sha256(model + system + prompt + cwd + allowedSkills + executor + runtime).
  * Loaded into memory on init, flushed to disk on save().
  *
- * v0.22 — key prefix bump v1 → v2 invalidates pre-isolation cache entries
- * (otherwise a strict-baseline run could replay a non-isolated cached result
- * and silently keep the contamination).
+ * Prefix bumps intentionally invalidate old entries when construct-validity
+ * dimensions enter the key:
+ * - v2: allowedSkills / strict isolation
+ * - v3: executor name
+ * - v4: executor runtime fingerprint
  */
 import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
@@ -48,16 +50,18 @@ export function createCache(cacheDir) {
         },
     };
 }
-export function cacheKey(model, system, prompt, cwd, allowedSkills) {
+export function cacheKey(model, system, prompt, cwd, allowedSkills, executor, runtimeFingerprint) {
     // allowedSkills 序列化:undefined → "" / [] → "[]" / [...] → 排序后 JSON。
     // 排序保证 ["a","b"] 和 ["b","a"] 命中同一缓存(语义等价)。
     const isoStr = allowedSkills === undefined
         ? ''
         : JSON.stringify([...allowedSkills].sort());
+    // executor + runtime 进 cache key:同 model 名走不同 executor 或同 executor
+    // 换 binary/SDK 版本时输出可能不同,旧 cache 不可复用。
     const hash = createHash('sha256')
-        .update(`${model || ''}\n${system || ''}\n${prompt || ''}\n${cwd || ''}\n${isoStr}`)
+        .update(`${model || ''}\n${system || ''}\n${prompt || ''}\n${cwd || ''}\n${isoStr}\n${executor || ''}\n${runtimeFingerprint || ''}`)
         .digest('hex')
         .slice(0, 16);
-    return `v2:${hash}`;
+    return `v4:${hash}`;
 }
 //# sourceMappingURL=cache.js.map

package/dist/src/eval-core/cache.js.map

@@ -1 +1 @@
-{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../../src/eval-core/cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAEzC,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAE5C,IAAI,KAAK,GAA+B,EAAE,CAAC;IAC3C,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAED,IAAI,KAAK,GAAG,KAAK,CAAC;IAElB,OAAO;QACL,GAAG,CAAC,GAAW;YACb,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QAC5B,CAAC;QAED,GAAG,CAAC,GAAW,EAAE,KAAiB;YAChC,MAAM,SAAS,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;YAC/B,OAAO,SAAS,CAAC,KAAK,CAAC;YACvB,OAAO,SAAS,CAAC,SAAS,CAAC;YAC3B,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;YACvB,KAAK,GAAG,IAAI,CAAC;QACf,CAAC;QAED,IAAI;YACF,IAAI,CAAC,KAAK;gBAAE,OAAO;YACnB,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACxD,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;QAED,IAAI;YACF,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACnC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,KAAa,EACb,MAAc,EACd,MAAc,EACd,GAAmB,EACnB,aAAwB;IAExB,mEAAmE;IACnE,2CAA2C;IAC3C,MAAM,MAAM,GAAG,aAAa,KAAK,SAAS;QACxC,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;SAC9B,MAAM,CAAC,GAAG,KAAK,IAAI,EAAE,KAAK,MAAM,IAAI,EAAE,KAAK,MAAM,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,MAAM,EAAE,CAAC;SACnF,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChB,OAAO,MAAM,IAAI,EAAE,CAAC;AACtB,CAAC"}
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../../src/eval-core/cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAEzC,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAE5C,IAAI,KAAK,GAA+B,EAAE,CAAC;IAC3C,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAED,IAAI,KAAK,GAAG,KAAK,CAAC;IAElB,OAAO;QACL,GAAG,CAAC,GAAW;YACb,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QAC5B,CAAC;QAED,GAAG,CAAC,GAAW,EAAE,KAAiB;YAChC,MAAM,SAAS,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;YAC/B,OAAO,SAAS,CAAC,KAAK,CAAC;YACvB,OAAO,SAAS,CAAC,SAAS,CAAC;YAC3B,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;YACvB,KAAK,GAAG,IAAI,CAAC;QACf,CAAC;QAED,IAAI;YACF,IAAI,CAAC,KAAK;gBAAE,OAAO;YACnB,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACxD,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;QAED,IAAI;YACF,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACnC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,KAAa,EACb,MAAc,EACd,MAAc,EACd,GAAmB,EACnB,aAAwB,EACxB,QAAiB,EACjB,kBAA2B;IAE3B,mEAAmE;IACnE,2CAA2C;IAC3C,MAAM,MAAM,GAAG,aAAa,KAAK,SAAS;QACxC,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,mEAAmE;IACnE,uCAAuC;IACvC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;SAC9B,MAAM,CAAC,GAAG,KAAK,IAAI,EAAE,KAAK,MAAM,IAAI,EAAE,KAAK,MAAM,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,MAAM,KAAK,QAAQ,IAAI,EAAE,KAAK,kBAAkB,IAAI,EAAE,EAAE,CAAC;SACnI,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChB,OAAO,MAAM,IAAI,EAAE,CAAC;AACtB,CAAC"}

package/dist/src/eval-core/comparability.d.ts

@@ -0,0 +1,11 @@
+import type { Lang, Report } from '../types/index.js';
+export interface ComparabilityWarning {
+    code: string;
+    severity: 'warning';
+    zh: string;
+    en: string;
+}
+export declare function reportComparabilityWarnings(report: Report): ComparabilityWarning[];
+export declare function crossReportComparabilityWarnings(before: Report, after: Report): ComparabilityWarning[];
+export declare function formatComparabilityWarnings(warnings: ComparabilityWarning[], lang?: Lang): string;
+//# sourceMappingURL=comparability.d.ts.map

package/dist/src/eval-core/comparability.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparability.d.ts","sourceRoot":"","sources":["../../../src/eval-core/comparability.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,EAAkE,MAAM,mBAAmB,CAAC;AAEtH,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,SAAS,CAAC;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;CACZ;AAsJD,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,MAAM,GAAG,oBAAoB,EAAE,CA8DlF;AAED,wBAAgB,gCAAgC,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,oBAAoB,EAAE,CAsKtG;AAED,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,oBAAoB,EAAE,EAAE,IAAI,GAAE,IAAW,GAAG,MAAM,CAMvG"}

package/dist/src/eval-core/comparability.js

@@ -0,0 +1,271 @@
+function stableStringify(value) {
+    if (value === null || typeof value !== 'object')
+        return JSON.stringify(value);
+    if (Array.isArray(value))
+        return '[' + value.map(stableStringify).join(',') + ']';
+    const entries = Object.keys(value).sort();
+    return '{' + entries.map((k) => JSON.stringify(k) + ':' + stableStringify(value[k])).join(',') + '}';
+}
+function runtimeLabel(runtime) {
+    if (!runtime)
+        return 'missing';
+    const bits = [`${runtime.executor}:${runtime.model}`, `fp=${runtime.fingerprint}`];
+    if (runtime.binary?.version)
+        bits.push(`binary=${runtime.binary.version}`);
+    if (runtime.sdk?.version)
+        bits.push(`sdk=${runtime.sdk.version}`);
+    return bits.join(' ');
+}
+function push(warnings, code, zh, en) {
+    warnings.push({ code, severity: 'warning', zh, en });
+}
+function sorted(values) {
+    return [...(values ?? [])].sort();
+}
+function hasJudge(meta) {
+    return Boolean(meta.judgeModel || (meta.judgeModels && meta.judgeModels.length > 0));
+}
+function effectiveJudgeRepeat(meta) {
+    return meta.judgeRepeat ?? 1;
+}
+function packageReasons(pkg, label) {
+    if (!pkg)
+        return [`${label} missing`];
+    const reasons = [];
+    if (pkg.error)
+        reasons.push(`${label} probe failed`);
+    if (!pkg.version)
+        reasons.push(`${label} version missing`);
+    return reasons;
+}
+function runtimeUnverifiableReasons(runtime) {
+    const reasons = [];
+    if (runtime.kind === 'api')
+        return reasons;
+    if (runtime.kind === 'agent-sdk') {
+        reasons.push(...packageReasons(runtime.sdk, 'sdk'));
+    }
+    const binary = runtime.binary;
+    if (!binary) {
+        reasons.push('binary missing');
+        return reasons;
+    }
+    if (binary.error)
+        reasons.push('binary probe failed');
+    if (binary.source === 'path') {
+        if (!binary.path)
+            reasons.push('binary path missing');
+        if (!binary.version)
+            reasons.push('binary version missing');
+    }
+    else if (binary.source === 'bundled') {
+        if (!binary.version && !binary.package?.version)
+            reasons.push('bundled binary version missing');
+        if (binary.package)
+            reasons.push(...packageReasons(binary.package, 'bundled package').filter((r) => r !== 'bundled package missing'));
+    }
+    else if (binary.source === 'unknown') {
+        reasons.push('binary source unknown');
+    }
+    if (runtime.kind === 'unknown')
+        reasons.push('runtime kind unknown');
+    return [...new Set(reasons)];
+}
+function pushRuntimeUnverifiable(warnings, code, zhSubject, enSubject, runtime) {
+    if (!runtime)
+        return;
+    const reasons = runtimeUnverifiableReasons(runtime);
+    if (reasons.length === 0)
+        return;
+    push(warnings, code, `${zhSubject} runtime 指纹不可完全审计: ${runtimeLabel(runtime)} (${reasons.join('; ')})。`, `${enSubject} runtime fingerprint is not fully verifiable: ${runtimeLabel(runtime)} (${reasons.join('; ')}).`);
+}
+function runtimeMapKeys(meta) {
+    return sorted(meta.executorRuntimes ? Object.keys(meta.executorRuntimes) : undefined);
+}
+function reportExecutorRuntimeWarnings(report, warnings) {
+    const runtimes = report.meta.executorRuntimes;
+    if (runtimes && Object.keys(runtimes).length > 0) {
+        const missing = report.meta.variants.filter((variant) => !runtimes[variant]);
+        if (missing.length > 0) {
+            push(warnings, 'executor_runtime_missing', `报告缺少部分 variant 的 executor runtime 指纹: ${missing.join(', ')}。`, `Report is missing executor runtime fingerprints for variants: ${missing.join(', ')}.`);
+        }
+        for (const [variant, runtime] of Object.entries(runtimes)) {
+            pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', `variant ${variant} executor`, `Variant ${variant} executor`, runtime);
+        }
+        return;
+    }
+    if (!report.meta.executorRuntime) {
+        push(warnings, 'executor_runtime_missing', '报告缺少 executor runtime 指纹；无法审计 binary / SDK 版本，跨报告严格比较需谨慎。', 'Report is missing executor runtime fingerprint; binary / SDK versions cannot be audited for strict cross-report comparison.');
+        return;
+    }
+    if (report.meta.variants.length > 1) {
+        push(warnings, 'executor_runtimes_missing', '报告缺少 per-variant executor runtime 指纹；多 variant run 无法审计每个分组实际使用的 binary / SDK 版本。', 'Report is missing per-variant executor runtime fingerprints; multi-variant runs cannot audit each group’s actual binary / SDK version.');
+    }
+    pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', 'executor', 'Executor', report.meta.executorRuntime);
+}
+function countSampleHashMismatches(a, b) {
+    const ah = a.meta.sampleHashes;
+    const bh = b.meta.sampleHashes;
+    if (!ah || !bh)
+        return null;
+    let mismatched = 0;
+    let missing = 0;
+    let common = 0;
+    const ids = new Set([...Object.keys(ah), ...Object.keys(bh)]);
+    for (const id of ids) {
+        if (ah[id] == null || bh[id] == null) {
+            missing++;
+            continue;
+        }
+        common++;
+        if (ah[id] !== bh[id])
+            mismatched++;
+    }
+    return { mismatched, missing, common };
+}
+export function reportComparabilityWarnings(report) {
+    const warnings = [];
+    reportExecutorRuntimeWarnings(report, warnings);
+    if (hasJudge(report.meta) && !report.meta.judgePromptHash) {
+        push(warnings, 'judge_prompt_hash_missing', '报告缺少评委提示词指纹；无法确认 LLM 评分语义是否一致。', 'Report is missing judge prompt fingerprint; LLM scoring semantics cannot be verified.');
+    }
+    if (report.meta.judgeModel && !report.meta.judgeRuntime) {
+        push(warnings, 'judge_runtime_missing', '报告缺少评委 runtime 指纹；无法审计评委 executor 的 binary / SDK 版本。', 'Report is missing judge runtime fingerprint; judge executor binary / SDK versions cannot be audited.');
+    }
+    else if (report.meta.judgeRuntime) {
+        pushRuntimeUnverifiable(warnings, 'judge_runtime_unverifiable', '评委', 'Judge', report.meta.judgeRuntime);
+    }
+    const judgeModels = sorted(report.meta.judgeModels);
+    if (judgeModels.length >= 2) {
+        if (!report.meta.judgeRuntimes) {
+            push(warnings, 'judge_runtime_missing', '报告缺少多评委 ensemble 的 runtime 指纹；无法审计每个评委 executor 的 binary / SDK 版本。', 'Report is missing multi-judge ensemble runtime fingerprints; each judge executor binary / SDK version cannot be audited.');
+        }
+        else {
+            const missing = judgeModels.filter((model) => !report.meta.judgeRuntimes?.[model]);
+            if (missing.length > 0) {
+                push(warnings, 'judge_runtime_missing', `多评委 ensemble 缺少 runtime 指纹: ${missing.join(', ')}。`, `Multi-judge ensemble is missing runtime fingerprints: ${missing.join(', ')}.`);
+            }
+            for (const key of judgeModels) {
+                pushRuntimeUnverifiable(warnings, 'judge_runtime_unverifiable', `评委 ${key}`, `Judge ${key}`, report.meta.judgeRuntimes[key]);
+            }
+        }
+    }
+    if (!report.meta.sampleHashes) {
+        push(warnings, 'sample_hashes_missing', '报告缺少用例指纹；无法确认不同 run 是否测的是同一组用例。', 'Report is missing sample fingerprints; runs cannot be verified to use the same test cases.');
+    }
+    if (!report.meta.skillIsolation) {
+        push(warnings, 'skill_isolation_missing', '报告缺少 skill isolation 快照；baseline 是否被 skill 污染不可审计。', 'Report is missing skill-isolation snapshot; baseline contamination cannot be audited.');
+    }
+    return warnings;
+}
+export function crossReportComparabilityWarnings(before, after) {
+    const warnings = [];
+    const b = before.meta;
+    const a = after.meta;
+    if (b.model !== a.model) {
+        push(warnings, 'model_mismatch', `任务执行模型不同: ${b.model} → ${a.model}。`, `Task execution model changed: ${b.model} → ${a.model}.`);
+    }
+    if (b.executor !== a.executor) {
+        push(warnings, 'executor_mismatch', `executor 不同: ${b.executor} → ${a.executor}。`, `Executor changed: ${b.executor} → ${a.executor}.`);
+    }
+    const bExecutorRuntimeKeys = runtimeMapKeys(b);
+    const aExecutorRuntimeKeys = runtimeMapKeys(a);
+    const hasExecutorRuntimeMap = bExecutorRuntimeKeys.length > 0 || aExecutorRuntimeKeys.length > 0;
+    if (hasExecutorRuntimeMap) {
+        const keys = sorted([...new Set([...b.variants, ...a.variants, ...bExecutorRuntimeKeys, ...aExecutorRuntimeKeys])]);
+        const missing = keys.filter((key) => !b.executorRuntimes?.[key] || !a.executorRuntimes?.[key]);
+        if (missing.length > 0) {
+            push(warnings, 'executor_runtime_missing', `至少一份报告缺少 per-variant executor runtime 指纹: ${missing.join(', ')}。`, `At least one report is missing per-variant executor runtime fingerprints: ${missing.join(', ')}.`);
+        }
+        for (const key of keys) {
+            const beforeRuntime = b.executorRuntimes?.[key];
+            const afterRuntime = a.executorRuntimes?.[key];
+            if (beforeRuntime?.fingerprint && afterRuntime?.fingerprint && beforeRuntime.fingerprint !== afterRuntime.fingerprint) {
+                push(warnings, 'executor_runtime_mismatch', `variant ${key} executor runtime 指纹不同: ${runtimeLabel(beforeRuntime)} → ${runtimeLabel(afterRuntime)}。`, `Variant ${key} executor runtime fingerprint changed: ${runtimeLabel(beforeRuntime)} → ${runtimeLabel(afterRuntime)}.`);
+            }
+            pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', `before variant ${key} executor`, `Before variant ${key} executor`, beforeRuntime);
+            pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', `after variant ${key} executor`, `After variant ${key} executor`, afterRuntime);
+        }
+    }
+    else if (b.executorRuntime?.fingerprint && a.executorRuntime?.fingerprint) {
+        if (b.executorRuntime.fingerprint !== a.executorRuntime.fingerprint) {
+            push(warnings, 'executor_runtime_mismatch', `executor runtime 指纹不同: ${runtimeLabel(b.executorRuntime)} → ${runtimeLabel(a.executorRuntime)}。`, `Executor runtime fingerprint changed: ${runtimeLabel(b.executorRuntime)} → ${runtimeLabel(a.executorRuntime)}.`);
+        }
+    }
+    else {
+        push(warnings, 'executor_runtime_missing', '至少一份报告缺少 executor runtime 指纹；无法确认 binary / SDK 版本一致。', 'At least one report is missing executor runtime fingerprint; binary / SDK version parity cannot be verified.');
+    }
+    if (!hasExecutorRuntimeMap) {
+        pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', 'before executor', 'Before executor', b.executorRuntime);
+        pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', 'after executor', 'After executor', a.executorRuntime);
+    }
+    const bJudgeModels = sorted(b.judgeModels);
+    const aJudgeModels = sorted(a.judgeModels);
+    const ensembleInEither = bJudgeModels.length > 0 || aJudgeModels.length > 0;
+    if (!ensembleInEither && (b.judgeModel || a.judgeModel) && b.judgeModel !== a.judgeModel) {
+        push(warnings, 'judge_model_mismatch', `评委模型不同: ${b.judgeModel ?? 'none'} → ${a.judgeModel ?? 'none'}。`, `Judge model changed: ${b.judgeModel ?? 'none'} → ${a.judgeModel ?? 'none'}.`);
+    }
+    if (ensembleInEither && stableStringify(bJudgeModels) !== stableStringify(aJudgeModels)) {
+        push(warnings, 'judge_models_mismatch', `多评委 ensemble 配置不同: ${bJudgeModels.join(', ') || 'none'} → ${aJudgeModels.join(', ') || 'none'}。`, `Multi-judge ensemble changed: ${bJudgeModels.join(', ') || 'none'} → ${aJudgeModels.join(', ') || 'none'}.`);
+    }
+    if (effectiveJudgeRepeat(b) !== effectiveJudgeRepeat(a)) {
+        push(warnings, 'judge_repeat_mismatch', `每条用例评委评价次数不同: ${effectiveJudgeRepeat(b)} → ${effectiveJudgeRepeat(a)}。`, `Judge repeat count changed: ${effectiveJudgeRepeat(b)} → ${effectiveJudgeRepeat(a)}.`);
+    }
+    if (ensembleInEither) {
+        const keys = sorted([...new Set([...bJudgeModels, ...aJudgeModels])]);
+        const missing = keys.filter((key) => !b.judgeRuntimes?.[key] || !a.judgeRuntimes?.[key]);
+        if (missing.length > 0) {
+            push(warnings, 'judge_runtime_missing', `至少一份报告缺少多评委 runtime 指纹: ${missing.join(', ')}。`, `At least one report is missing multi-judge runtime fingerprints: ${missing.join(', ')}.`);
+        }
+        for (const key of keys) {
+            const beforeRuntime = b.judgeRuntimes?.[key];
+            const afterRuntime = a.judgeRuntimes?.[key];
+            if (beforeRuntime?.fingerprint && afterRuntime?.fingerprint && beforeRuntime.fingerprint !== afterRuntime.fingerprint) {
+                push(warnings, 'judge_runtime_mismatch', `评委 ${key} runtime 指纹不同: ${runtimeLabel(beforeRuntime)} → ${runtimeLabel(afterRuntime)}。`, `Judge ${key} runtime fingerprint changed: ${runtimeLabel(beforeRuntime)} → ${runtimeLabel(afterRuntime)}.`);
+            }
+            pushRuntimeUnverifiable(warnings, 'judge_runtime_unverifiable', `before 评委 ${key}`, `Before judge ${key}`, beforeRuntime);
+            pushRuntimeUnverifiable(warnings, 'judge_runtime_unverifiable', `after 评委 ${key}`, `After judge ${key}`, afterRuntime);
+        }
+    }
+    else if (b.judgeRuntime?.fingerprint && a.judgeRuntime?.fingerprint) {
+        if (b.judgeRuntime.fingerprint !== a.judgeRuntime.fingerprint) {
+            push(warnings, 'judge_runtime_mismatch', `评委 runtime 指纹不同: ${runtimeLabel(b.judgeRuntime)} → ${runtimeLabel(a.judgeRuntime)}。`, `Judge runtime fingerprint changed: ${runtimeLabel(b.judgeRuntime)} → ${runtimeLabel(a.judgeRuntime)}.`);
+        }
+    }
+    else if (b.judgeModel || a.judgeModel) {
+        push(warnings, 'judge_runtime_missing', '至少一份报告缺少评委 runtime 指纹；无法确认评委 binary / SDK 版本一致。', 'At least one report is missing judge runtime fingerprint; judge binary / SDK version parity cannot be verified.');
+    }
+    if (!ensembleInEither) {
+        pushRuntimeUnverifiable(warnings, 'judge_runtime_unverifiable', 'before 评委', 'Before judge', b.judgeRuntime);
+        pushRuntimeUnverifiable(warnings, 'judge_runtime_unverifiable', 'after 评委', 'After judge', a.judgeRuntime);
+    }
+    if ((hasJudge(b) || hasJudge(a)) && (!b.judgePromptHash || !a.judgePromptHash)) {
+        push(warnings, 'judge_prompt_hash_missing', `至少一份报告缺少评委提示词指纹: ${b.judgePromptHash ?? 'missing'} → ${a.judgePromptHash ?? 'missing'}。`, `At least one report is missing judge prompt hash: ${b.judgePromptHash ?? 'missing'} → ${a.judgePromptHash ?? 'missing'}.`);
+    }
+    else if ((b.judgePromptHash || a.judgePromptHash) && b.judgePromptHash !== a.judgePromptHash) {
+        push(warnings, 'judge_prompt_hash_mismatch', `评委提示词指纹不同: ${b.judgePromptHash ?? 'missing'} → ${a.judgePromptHash ?? 'missing'}。`, `Judge prompt hash changed: ${b.judgePromptHash ?? 'missing'} → ${a.judgePromptHash ?? 'missing'}.`);
+    }
+    if ((b.evaluationFramework || a.evaluationFramework) && b.evaluationFramework !== a.evaluationFramework) {
+        push(warnings, 'evaluation_framework_mismatch', `统计框架不同: ${b.evaluationFramework ?? 'legacy'} → ${a.evaluationFramework ?? 'legacy'}。`, `Evaluation framework changed: ${b.evaluationFramework ?? 'legacy'} → ${a.evaluationFramework ?? 'legacy'}.`);
+    }
+    if (stableStringify(b.skillIsolation ?? null) !== stableStringify(a.skillIsolation ?? null)) {
+        push(warnings, 'skill_isolation_mismatch', 'skill isolation 快照不同；baseline 污染边界不一致，严格比较需谨慎。', 'Skill-isolation snapshots differ; baseline contamination boundaries are not equivalent.');
+    }
+    const sampleHashDiff = countSampleHashMismatches(before, after);
+    if (!sampleHashDiff) {
+        push(warnings, 'sample_hashes_missing', '至少一份报告缺少用例指纹；无法确认两次 run 使用同一组用例。', 'At least one report is missing sample fingerprints; test-case parity cannot be verified.');
+    }
+    else if (sampleHashDiff.mismatched > 0 || sampleHashDiff.missing > 0) {
+        push(warnings, 'sample_hashes_mismatch', `用例指纹不一致: ${sampleHashDiff.mismatched} 条内容变化，${sampleHashDiff.missing} 条只出现在其中一份报告。`, `Sample fingerprints differ: ${sampleHashDiff.mismatched} content mismatches, ${sampleHashDiff.missing} samples only appear in one report.`);
+    }
+    return warnings;
+}
+export function formatComparabilityWarnings(warnings, lang = 'zh') {
+    if (warnings.length === 0)
+        return '';
+    const header = lang === 'zh'
+        ? '可比性提示: 以下差异会影响 strict comparison / construct validity'
+        : 'Comparability warnings: the following differences affect strict comparison / construct validity';
+    return [header, ...warnings.map((warning) => `  - ${lang === 'zh' ? warning.zh : warning.en}`)].join('\n');
+}
+//# sourceMappingURL=comparability.js.map

package/dist/src/eval-core/comparability.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparability.js","sourceRoot":"","sources":["../../../src/eval-core/comparability.ts"],"names":[],"mappings":"AASA,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9E,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IAClF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAgC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrE,OAAO,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,eAAe,CAAE,KAAiC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;AACpI,CAAC;AAED,SAAS,YAAY,CAAC,OAAsD;IAC1E,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,EAAE,EAAE,MAAM,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACnF,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;QAAE,IAAI,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3E,IAAI,OAAO,CAAC,GAAG,EAAE,OAAO;QAAE,IAAI,CAAC,IAAI,CAAC,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAClE,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,IAAI,CAAC,QAAgC,EAAE,IAAY,EAAE,EAAU,EAAE,EAAU;IAClF,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,MAAM,CAAC,MAA4B;IAC1C,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,IAAgB;IAChC,OAAO,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AACvF,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAgB;IAC5C,OAAO,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,cAAc,CAAC,GAAuC,EAAE,KAAa;IAC5E,IAAI,CAAC,GAAG;QAAE,OAAO,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;IACtC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,GAAG,CAAC,KAAK;QAAE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,eAAe,CAAC,CAAC;IACrD,IAAI,CAAC,GAAG,CAAC,OAAO;QAAE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,kBAAkB,CAAC,CAAC;IAC3D,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,0BAA0B,CAAC,OAAmC;IACrE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK;QAAE,OAAO,OAAO,CAAC;IAE3C,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC9B,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/B,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,MAAM,CAAC,KAAK;QAAE,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAEtD,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,IAAI;YAAE,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC9D,CAAC;SAAM,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO;YAAE,OAAO,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;QAChG,IAAI,MAAM,CAAC,OAAO;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,yBAAyB,CAAC,CAAC,CAAC;IACxI,CAAC;SAAM,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,uBAAuB,CAC9B,QAAgC,EAChC,IAAY,EACZ,SAAiB,EACjB,SAAiB,EACjB,OAAsD;IAEtD,IAAI,CAAC,OAAO;QAAE,OAAO;IACrB,MAAM,OAAO,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACjC,IAAI,CACF,QAAQ,EACR,IAAI,EACJ,GAAG,SAAS,sBAAsB,YAAY,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAClF,GAAG,SAAS,iDAAiD,YAAY,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAC9G,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,IAAgB;IACtC,OAAO,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,6BAA6B,CAAC,MAAc,EAAE,QAAgC;IACrF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC;IAC9C,IAAI,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CACF,QAAQ,EACR,0BAA0B,EAC1B,yCAAyC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAC9D,iEAAiE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACvF,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1D,uBAAuB,CAAC,QAAQ,EAAE,+BAA+B,EAAE,WAAW,OAAO,WAAW,EAAE,WAAW,OAAO,WAAW,EAAE,OAAO,CAAC,CAAC;QAC5I,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;QACjC,IAAI,CACF,QAAQ,EACR,0BAA0B,EAC1B,2DAA2D,EAC3D,6HAA6H,CAC9H,CAAC;QACF,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,IAAI,CACF,QAAQ,EACR,2BAA2B,EAC3B,mFAAmF,EACnF,wIAAwI,CACzI,CAAC;IACJ,CAAC;IACD,uBAAuB,CAAC,QAAQ,EAAE,+BAA+B,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;AAC1H,CAAC;AAED,SAAS,yBAAyB,CAAC,CAAS,EAAE,CAAS;IACrD,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;IAC/B,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;IAC/B,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IAC5B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9D,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,IAAI,EAAE,CAAC,EAAE,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;YACrC,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,MAAM,EAAE,CAAC;QACT,IAAI,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;YAAE,UAAU,EAAE,CAAC;IACtC,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,MAAc;IACxD,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,6BAA6B,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAChD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;QAC1D,IAAI,CACF,QAAQ,EACR,2BAA2B,EAC3B,gCAAgC,EAChC,uFAAuF,CACxF,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QACxD,IAAI,CACF,QAAQ,EACR,uBAAuB,EACvB,sDAAsD,EACtD,sGAAsG,CACvG,CAAC;IACJ,CAAC;SAAM,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QACpC,uBAAuB,CAAC,QAAQ,EAAE,4BAA4B,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3G,CAAC;IACD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpD,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YAC/B,IAAI,CACF,QAAQ,EACR,uBAAuB,EACvB,oEAAoE,EACpE,0HAA0H,CAC3H,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YACnF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,IAAI,CACF,QAAQ,EACR,uBAAuB,EACvB,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EACpD,yDAAyD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC/E,CAAC;YACJ,CAAC;YACD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;gBAC9B,uBAAuB,CAAC,QAAQ,EAAE,4BAA4B,EAAE,MAAM,GAAG,EAAE,EAAE,SAAS,GAAG,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/H,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QAC9B,IAAI,CACF,QAAQ,EACR,uBAAuB,EACvB,iCAAiC,EACjC,4FAA4F,CAC7F,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;QAChC,IAAI,CACF,QAAQ,EACR,yBAAyB,EACzB,oDAAoD,EACpD,uFAAuF,CACxF,CAAC;IACJ,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,gCAAgC,CAAC,MAAc,EAAE,KAAa;IAC5E,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC;IACtB,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC;IAErB,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,QAAQ,EAAE,gBAAgB,EAAE,aAAa,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,KAAK,GAAG,EAAE,iCAAiC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;IACnI,CAAC;IACD,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,QAAQ,EAAE,mBAAmB,EAAE,gBAAgB,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,QAAQ,GAAG,EAAE,qBAAqB,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;IACzI,CAAC;IACD,MAAM,oBAAoB,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,oBAAoB,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,qBAAqB,GAAG,oBAAoB,CAAC,MAAM,GAAG,CAAC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,CAAC;IACjG,IAAI,qBAAqB,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,oBAAoB,EAAE,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC;QACpH,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/F,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CACF,QAAQ,EACR,0BAA0B,EAC1B,6CAA6C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAClE,6EAA6E,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACnG,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,aAAa,GAAG,CAAC,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,CAAC,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,CAAC;YAC/C,IAAI,aAAa,EAAE,WAAW,IAAI,YAAY,EAAE,WAAW,IAAI,aAAa,CAAC,WAAW,KAAK,YAAY,CAAC,WAAW,EAAE,CAAC;gBACtH,IAAI,CACF,QAAQ,EACR,2BAA2B,EAC3B,WAAW,GAAG,2BAA2B,YAAY,CAAC,aAAa,CAAC,MAAM,YAAY,CAAC,YAAY,CAAC,GAAG,EACvG,WAAW,GAAG,0CAA0C,YAAY,CAAC,aAAa,CAAC,MAAM,YAAY,CAAC,YAAY,CAAC,GAAG,CACvH,CAAC;YACJ,CAAC;YACD,uBAAuB,CAAC,QAAQ,EAAE,+BAA+B,EAAE,kBAAkB,GAAG,WAAW,EAAE,kBAAkB,GAAG,WAAW,EAAE,aAAa,CAAC,CAAC;YACtJ,uBAAuB,CAAC,QAAQ,EAAE,+BAA+B,EAAE,iBAAiB,GAAG,WAAW,EAAE,iBAAiB,GAAG,WAAW,EAAE,YAAY,CAAC,CAAC;QACrJ,CAAC;IACH,CAAC;SAAM,IAAI,CAAC,CAAC,eAAe,EAAE,WAAW,IAAI,CAAC,CAAC,eAAe,EAAE,WAAW,EAAE,CAAC;QAC5E,IAAI,CAAC,CAAC,eAAe,CAAC,WAAW,KAAK,CAAC,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YACpE,IAAI,CACF,QAAQ,EACR,2BAA2B,EAC3B,0BAA0B,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,EACjG,yCAAyC,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CACjH,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,CACF,QAAQ,EACR,0BAA0B,EAC1B,sDAAsD,EACtD,8GAA8G,CAC/G,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC3B,uBAAuB,CAAC,QAAQ,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;QAC5H,uBAAuB,CAAC,QAAQ,EAAE,+BAA+B,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;IAC5H,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5E,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;QACzF,IAAI,CAAC,QAAQ,EAAE,sBAAsB,EAAE,WAAW,CAAC,CAAC,UAAU,IAAI,MAAM,MAAM,CAAC,CAAC,UAAU,IAAI,MAAM,GAAG,EAAE,wBAAwB,CAAC,CAAC,UAAU,IAAI,MAAM,MAAM,CAAC,CAAC,UAAU,IAAI,MAAM,GAAG,CAAC,CAAC;IAC1L,CAAC;IACD,IAAI,gBAAgB,IAAI,eAAe,CAAC,YAAY,CAAC,KAAK,eAAe,CAAC,YAAY,CAAC,EAAE,CAAC;QACxF,IAAI,CACF,QAAQ,EACR,uBAAuB,EACvB,sBAAsB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,GAAG,EACjG,iCAAiC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,GAAG,CAC7G,CAAC;IACJ,CAAC;IACD,IAAI,oBAAoB,CAAC,CAAC,CAAC,KAAK,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,IAAI,CACF,QAAQ,EACR,uBAAuB,EACvB,iBAAiB,oBAAoB,CAAC,CAAC,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC,GAAG,EACxE,+BAA+B,oBAAoB,CAAC,CAAC,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC,GAAG,CACvF,CAAC;IACJ,CAAC;IAED,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACzF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CACF,QAAQ,EACR,uBAAuB,EACvB,2BAA2B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAChD,oEAAoE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC1F,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,aAAa,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC;YAC7C,MAAM,YAAY,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,aAAa,EAAE,WAAW,IAAI,YAAY,EAAE,WAAW,IAAI,aAAa,CAAC,WAAW,KAAK,YAAY,CAAC,WAAW,EAAE,CAAC;gBACtH,IAAI,CACF,QAAQ,EACR,wBAAwB,EACxB,MAAM,GAAG,kBAAkB,YAAY,CAAC,aAAa,CAAC,MAAM,YAAY,CAAC,YAAY,CAAC,GAAG,EACzF,SAAS,GAAG,iCAAiC,YAAY,CAAC,aAAa,CAAC,MAAM,YAAY,CAAC,YAAY,CAAC,GAAG,CAC5G,CAAC;YACJ,CAAC;YACD,uBAAuB,CAAC,QAAQ,EAAE,4BAA4B,EAAE,aAAa,GAAG,EAAE,EAAE,gBAAgB,GAAG,EAAE,EAAE,aAAa,CAAC,CAAC;YAC1H,uBAAuB,CAAC,QAAQ,EAAE,4BAA4B,EAAE,YAAY,GAAG,EAAE,EAAE,eAAe,GAAG,EAAE,EAAE,YAAY,CAAC,CAAC;QACzH,CAAC;IACH,CAAC;SAAM,IAAI,CAAC,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,YAAY,EAAE,WAAW,EAAE,CAAC;QACtE,IAAI,CAAC,CAAC,YAAY,CAAC,WAAW,KAAK,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;YAC9D,IAAI,CACF,QAAQ,EACR,wBAAwB,EACxB,oBAAoB,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,EACrF,sCAAsC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CACxG,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;QACxC,IAAI,CACF,QAAQ,EACR,uBAAuB,EACvB,iDAAiD,EACjD,iHAAiH,CAClH,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,uBAAuB,CAAC,QAAQ,EAAE,4BAA4B,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC;QAC7G,uBAAuB,CAAC,QAAQ,EAAE,4BAA4B,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC;IAC7G,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC;QAC/E,IAAI,CAAC,QAAQ,EAAE,2BAA2B,EAAE,oBAAoB,CAAC,CAAC,eAAe,IAAI,SAAS,MAAM,CAAC,CAAC,eAAe,IAAI,SAAS,GAAG,EAAE,qDAAqD,CAAC,CAAC,eAAe,IAAI,SAAS,MAAM,CAAC,CAAC,eAAe,IAAI,SAAS,GAAG,CAAC,CAAC;IACrQ,CAAC;SAAM,IAAI,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,eAAe,EAAE,CAAC;QAC/F,IAAI,CAAC,QAAQ,EAAE,4BAA4B,EAAE,cAAc,CAAC,CAAC,eAAe,IAAI,SAAS,MAAM,CAAC,CAAC,eAAe,IAAI,SAAS,GAAG,EAAE,8BAA8B,CAAC,CAAC,eAAe,IAAI,SAAS,MAAM,CAAC,CAAC,eAAe,IAAI,SAAS,GAAG,CAAC,CAAC;IACzO,CAAC;IACD,IAAI,CAAC,CAAC,CAAC,mBAAmB,IAAI,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,mBAAmB,KAAK,CAAC,CAAC,mBAAmB,EAAE,CAAC;QACxG,IAAI,CAAC,QAAQ,EAAE,+BAA+B,EAAE,WAAW,CAAC,CAAC,mBAAmB,IAAI,QAAQ,MAAM,CAAC,CAAC,mBAAmB,IAAI,QAAQ,GAAG,EAAE,iCAAiC,CAAC,CAAC,mBAAmB,IAAI,QAAQ,MAAM,CAAC,CAAC,mBAAmB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACxP,CAAC;IACD,IAAI,eAAe,CAAC,CAAC,CAAC,cAAc,IAAI,IAAI,CAAC,KAAK,eAAe,CAAC,CAAC,CAAC,cAAc,IAAI,IAAI,CAAC,EAAE,CAAC;QAC5F,IAAI,CACF,QAAQ,EACR,0BAA0B,EAC1B,gDAAgD,EAChD,yFAAyF,CAC1F,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,yBAAyB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAChE,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,IAAI,CACF,QAAQ,EACR,uBAAuB,EACvB,kCAAkC,EAClC,0FAA0F,CAC3F,CAAC;IACJ,CAAC;SAAM,IAAI,cAAc,CAAC,UAAU,GAAG,CAAC,IAAI,cAAc,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;QACvE,IAAI,CACF,QAAQ,EACR,wBAAwB,EACxB,YAAY,cAAc,CAAC,UAAU,UAAU,cAAc,CAAC,OAAO,eAAe,EACpF,+BAA+B,cAAc,CAAC,UAAU,wBAAwB,cAAc,CAAC,OAAO,qCAAqC,CAC5I,CAAC;IACJ,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,QAAgC,EAAE,OAAa,IAAI;IAC7F,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,KAAK,IAAI;QAC1B,CAAC,CAAC,uDAAuD;QACzD,CAAC,CAAC,iGAAiG,CAAC;IACtG,OAAO,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7G,CAAC"}

package/dist/src/eval-core/dependency-checker.d.ts

@@ -33,7 +33,7 @@ export declare function extractDependencies(skillContents: string[], samples: Sa
  *   - defaultCwd         (其他)
  *
  * 必须用 per-artifact 分桶,否则两个 skill 各自的 assets/foo.md 在单一 cwd 下
- * 互相错位(WCC 评测里 41 个 false-positive 的根因)。
+ * 互相错位,导致大量 false-positive missing。
  */
 export declare function extractFilesByBase(artifacts: Artifact[], defaultCwd: string): Map<string, Set<string>>;
 /**

package/dist/src/eval-core/dependency-checker.js

@@ -118,7 +118,7 @@ export function extractDependencies(skillContents, samples) {
  *   - defaultCwd         (其他)
  *
  * 必须用 per-artifact 分桶,否则两个 skill 各自的 assets/foo.md 在单一 cwd 下
- * 互相错位(WCC 评测里 41 个 false-positive 的根因)。
+ * 互相错位,导致大量 false-positive missing。
  */
 export function extractFilesByBase(artifacts, defaultCwd) {
     const map = new Map();

package/dist/src/eval-core/evaluation-execution.d.ts

@@ -2,6 +2,9 @@ import type { ExecutorFn, ProgressCallback, Task, VariantResult } from '../types
 export interface ExecuteTasksOptions {
     tasks: Task[];
     executor: ExecutorFn;
+    /** Executor name (e.g. 'claude' / 'codex' / 'openai-api'). Used in cache key
+     *  to prevent cross-executor pollution when same model name is used. */
+    executorName?: string;
     judgeExecutor: ExecutorFn;
     model: string;
     judgeModel: string;
@@ -24,13 +27,13 @@ export interface ExecuteTasksOptions {
     judgeExecutors?: Record<string, ExecutorFn>;
     /** v0.21 length-debias toggle. Default true; CLI's --no-debias-length sets it false. */
     lengthDebias?: boolean;
-    /** v0.22 — hard budget caps. When totalUSD is exceeded mid-run, remaining
+    /** hard budget caps. When totalUSD is exceeded mid-run, remaining
      *  tasks are skipped and the partial result set is returned with
      *  budgetExhausted: true. Per-sample caps don't abort but mark offending
      *  tasks as failed. */
     budget?: import('../types/index.js').EvalBudget;
 }
-export declare function executeTasks({ tasks, executor, judgeExecutor, model, judgeModel, noJudge, samplesPath, concurrency, timeoutMs, noCache, verbose, onProgress, retry, existingResults, judgeRepeat, judgeModels, judgeExecutors, lengthDebias, budget, }: ExecuteTasksOptions): Promise<{
+export declare function executeTasks({ tasks, executor, executorName, judgeExecutor, model, judgeModel, noJudge, samplesPath, concurrency, timeoutMs, noCache, verbose, onProgress, retry, existingResults, judgeRepeat, judgeModels, judgeExecutors, lengthDebias, budget, }: ExecuteTasksOptions): Promise<{
     results: Record<string, Record<string, VariantResult>>;
     totalCostUSD: number;
     skipped: number;

package/dist/src/eval-core/evaluation-execution.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation-execution.d.ts","sourceRoot":"","sources":["../../../src/eval-core/evaluation-execution.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAEV,UAAU,EAGV,gBAAgB,EAChB,IAAI,EACJ,aAAa,EACd,MAAM,mBAAmB,CAAC;AAE3B,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,QAAQ,EAAE,UAAU,CAAC;IACrB,aAAa,EAAE,UAAU,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAChE,iFAAiF;IACjF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yEAAyE;IACzE,WAAW,CAAC,EAAE,OAAO,mBAAmB,EAAE,WAAW,EAAE,CAAC;IACxD,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC5C,wFAAwF;IACxF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;2BAGuB;IACvB,MAAM,CAAC,EAAE,OAAO,mBAAmB,EAAE,UAAU,CAAC;CACjD;AAmCD,wBAAsB,YAAY,CAAC,EACjC,KAAK,EACL,QAAQ,EACR,aAAa,EACb,KAAK,EACL,UAAU,EACV,OAAO,EACP,WAAW,EACX,WAAW,EACX,SAAS,EACT,OAAO,EACP,OAAO,EACP,UAAU,EACV,KAAS,EACT,eAAe,EACf,WAAe,EACf,WAAW,EACX,cAAc,EACd,YAAmB,EACnB,MAAM,GACP,EAAE,mBAAmB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,OAAO,CAAA;CAAE,CAAC,CAkM5J;AAED,wBAAsB,SAAS,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAW7G"}
+{"version":3,"file":"evaluation-execution.d.ts","sourceRoot":"","sources":["../../../src/eval-core/evaluation-execution.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAEV,UAAU,EAGV,gBAAgB,EAChB,IAAI,EACJ,aAAa,EACd,MAAM,mBAAmB,CAAC;AAE3B,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,QAAQ,EAAE,UAAU,CAAC;IACrB;4EACwE;IACxE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,UAAU,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAChE,iFAAiF;IACjF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yEAAyE;IACzE,WAAW,CAAC,EAAE,OAAO,mBAAmB,EAAE,WAAW,EAAE,CAAC;IACxD,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC5C,wFAAwF;IACxF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;2BAGuB;IACvB,MAAM,CAAC,EAAE,OAAO,mBAAmB,EAAE,UAAU,CAAC;CACjD;AAmCD,wBAAsB,YAAY,CAAC,EACjC,KAAK,EACL,QAAQ,EACR,YAAY,EACZ,aAAa,EACb,KAAK,EACL,UAAU,EACV,OAAO,EACP,WAAW,EACX,WAAW,EACX,SAAS,EACT,OAAO,EACP,OAAO,EACP,UAAU,EACV,KAAS,EACT,eAAe,EACf,WAAe,EACf,WAAW,EACX,cAAc,EACd,YAAmB,EACnB,MAAM,GACP,EAAE,mBAAmB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,OAAO,CAAA;CAAE,CAAC,CA2M5J;AAED,wBAAsB,SAAS,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAW7G"}

package/dist/src/eval-core/evaluation-execution.js

@@ -3,6 +3,7 @@ import { buildVariantResult } from './schema.js';
 import { grade } from '../grading/index.js';
 import { checkFacts } from './fact-checker.js';
 import { resolveExecutionStrategy } from './execution-strategy.js';
+import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
 import { resolve, join, dirname } from 'node:path';
 import { homedir } from 'node:os';
 async function runWithConcurrency(tasks, concurrency, fn) {
@@ -35,7 +36,7 @@ function makeErrorResult(error) {
 function sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
-export async function executeTasks({ tasks, executor, judgeExecutor, model, judgeModel, noJudge, samplesPath, concurrency, timeoutMs, noCache, verbose, onProgress, retry = 0, existingResults, judgeRepeat = 1, judgeModels, judgeExecutors, lengthDebias = true, budget, }) {
+export async function executeTasks({ tasks, executor, executorName, judgeExecutor, model, judgeModel, noJudge, samplesPath, concurrency, timeoutMs, noCache, verbose, onProgress, retry = 0, existingResults, judgeRepeat = 1, judgeModels, judgeExecutors, lengthDebias = true, budget, }) {
     const results = {};
     let started = 0;
     let completed = 0;
@@ -59,7 +60,7 @@ export async function executeTasks({ tasks, executor, judgeExecutor, model, judg
             onProgress?.({ phase: 'done', completed, total: tasks.length, sample_id: task.sample_id, variant: task.variant, skipped: true });
             return;
         }
-        // v0.22 budget abort: if a previous task tripped the total-USD cap, skip
+        //  budget abort: if a previous task tripped the total-USD cap, skip
         // remaining tasks. The partial report is still persisted so the user sees
         // what completed before the run stopped.
         if (budgetExhausted) {
@@ -74,10 +75,15 @@ export async function executeTasks({ tasks, executor, judgeExecutor, model, judg
         const total = tasks.length;
         onProgress?.({ phase: 'start', completed: idx, total, sample_id: task.sample_id, variant: task.variant });
         const executionPlan = resolveExecutionStrategy(task, model, timeoutMs, verbose);
+        const effectiveExecutorName = executorName || 'claude';
+        const executorRuntime = getExecutorRuntimeFingerprint(effectiveExecutorName, model, {
+            skillDir: executionPlan.input.skillDir,
+        });
         let execResult;
-        // v0.22 — include allowedSkills in cache key so isolation-on / isolation-off runs
-        // don't share cache entries (would otherwise replay contaminated baseline results).
-        const key = cacheKey(model, executionPlan.cacheSystem, executionPlan.input.prompt, executionPlan.input.cwd, task.artifact.allowedSkills);
+        // include allowedSkills in cache key so isolation-on / isolation-off runs
+        // don't share cache entries, and include runtime fingerprint so a binary/SDK bump
+        // cannot replay old-runtime outputs under new-runtime report metadata.
+        const key = cacheKey(model, executionPlan.cacheSystem, executionPlan.input.prompt, executionPlan.input.cwd, task.artifact.allowedSkills, effectiveExecutorName, executorRuntime.fingerprint);
         const cached = cache?.get(key);
         const execStart = Date.now();
         if (cached) {
@@ -167,20 +173,6 @@ export async function executeTasks({ tasks, executor, judgeExecutor, model, judg
                     totalCostUSD += gradeResult.judgeCostUSD;
             }
         }
-        completed++;
-        onProgress?.({
-            phase: 'done',
-            strategy: executionPlan.strategy,
-            completed,
-            total,
-            sample_id: task.sample_id,
-            variant: task.variant,
-            durationMs: execResult.durationMs,
-            inputTokens: execResult.inputTokens,
-            outputTokens: execResult.outputTokens,
-            costUSD: execResult.costUSD,
-            score: gradeResult?.compositeScore,
-        });
         let factCheck;
         if (execResult.ok && execResult.output && task.cwd) {
             factCheck = checkFacts(execResult.output, resolve(task.cwd));
@@ -188,7 +180,7 @@ export async function executeTasks({ tasks, executor, judgeExecutor, model, judg
         if (!results[task.sample_id])
             results[task.sample_id] = {};
         const variantResult = buildVariantResult(execResult, gradeResult, { execMs, gradeMs, factCheck });
-        // v0.22 per-sample budget enforcement. If a sample's cost or latency
+        //  per-sample budget enforcement. If a sample's cost or latency
         // exceeds the per-sample cap, the result is kept (so the user can see
         // what happened) but flagged as a budget overrun. The run continues.
         if (budget?.perSampleUSD != null && variantResult.costUSD > budget.perSampleUSD) {
@@ -200,7 +192,23 @@ export async function executeTasks({ tasks, executor, judgeExecutor, model, judg
             variantResult.error = `budget overrun: per-sample latency ${execMs + (gradeMs ?? 0)}ms > cap ${budget.perSampleMs}ms`;
         }
         results[task.sample_id][task.variant] = variantResult;
-        // v0.22 total-USD budget enforcement. Once the global cap is exceeded,
+        completed++;
+        onProgress?.({
+            phase: 'done',
+            strategy: executionPlan.strategy,
+            completed,
+            total,
+            sample_id: task.sample_id,
+            variant: task.variant,
+            durationMs: variantResult.durationMs,
+            inputTokens: variantResult.inputTokens,
+            outputTokens: variantResult.outputTokens,
+            costUSD: variantResult.execCostUSD,
+            score: gradeResult?.compositeScore,
+            ok: variantResult.ok,
+            error: variantResult.error,
+        });
+        //  total-USD budget enforcement. Once the global cap is exceeded,
         // flip the abort flag so subsequent tasks short-circuit. The current task
         // is kept (already paid for it).
         if (budget?.totalUSD != null && totalCostUSD > budget.totalUSD && !budgetExhausted) {

package/dist/src/eval-core/evaluation-execution.js.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation-execution.js","sourceRoot":"","sources":["../../../src/eval-core/evaluation-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AA2ClC,KAAK,UAAU,kBAAkB,CAAI,KAAU,EAAE,WAAmB,EAAE,EAA8B;IAClG,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,UAAU,MAAM;QACnB,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,KAAK,EAAE,CAAC;YAClB,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO;QACL,EAAE,EAAE,KAAK;QACT,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,CAAC;QAChB,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,eAAe,EAAE,CAAC;QAClB,mBAAmB,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;QACV,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE,CAAC;QACX,KAAK,EAAE,OAAO;KACf,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,KAAK,EACL,QAAQ,EACR,aAAa,EACb,KAAK,EACL,UAAU,EACV,OAAO,EACP,WAAW,EACX,WAAW,EACX,SAAS,EACT,OAAO,EACP,OAAO,EACP,UAAU,EACV,KAAK,GAAG,CAAC,EACT,eAAe,EACf,WAAW,GAAG,CAAC,EACf,WAAW,EACX,cAAc,EACd,YAAY,GAAG,IAAI,EACnB,MAAM,GACc;IACpB,MAAM,OAAO,GAAkD,EAAE,CAAC;IAClE,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,eAAe,GAAG,KAAK,CAAC;IAE5B,4CAA4C;IAC5C,IAAI,eAAe,EAAE,CAAC;QACpB,KAAK,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;YACnE,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QACtC,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAyB,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAE3E,KAAK,UAAU,WAAW,CAAC,IAAU;QACnC,sDAAsD;QACtD,IAAI,eAAe,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC;YAC1D,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACjI,OAAO;QACT,CAAC;QAED,yEAAyE;QACzE,0EAA0E;QAC1E,yCAAyC;QACzC,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACjI,OAAO;QACT,CAAC;QAED,OAAO,EAAE,CAAC;QACV,MAAM,GAAG,GAAG,OAAO,CAAC;QACpB,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAE1G,MAAM,aAAa,GAAG,wBAAwB,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAEhF,IAAI,UAAsB,CAAC;QAC3B,kFAAkF;QAClF,oFAAoF;QACpF,MAAM,GAAG,GAAG,QAAQ,CAClB,KAAK,EACL,aAAa,CAAC,WAAW,EACzB,aAAa,CAAC,KAAK,CAAC,MAAM,EAC1B,aAAa,CAAC,KAAK,CAAC,GAAG,EACvB,IAAI,CAAC,QAAQ,CAAC,aAAa,CAC5B,CAAC;QACF,MAAM,MAAM,GAAG,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,MAAM,EAAE,CAAC;YACX,UAAU,GAAG,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,gCAAgC;YAChC,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAC3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;gBACxD,IAAI,CAAC;oBACH,UAAU,GAAG,MAAM,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;gBACnD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,UAAU,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;gBACpC,CAAC;gBACD,IAAI,UAAW,CAAC,EAAE,IAAI,OAAO,KAAK,WAAW;oBAAE,MAAM;gBACrD,mCAAmC;gBACnC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC7D,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;gBAChI,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;YACzB,CAAC;YACD,IAAI,KAAK,IAAI,UAAW,CAAC,EAAE;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,UAAW,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACtC,YAAY,IAAI,UAAW,CAAC,OAAO,CAAC;QAEpC,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;YAC1B,UAAU,CAAC;gBACT,KAAK,EAAE,WAAW;gBAClB,QAAQ,EAAE,aAAa,CAAC,QAAQ;gBAChC,SAAS,EAAE,GAAG;gBACd,KAAK;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,UAAU,EAAE,UAAW,CAAC,UAAU;gBAClC,WAAW,EAAE,UAAW,CAAC,WAAW;gBACpC,YAAY,EAAE,UAAW,CAAC,YAAY;gBACtC,OAAO,EAAE,UAAW,CAAC,OAAO;gBAC5B,aAAa,EAAE,UAAW,CAAC,MAAM,CAAC,CAAC,CAAC,UAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;aAC5E,CAAC,CAAC;QACL,CAAC;QAED,IAAI,WAAW,GAAuB,IAAI,CAAC;QAC3C,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,UAAW,CAAC,EAAE,EAAE,CAAC;YACnB,MAAM,kBAAkB,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;YAC9H,IAAI,kBAAkB,EAAE,CAAC;gBACvB,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;oBAC1B,UAAU,CAAC;wBACT,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,aAAa,CAAC,QAAQ;wBAChC,SAAS,EAAE,GAAG;wBACd,KAAK;wBACL,SAAS,EAAE,IAAI,CAAC,SAAS;wBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;qBACtB,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC9B,IAAI,CAAC;oBACH,WAAW,GAAG,MAAM,KAAK,CAAC;wBACxB,MAAM,EAAE,UAAW,CAAC,MAAO;wBAC3B,MAAM,EAAE,IAAI,CAAC,OAAO;wBACpB,QAAQ,EAAE,aAAa;wBACvB,UAAU;wBACV,aAAa,EAAE,CAAC,OAAO;wBACvB,WAAW,EAAE;4BACX,OAAO,EAAE,UAAW,CAAC,OAAO;4BAC5B,UAAU,EAAE,UAAW,CAAC,UAAU;4BAClC,QAAQ,EAAE,UAAW,CAAC,QAAQ;4BAC9B,SAAS,EAAE,UAAW,CAAC,SAAS;4BAChC,KAAK,EAAE,UAAW,CAAC,KAAK;yBACzB;wBACD,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;wBACzC,WAAW;wBACX,WAAW;wBACX,cAAc;wBACd,YAAY;qBACb,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,WAAW,GAAG,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;oBACpC,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC7D,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,EAAE,EAAE,CAAC,CAAC;gBACnI,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;gBAClC,IAAI,WAAW,CAAC,YAAY;oBAAE,YAAY,IAAI,WAAW,CAAC,YAAY,CAAC;YACzE,CAAC;QACH,CAAC;QAED,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,aAAa,CAAC,QAAQ;YAChC,SAAS;YACT,KAAK;YACL,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,UAAW,CAAC,UAAU;YAClC,WAAW,EAAE,UAAW,CAAC,WAAW;YACpC,YAAY,EAAE,UAAW,CAAC,YAAY;YACtC,OAAO,EAAE,UAAW,CAAC,OAAO;YAC5B,KAAK,EAAE,WAAW,EAAE,cAAc;SACnC,CAAC,CAAC;QAEH,IAAI,SAAsC,CAAC;QAC3C,IAAI,UAAW,CAAC,EAAE,IAAI,UAAW,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACrD,SAAS,GAAG,UAAU,CAAC,UAAW,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;QAC3D,MAAM,aAAa,GAAG,kBAAkB,CAAC,UAAW,EAAE,WAAW,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;QAEnG,qEAAqE;QACrE,sEAAsE;QACtE,qEAAqE;QACrE,IAAI,MAAM,EAAE,YAAY,IAAI,IAAI,IAAI,aAAa,CAAC,OAAO,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC;YAChF,aAAa,CAAC,EAAE,GAAG,KAAK,CAAC;YACzB,aAAa,CAAC,KAAK,GAAG,oCAAoC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACxI,CAAC;QACD,IAAI,MAAM,EAAE,WAAW,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;YAClF,aAAa,CAAC,EAAE,GAAG,KAAK,CAAC;YACzB,aAAa,CAAC,KAAK,GAAG,sCAAsC,MAAM,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,YAAY,MAAM,CAAC,WAAW,IAAI,CAAC;QACxH,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,aAAa,CAAC;QAEtD,uEAAuE;QACvE,0EAA0E;QAC1E,iCAAiC;QACjC,IAAI,MAAM,EAAE,QAAQ,IAAI,IAAI,IAAI,YAAY,GAAG,MAAM,CAAC,QAAQ,IAAI,CAAC,eAAe,EAAE,CAAC;YACnF,eAAe,GAAG,IAAI,CAAC;YACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,KAAK,CAAC,MAAM,GAAG,OAAO,2BAA2B,CAAC,CAAC;QACtM,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,kBAAkB,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;IAC5D,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAoB,EAAE,KAAa,EAAE,YAAoB,KAAK;IAC5F,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;QAC5B,KAAK;QACL,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,IAAI;QACZ,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;QAClB,SAAS;KACV,CAAC,CAAC;IACH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,qBAAqB,KAAK,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAClE,CAAC;AACH,CAAC"}
+{"version":3,"file":"evaluation-execution.js","sourceRoot":"","sources":["../../../src/eval-core/evaluation-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,qCAAqC,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AA8ClC,KAAK,UAAU,kBAAkB,CAAI,KAAU,EAAE,WAAmB,EAAE,EAA8B;IAClG,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,UAAU,MAAM;QACnB,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,KAAK,EAAE,CAAC;YAClB,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO;QACL,EAAE,EAAE,KAAK;QACT,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,CAAC;QAChB,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,eAAe,EAAE,CAAC;QAClB,mBAAmB,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;QACV,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE,CAAC;QACX,KAAK,EAAE,OAAO;KACf,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,KAAK,EACL,QAAQ,EACR,YAAY,EACZ,aAAa,EACb,KAAK,EACL,UAAU,EACV,OAAO,EACP,WAAW,EACX,WAAW,EACX,SAAS,EACT,OAAO,EACP,OAAO,EACP,UAAU,EACV,KAAK,GAAG,CAAC,EACT,eAAe,EACf,WAAW,GAAG,CAAC,EACf,WAAW,EACX,cAAc,EACd,YAAY,GAAG,IAAI,EACnB,MAAM,GACc;IACpB,MAAM,OAAO,GAAkD,EAAE,CAAC;IAClE,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,eAAe,GAAG,KAAK,CAAC;IAE5B,4CAA4C;IAC5C,IAAI,eAAe,EAAE,CAAC;QACpB,KAAK,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;YACnE,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QACtC,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAyB,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAE3E,KAAK,UAAU,WAAW,CAAC,IAAU;QACnC,sDAAsD;QACtD,IAAI,eAAe,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC;YAC1D,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACjI,OAAO;QACT,CAAC;QAED,oEAAoE;QACpE,0EAA0E;QAC1E,yCAAyC;QACzC,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACjI,OAAO;QACT,CAAC;QAED,OAAO,EAAE,CAAC;QACV,MAAM,GAAG,GAAG,OAAO,CAAC;QACpB,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3B,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAE1G,MAAM,aAAa,GAAG,wBAAwB,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAChF,MAAM,qBAAqB,GAAG,YAAY,IAAI,QAAQ,CAAC;QACvD,MAAM,eAAe,GAAG,6BAA6B,CAAC,qBAAqB,EAAE,KAAK,EAAE;YAClF,QAAQ,EAAE,aAAa,CAAC,KAAK,CAAC,QAAQ;SACvC,CAAC,CAAC;QAEH,IAAI,UAAsB,CAAC;QAC3B,0EAA0E;QAC1E,kFAAkF;QAClF,uEAAuE;QACvE,MAAM,GAAG,GAAG,QAAQ,CAClB,KAAK,EACL,aAAa,CAAC,WAAW,EACzB,aAAa,CAAC,KAAK,CAAC,MAAM,EAC1B,aAAa,CAAC,KAAK,CAAC,GAAG,EACvB,IAAI,CAAC,QAAQ,CAAC,aAAa,EAC3B,qBAAqB,EACrB,eAAe,CAAC,WAAW,CAC5B,CAAC;QACF,MAAM,MAAM,GAAG,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,MAAM,EAAE,CAAC;YACX,UAAU,GAAG,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,gCAAgC;YAChC,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAC3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;gBACxD,IAAI,CAAC;oBACH,UAAU,GAAG,MAAM,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;gBACnD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,UAAU,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;gBACpC,CAAC;gBACD,IAAI,UAAW,CAAC,EAAE,IAAI,OAAO,KAAK,WAAW;oBAAE,MAAM;gBACrD,mCAAmC;gBACnC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC7D,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;gBAChI,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;YACzB,CAAC;YACD,IAAI,KAAK,IAAI,UAAW,CAAC,EAAE;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,UAAW,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACtC,YAAY,IAAI,UAAW,CAAC,OAAO,CAAC;QAEpC,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;YAC1B,UAAU,CAAC;gBACT,KAAK,EAAE,WAAW;gBAClB,QAAQ,EAAE,aAAa,CAAC,QAAQ;gBAChC,SAAS,EAAE,GAAG;gBACd,KAAK;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,UAAU,EAAE,UAAW,CAAC,UAAU;gBAClC,WAAW,EAAE,UAAW,CAAC,WAAW;gBACpC,YAAY,EAAE,UAAW,CAAC,YAAY;gBACtC,OAAO,EAAE,UAAW,CAAC,OAAO;gBAC5B,aAAa,EAAE,UAAW,CAAC,MAAM,CAAC,CAAC,CAAC,UAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;aAC5E,CAAC,CAAC;QACL,CAAC;QAED,IAAI,WAAW,GAAuB,IAAI,CAAC;QAC3C,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,UAAW,CAAC,EAAE,EAAE,CAAC;YACnB,MAAM,kBAAkB,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;YAC9H,IAAI,kBAAkB,EAAE,CAAC;gBACvB,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;oBAC1B,UAAU,CAAC;wBACT,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,aAAa,CAAC,QAAQ;wBAChC,SAAS,EAAE,GAAG;wBACd,KAAK;wBACL,SAAS,EAAE,IAAI,CAAC,SAAS;wBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;qBACtB,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC9B,IAAI,CAAC;oBACH,WAAW,GAAG,MAAM,KAAK,CAAC;wBACxB,MAAM,EAAE,UAAW,CAAC,MAAO;wBAC3B,MAAM,EAAE,IAAI,CAAC,OAAO;wBACpB,QAAQ,EAAE,aAAa;wBACvB,UAAU;wBACV,aAAa,EAAE,CAAC,OAAO;wBACvB,WAAW,EAAE;4BACX,OAAO,EAAE,UAAW,CAAC,OAAO;4BAC5B,UAAU,EAAE,UAAW,CAAC,UAAU;4BAClC,QAAQ,EAAE,UAAW,CAAC,QAAQ;4BAC9B,SAAS,EAAE,UAAW,CAAC,SAAS;4BAChC,KAAK,EAAE,UAAW,CAAC,KAAK;yBACzB;wBACD,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;wBACzC,WAAW;wBACX,WAAW;wBACX,cAAc;wBACd,YAAY;qBACb,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,WAAW,GAAG,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;oBACpC,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC7D,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,EAAE,EAAE,CAAC,CAAC;gBACnI,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;gBAClC,IAAI,WAAW,CAAC,YAAY;oBAAE,YAAY,IAAI,WAAW,CAAC,YAAY,CAAC;YACzE,CAAC;QACH,CAAC;QAED,IAAI,SAAsC,CAAC;QAC3C,IAAI,UAAW,CAAC,EAAE,IAAI,UAAW,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACrD,SAAS,GAAG,UAAU,CAAC,UAAW,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;QAC3D,MAAM,aAAa,GAAG,kBAAkB,CAAC,UAAW,EAAE,WAAW,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;QAEnG,gEAAgE;QAChE,sEAAsE;QACtE,qEAAqE;QACrE,IAAI,MAAM,EAAE,YAAY,IAAI,IAAI,IAAI,aAAa,CAAC,OAAO,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC;YAChF,aAAa,CAAC,EAAE,GAAG,KAAK,CAAC;YACzB,aAAa,CAAC,KAAK,GAAG,oCAAoC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACxI,CAAC;QACD,IAAI,MAAM,EAAE,WAAW,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;YAClF,aAAa,CAAC,EAAE,GAAG,KAAK,CAAC;YACzB,aAAa,CAAC,KAAK,GAAG,sCAAsC,MAAM,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,YAAY,MAAM,CAAC,WAAW,IAAI,CAAC;QACxH,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,aAAa,CAAC;QAEtD,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,aAAa,CAAC,QAAQ;YAChC,SAAS;YACT,KAAK;YACL,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,aAAa,CAAC,UAAU;YACpC,WAAW,EAAE,aAAa,CAAC,WAAW;YACtC,YAAY,EAAE,aAAa,CAAC,YAAY;YACxC,OAAO,EAAE,aAAa,CAAC,WAAW;YAClC,KAAK,EAAE,WAAW,EAAE,cAAc;YAClC,EAAE,EAAE,aAAa,CAAC,EAAE;YACpB,KAAK,EAAE,aAAa,CAAC,KAAK;SAC3B,CAAC,CAAC;QAEH,kEAAkE;QAClE,0EAA0E;QAC1E,iCAAiC;QACjC,IAAI,MAAM,EAAE,QAAQ,IAAI,IAAI,IAAI,YAAY,GAAG,MAAM,CAAC,QAAQ,IAAI,CAAC,eAAe,EAAE,CAAC;YACnF,eAAe,GAAG,IAAI,CAAC;YACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,KAAK,CAAC,MAAM,GAAG,OAAO,2BAA2B,CAAC,CAAC;QACtM,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,kBAAkB,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;IAC5D,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAoB,EAAE,KAAa,EAAE,YAAoB,KAAK;IAC5F,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;QAC5B,KAAK;QACL,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,IAAI;QACZ,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;QAClB,SAAS;KACV,CAAC,CAAC;IACH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,qBAAqB,KAAK,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAClE,CAAC;AACH,CAAC"}

package/dist/src/eval-core/evaluation-job.d.ts

@@ -1,5 +1,5 @@
 import type { Artifact, EvaluationErrorCategory, EvaluationJob, EvaluationRequest, EvaluationRun, JudgeConfig } from '../types/index.js';
-export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, judgeModel, executor, judgeExecutor, noJudge, concurrency, timeoutMs, noCache, dryRun, blind, project, owner, tags, repeat, each, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, }: {
+export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, judgeModel, executor, judgeExecutor, noJudge, concurrency, timeoutMs, noCache, dryRun, blind, project, owner, tags, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, }: {
     samplesPath: string;
     skillDir: string;
     artifacts: Artifact[];
@@ -17,7 +17,7 @@ export declare function buildEvaluationRequest({ samplesPath, skillDir, artifact
     owner?: string;
     tags?: string[];
     repeat?: number;
-    each?: boolean;
+    batch?: boolean;
     judgeRepeat?: number;
     judgeModels?: JudgeConfig[];
     bootstrap?: boolean;

package/dist/src/eval-core/evaluation-job.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation-job.d.ts","sourceRoot":"","sources":["../../../src/eval-core/evaluation-job.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,uBAAuB,EAAE,aAAa,EAAE,iBAAiB,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAMzI,wBAAgB,sBAAsB,CAAC,EACrC,WAAW,EACX,QAAQ,EACR,SAAS,EACT,KAAK,EACL,UAAU,EACV,QAAQ,EACR,aAAa,EACb,OAAO,EACP,WAAW,EACX,SAAS,EACT,OAAO,EACP,MAAM,EACN,KAAK,EACL,OAAO,EACP,KAAK,EACL,IAAI,EACJ,MAAM,EACN,IAAI,EACJ,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,MAAM,GACP,EAAE;IACD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,WAAW,EAAE,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,mBAAmB,EAAE,UAAU,CAAC;CACjD,GAAG,iBAAiB,CA2BpB;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAiB,GAAG;IAAE,GAAG,EAAE,aAAa,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAS1H;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,aAAa,EAAE,UAAU,GAAE,MAAiB,GAAG,aAAa,CAMtG;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,aAAa,EAAE,UAAU,GAAE,MAAiB,GAAG,aAAa,CAMlG;AAED,wBAAgB,eAAe,CAAC,EAC9B,KAAK,EACL,OAAO,EACP,SAAoB,GACrB,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,iBAAiB,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,aAAa,CAQhB;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAiB,GAAG,aAAa,CAQ7G;AAED,wBAAgB,kBAAkB,CAAC,EACjC,KAAK,EACL,KAAK,EACL,QAAQ,EACR,OAAO,EACP,SAAS,EACT,SAAS,EACT,UAAU,GACX,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,aAAa,CAYhB;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,uBAAuB,CA4B9E;AAED,wBAAgB,eAAe,CAAC,EAC9B,GAAG,EACH,KAAK,EACL,aAA8C,EAC9C,UAAqB,GACtB,EAAE;IACD,GAAG,EAAE,aAAa,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,uBAAuB,CAAC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,aAAa,CAShB"}
+{"version":3,"file":"evaluation-job.d.ts","sourceRoot":"","sources":["../../../src/eval-core/evaluation-job.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,uBAAuB,EAAE,aAAa,EAAE,iBAAiB,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAMzI,wBAAgB,sBAAsB,CAAC,EACrC,WAAW,EACX,QAAQ,EACR,SAAS,EACT,KAAK,EACL,UAAU,EACV,QAAQ,EACR,aAAa,EACb,OAAO,EACP,WAAW,EACX,SAAS,EACT,OAAO,EACP,MAAM,EACN,KAAK,EACL,OAAO,EACP,KAAK,EACL,IAAI,EACJ,MAAM,EACN,KAAK,EACL,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,MAAM,GACP,EAAE;IACD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,WAAW,EAAE,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,mBAAmB,EAAE,UAAU,CAAC;CACjD,GAAG,iBAAiB,CA2BpB;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAiB,GAAG;IAAE,GAAG,EAAE,aAAa,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAS1H;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,aAAa,EAAE,UAAU,GAAE,MAAiB,GAAG,aAAa,CAMtG;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,aAAa,EAAE,UAAU,GAAE,MAAiB,GAAG,aAAa,CAMlG;AAED,wBAAgB,eAAe,CAAC,EAC9B,KAAK,EACL,OAAO,EACP,SAAoB,GACrB,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,iBAAiB,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,aAAa,CAQhB;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAiB,GAAG,aAAa,CAQ7G;AAED,wBAAgB,kBAAkB,CAAC,EACjC,KAAK,EACL,KAAK,EACL,QAAQ,EACR,OAAO,EACP,SAAS,EACT,SAAS,EACT,UAAU,GACX,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,aAAa,CAYhB;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,uBAAuB,CA4B9E;AAED,wBAAgB,eAAe,CAAC,EAC9B,GAAG,EACH,KAAK,EACL,aAA8C,EAC9C,UAAqB,GACtB,EAAE;IACD,GAAG,EAAE,aAAa,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,uBAAuB,CAAC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,aAAa,CAShB"}