niceeval 0.18.0 → 0.19.0

package/dist/runner/eval-selection.cjs

@@ -77,8 +77,8 @@ function resolveExperimentEvals(input) {
     }
     if (input.validateModelRequirements !== false) {
         const errors = selectorEvals.flatMap((evalDef) => {
-            const missing = evalDef.requiredModelSlots?.filter((key) => !Object.hasOwn(input.models ?? {}, key)) ?? [];
-            return missing.length === 0 ? [] : [`Eval "${evalDef.id}" requires undeclared model slots: ${missing.join(", ")}`];
+            const missing = evalDef.requiredModelRoles?.filter((key) => !Object.hasOwn(input.models ?? {}, key)) ?? [];
+            return missing.length === 0 ? [] : [`Eval "${evalDef.id}" requires undeclared model roles: ${missing.join(", ")}`];
         });
         if (errors.length > 0)
             throw new TypeError(`Experiment "${experimentId}": ${errors.join("; ")}.`);

package/dist/runner/eval-selection.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"eval-selection.js","sourceRoot":"","sources":["eval-selection.ts"],"names":[],"mappings":";;AAQA,4CAUC;AAyBD,wDA8DC;AAUD,sDAOC;AAED,kDAGC;AA9HD,6DAA6D;AAC7D,2DAA2D;AAC3D,6DAA6D;AAC7D,2EAA2E;AAC3E,yDAA6D;AAE7D,sDAAsD;AACtD,SAAgB,gBAAgB,CAAC,OAAuB;IACpD,OAAO,MAAM,CAAC,MAAM,CAAC;QACjB,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClF,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;QAC9C,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,MAAM;QAChD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC;YAC9C,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE;YACtD,CAAC,CAAC,EAAE,CAAC;KACZ,CAAC,CAAC;AACP,CAAC;AAoBD;;;;GAIG;AACH,SAAgB,sBAAsB,CAAC,KAAkC;IACrE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;IAC7D,MAAM,aAAa,GAAG,IAAA,kCAAmB,EAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACjG,IAAI,cAAoD,CAAC;IACzD,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;QAC7C,cAAc,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC;IAChC,CAAC;SACI,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,QAAQ,CAAC;QAC3B,cAAc,GAAG,CAAC,OAAO,EAAE,EAAE;YACzB,IAAI,MAAe,CAAC;YACpB,IAAI,CAAC;gBACD,MAAM,GAAG,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;YAClD,CAAC;YACD,OAAO,CAAC,EAAE,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,eAAe,YAAY,qCAAqC,OAAO,CAAC,EAAE,KAAK;oBAC3F,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YACvE,CAAC;YACD,IAAI,MAAM,YAAY,OAAO,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,eAAe,YAAY,kDAAkD,OAAO,CAAC,EAAE,KAAK;oBACxG,kEAAkE,CAAC,CAAC;YAC5E,CAAC;YACD,IAAI,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,eAAe,YAAY,8BAA8B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,mBAAmB;oBAC9G,aAAa,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;YACrC,CAAC;YACD,OAAO,MAAM,CAAC;QAClB,CAAC,CAAC;IACN,CAAC;SACI,CAAC;QACF,MAAM,WAAW,GAAG,IAAA,kCAAmB,EAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;QACvD,cAAc,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,aAAa,GAAqB,EAAE,CAAC;IAC3C,MAAM,aAAa,GAAqB,EAAE,CAAC;IAC3C,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACpB,SAAS;QACb,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;YACxB,SAAS;QACb,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACrB,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC;YAC5D,SAAS;QACb,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1B,SAAS;QACb,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,KAAK,CAAC,yBAAyB,KAAK,KAAK,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3G,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,EAAE,sCAAsC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvH,CAAC,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YACjB,MAAM,IAAI,SAAS,CAAC,eAAe,YAAY,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnF,CAAC;IACD,OAAO;QACH,aAAa;QACb,eAAe,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,KAAK,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAI,CAAC,CAAC;KAChI,CAAC;AACN,CAAC;AAMD;;;GAGG;AACH,SAAgB,qBAAqB,CAAC,aAAwC;IAC1E,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;QAClC,CAAC,OAAO,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AACD,uDAAuD;AACvD,SAAgB,mBAAmB,CAAC,GAA8B,EAAE,GAAsC;IACtG,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACzC,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5C,CAAC","sourcesContent":["import type { ResolvedModelSlots } from \"../model-slots.ts\";\n// 纯选题边界:发现结果 → EvalDescriptor 投影 → 用户谓词求值 → selectedEvalIds。\n// CLI 在构造每个 AgentRun 前对候选 eval 各调用一次谓词并把结果落进这里;下游(dry-run、\n// sandbox 查表、fingerprint/carry、attempt 展开、hook ctx、快照)只消费返回的\n// selectedEvalIds,不重新调用用户谓词(见 docs/feature/experiments/library.md「evals」)。\nimport { evalPrefixPredicate } from \"../shared/aggregate.ts\";\nimport type { AgentRun, DiscoveredEval, EvalDescriptor, ExperimentAuthorFields } from \"./types.ts\";\n/** `DiscoveredEval` → 用户谓词可见的显式白名单投影;不透传内部路径/执行字段。 */\nexport function evalDescriptorOf(evalDef: DiscoveredEval): EvalDescriptor {\n    return Object.freeze({\n        id: evalDef.id,\n        ...(evalDef.description !== undefined ? { description: evalDef.description } : {}),\n        tags: Object.freeze([...(evalDef.tags ?? [])]),\n        evaluationKind: evalDef.evaluationKind ?? \"pass\",\n        ...(Object.keys(evalDef.metadata ?? {}).length > 0\n            ? { metadata: Object.freeze({ ...evalDef.metadata }) }\n            : {}),\n    });\n}\nexport interface ResolveExperimentEvalsInput {\n    experimentId: string;\n    models?: ResolvedModelSlots;\n    tag?: string;\n    validateModelRequirements?: boolean;\n    selector: ExperimentAuthorFields[\"evals\"];\n    cliPatterns: readonly string[];\n    evals: readonly DiscoveredEval[];\n}\nexport interface ResolveExperimentEvalsResult {\n    selectedEvals: readonly DiscoveredEval[];\n    selectedEvalIds: readonly string[];\n    /**\n     * 只过了实验自身 `evals` 选择器、还没被 CLI 尾随前缀收窄的那一集——即这个实验的**发现集**。\n     * CLI 拿它判定「某个尾随前缀在选中实验里匹配 0 条」:交集为空时两种原因(前缀写错 / 实验本来\n     * 就没选中任何 eval)要报不同的错,只看交集分辨不出来。谓词仍然只求值一次。\n     */\n    selectorEvals: readonly DiscoveredEval[];\n}\n/**\n * 对某个 experiment 的候选 eval 集合各求值谓词一次,与 CLI 追加的位置参数前缀取交集。\n * 返回顺序 = discovery 稳定顺序,id 去重——这是 dry-run、attempt 派发顺序与落盘\n * `selectedEvalIds` 的共同来源(见 docs/feature/eval/README.md「路径即身份」)。\n */\nexport function resolveExperimentEvals(input: ResolveExperimentEvalsInput): ResolveExperimentEvalsResult {\n    const { experimentId, selector, cliPatterns, evals } = input;\n    const patternFilter = evalPrefixPredicate(cliPatterns.length > 0 ? [...cliPatterns] : undefined);\n    let selectorFilter: (evalDef: DiscoveredEval) => boolean;\n    if (selector === undefined || selector === \"*\") {\n        selectorFilter = () => true;\n    }\n    else if (typeof selector === \"function\") {\n        const predicate = selector;\n        selectorFilter = (evalDef) => {\n            let result: unknown;\n            try {\n                result = predicate(evalDescriptorOf(evalDef));\n            }\n            catch (e) {\n                throw new Error(`experiment \"${experimentId}\" evals predicate threw for eval \"${evalDef.id}\": ` +\n                    `${e instanceof Error ? e.message : String(e)}`, { cause: e });\n            }\n            if (result instanceof Promise) {\n                throw new Error(`experiment \"${experimentId}\" evals predicate returned a Promise for eval \"${evalDef.id}\"; ` +\n                    \"the predicate must be synchronous — do not await inside evals().\");\n            }\n            if (typeof result !== \"boolean\") {\n                throw new Error(`experiment \"${experimentId}\" evals predicate returned ${JSON.stringify(result)} (not a boolean) ` +\n                    `for eval \"${evalDef.id}\".`);\n            }\n            return result;\n        };\n    }\n    else {\n        const arrayFilter = evalPrefixPredicate([...selector]);\n        selectorFilter = (evalDef) => arrayFilter(evalDef.id);\n    }\n    const seen = new Set<string>();\n    const selectedEvals: DiscoveredEval[] = [];\n    const selectorEvals: DiscoveredEval[] = [];\n    for (const evalDef of evals) {\n        if (seen.has(evalDef.id))\n            continue;\n        if (!selectorFilter(evalDef))\n            continue;\n        seen.add(evalDef.id);\n        selectorEvals.push(evalDef);\n        if (input.tag !== undefined && !evalDef.tags.includes(input.tag))\n            continue;\n        if (!patternFilter(evalDef.id))\n            continue;\n        selectedEvals.push(evalDef);\n    }\n    if (input.validateModelRequirements !== false) {\n        const errors = selectorEvals.flatMap((evalDef) => {\n            const missing = evalDef.requiredModelSlots?.filter((key) => !Object.hasOwn(input.models ?? {}, key)) ?? [];\n            return missing.length === 0 ? [] : [`Eval \"${evalDef.id}\" requires undeclared model slots: ${missing.join(\", \")}`];\n        });\n        if (errors.length > 0)\n            throw new TypeError(`Experiment \"${experimentId}\": ${errors.join(\"; \")}.`);\n    }\n    return {\n        selectedEvals,\n        selectedEvalIds: selectedEvals.map((e) => e.id),\n        selectorEvals: input.tag === undefined ? selectorEvals : selectorEvals.filter((evalDef) => evalDef.tags.includes(input.tag!)),\n    };\n}\n/** `resolveExperimentEvals` 选中的 eval 按题型分桶后的 id 列表(见 splitByEvaluationKind)。 */\nexport interface EvaluationKindSplit {\n    pass: string[];\n    score: string[];\n}\n/**\n * 按题型(`EvalDescriptor.evaluationKind`)把选中的 eval 分桶。Experiment admission 与 Eval Group\n * discovery 用它拒绝混型；Report 仍用同一投影读取 admission 生效前留下的历史 mixed Record。\n */\nexport function splitByEvaluationKind(selectedEvals: readonly DiscoveredEval[]): EvaluationKindSplit {\n    const pass: string[] = [];\n    const score: string[] = [];\n    for (const evalDef of selectedEvals) {\n        (evalDef.evaluationKind === \"score\" ? score : pass).push(evalDef.id);\n    }\n    return { pass, score };\n}\n/** 所有消费者按已解析的 `selectedEvalIds` 取 eval;不持有、不调用用户谓词。 */\nexport function selectedEvalsForRun(all: readonly DiscoveredEval[], run: Pick<AgentRun, \"selectedEvalIds\">): DiscoveredEval[] {\n    const ids = new Set(run.selectedEvalIds);\n    return all.filter((e) => ids.has(e.id));\n}\n"]}
+{"version":3,"file":"eval-selection.js","sourceRoot":"","sources":["eval-selection.ts"],"names":[],"mappings":";;AAQA,4CAUC;AAyBD,wDA8DC;AAUD,sDAOC;AAED,kDAGC;AA9HD,6DAA6D;AAC7D,2DAA2D;AAC3D,6DAA6D;AAC7D,2EAA2E;AAC3E,yDAA6D;AAE7D,sDAAsD;AACtD,SAAgB,gBAAgB,CAAC,OAAuB;IACpD,OAAO,MAAM,CAAC,MAAM,CAAC;QACjB,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClF,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;QAC9C,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,MAAM;QAChD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC;YAC9C,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE;YACtD,CAAC,CAAC,EAAE,CAAC;KACZ,CAAC,CAAC;AACP,CAAC;AAoBD;;;;GAIG;AACH,SAAgB,sBAAsB,CAAC,KAAkC;IACrE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;IAC7D,MAAM,aAAa,GAAG,IAAA,kCAAmB,EAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACjG,IAAI,cAAoD,CAAC;IACzD,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;QAC7C,cAAc,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC;IAChC,CAAC;SACI,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,QAAQ,CAAC;QAC3B,cAAc,GAAG,CAAC,OAAO,EAAE,EAAE;YACzB,IAAI,MAAe,CAAC;YACpB,IAAI,CAAC;gBACD,MAAM,GAAG,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;YAClD,CAAC;YACD,OAAO,CAAC,EAAE,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,eAAe,YAAY,qCAAqC,OAAO,CAAC,EAAE,KAAK;oBAC3F,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YACvE,CAAC;YACD,IAAI,MAAM,YAAY,OAAO,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,eAAe,YAAY,kDAAkD,OAAO,CAAC,EAAE,KAAK;oBACxG,kEAAkE,CAAC,CAAC;YAC5E,CAAC;YACD,IAAI,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,eAAe,YAAY,8BAA8B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,mBAAmB;oBAC9G,aAAa,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;YACrC,CAAC;YACD,OAAO,MAAM,CAAC;QAClB,CAAC,CAAC;IACN,CAAC;SACI,CAAC;QACF,MAAM,WAAW,GAAG,IAAA,kCAAmB,EAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;QACvD,cAAc,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,aAAa,GAAqB,EAAE,CAAC;IAC3C,MAAM,aAAa,GAAqB,EAAE,CAAC;IAC3C,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACpB,SAAS;QACb,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;YACxB,SAAS;QACb,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACrB,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC;YAC5D,SAAS;QACb,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1B,SAAS;QACb,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,KAAK,CAAC,yBAAyB,KAAK,KAAK,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3G,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,EAAE,sCAAsC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvH,CAAC,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YACjB,MAAM,IAAI,SAAS,CAAC,eAAe,YAAY,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnF,CAAC;IACD,OAAO;QACH,aAAa;QACb,eAAe,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,KAAK,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAI,CAAC,CAAC;KAChI,CAAC;AACN,CAAC;AAMD;;;GAGG;AACH,SAAgB,qBAAqB,CAAC,aAAwC;IAC1E,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;QAClC,CAAC,OAAO,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AACD,uDAAuD;AACvD,SAAgB,mBAAmB,CAAC,GAA8B,EAAE,GAAsC;IACtG,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACzC,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5C,CAAC","sourcesContent":["import type { ResolvedModelRoles } from \"../model-roles.ts\";\n// 纯选题边界:发现结果 → EvalDescriptor 投影 → 用户谓词求值 → selectedEvalIds。\n// CLI 在构造每个 AgentRun 前对候选 eval 各调用一次谓词并把结果落进这里;下游(dry-run、\n// sandbox 查表、fingerprint/carry、attempt 展开、hook ctx、快照)只消费返回的\n// selectedEvalIds,不重新调用用户谓词(见 docs/feature/experiments/library.md「evals」)。\nimport { evalPrefixPredicate } from \"../shared/aggregate.ts\";\nimport type { AgentRun, DiscoveredEval, EvalDescriptor, ExperimentAuthorFields } from \"./types.ts\";\n/** `DiscoveredEval` → 用户谓词可见的显式白名单投影;不透传内部路径/执行字段。 */\nexport function evalDescriptorOf(evalDef: DiscoveredEval): EvalDescriptor {\n    return Object.freeze({\n        id: evalDef.id,\n        ...(evalDef.description !== undefined ? { description: evalDef.description } : {}),\n        tags: Object.freeze([...(evalDef.tags ?? [])]),\n        evaluationKind: evalDef.evaluationKind ?? \"pass\",\n        ...(Object.keys(evalDef.metadata ?? {}).length > 0\n            ? { metadata: Object.freeze({ ...evalDef.metadata }) }\n            : {}),\n    });\n}\nexport interface ResolveExperimentEvalsInput {\n    experimentId: string;\n    models?: ResolvedModelRoles;\n    tag?: string;\n    validateModelRequirements?: boolean;\n    selector: ExperimentAuthorFields[\"evals\"];\n    cliPatterns: readonly string[];\n    evals: readonly DiscoveredEval[];\n}\nexport interface ResolveExperimentEvalsResult {\n    selectedEvals: readonly DiscoveredEval[];\n    selectedEvalIds: readonly string[];\n    /**\n     * 只过了实验自身 `evals` 选择器、还没被 CLI 尾随前缀收窄的那一集——即这个实验的**发现集**。\n     * CLI 拿它判定「某个尾随前缀在选中实验里匹配 0 条」:交集为空时两种原因(前缀写错 / 实验本来\n     * 就没选中任何 eval)要报不同的错,只看交集分辨不出来。谓词仍然只求值一次。\n     */\n    selectorEvals: readonly DiscoveredEval[];\n}\n/**\n * 对某个 experiment 的候选 eval 集合各求值谓词一次,与 CLI 追加的位置参数前缀取交集。\n * 返回顺序 = discovery 稳定顺序,id 去重——这是 dry-run、attempt 派发顺序与落盘\n * `selectedEvalIds` 的共同来源(见 docs/feature/eval/README.md「路径即身份」)。\n */\nexport function resolveExperimentEvals(input: ResolveExperimentEvalsInput): ResolveExperimentEvalsResult {\n    const { experimentId, selector, cliPatterns, evals } = input;\n    const patternFilter = evalPrefixPredicate(cliPatterns.length > 0 ? [...cliPatterns] : undefined);\n    let selectorFilter: (evalDef: DiscoveredEval) => boolean;\n    if (selector === undefined || selector === \"*\") {\n        selectorFilter = () => true;\n    }\n    else if (typeof selector === \"function\") {\n        const predicate = selector;\n        selectorFilter = (evalDef) => {\n            let result: unknown;\n            try {\n                result = predicate(evalDescriptorOf(evalDef));\n            }\n            catch (e) {\n                throw new Error(`experiment \"${experimentId}\" evals predicate threw for eval \"${evalDef.id}\": ` +\n                    `${e instanceof Error ? e.message : String(e)}`, { cause: e });\n            }\n            if (result instanceof Promise) {\n                throw new Error(`experiment \"${experimentId}\" evals predicate returned a Promise for eval \"${evalDef.id}\"; ` +\n                    \"the predicate must be synchronous — do not await inside evals().\");\n            }\n            if (typeof result !== \"boolean\") {\n                throw new Error(`experiment \"${experimentId}\" evals predicate returned ${JSON.stringify(result)} (not a boolean) ` +\n                    `for eval \"${evalDef.id}\".`);\n            }\n            return result;\n        };\n    }\n    else {\n        const arrayFilter = evalPrefixPredicate([...selector]);\n        selectorFilter = (evalDef) => arrayFilter(evalDef.id);\n    }\n    const seen = new Set<string>();\n    const selectedEvals: DiscoveredEval[] = [];\n    const selectorEvals: DiscoveredEval[] = [];\n    for (const evalDef of evals) {\n        if (seen.has(evalDef.id))\n            continue;\n        if (!selectorFilter(evalDef))\n            continue;\n        seen.add(evalDef.id);\n        selectorEvals.push(evalDef);\n        if (input.tag !== undefined && !evalDef.tags.includes(input.tag))\n            continue;\n        if (!patternFilter(evalDef.id))\n            continue;\n        selectedEvals.push(evalDef);\n    }\n    if (input.validateModelRequirements !== false) {\n        const errors = selectorEvals.flatMap((evalDef) => {\n            const missing = evalDef.requiredModelRoles?.filter((key) => !Object.hasOwn(input.models ?? {}, key)) ?? [];\n            return missing.length === 0 ? [] : [`Eval \"${evalDef.id}\" requires undeclared model roles: ${missing.join(\", \")}`];\n        });\n        if (errors.length > 0)\n            throw new TypeError(`Experiment \"${experimentId}\": ${errors.join(\"; \")}.`);\n    }\n    return {\n        selectedEvals,\n        selectedEvalIds: selectedEvals.map((e) => e.id),\n        selectorEvals: input.tag === undefined ? selectorEvals : selectorEvals.filter((evalDef) => evalDef.tags.includes(input.tag!)),\n    };\n}\n/** `resolveExperimentEvals` 选中的 eval 按题型分桶后的 id 列表(见 splitByEvaluationKind)。 */\nexport interface EvaluationKindSplit {\n    pass: string[];\n    score: string[];\n}\n/**\n * 按题型(`EvalDescriptor.evaluationKind`)把选中的 eval 分桶。Experiment admission 与 Eval Group\n * discovery 用它拒绝混型；Report 仍用同一投影读取 admission 生效前留下的历史 mixed Record。\n */\nexport function splitByEvaluationKind(selectedEvals: readonly DiscoveredEval[]): EvaluationKindSplit {\n    const pass: string[] = [];\n    const score: string[] = [];\n    for (const evalDef of selectedEvals) {\n        (evalDef.evaluationKind === \"score\" ? score : pass).push(evalDef.id);\n    }\n    return { pass, score };\n}\n/** 所有消费者按已解析的 `selectedEvalIds` 取 eval;不持有、不调用用户谓词。 */\nexport function selectedEvalsForRun(all: readonly DiscoveredEval[], run: Pick<AgentRun, \"selectedEvalIds\">): DiscoveredEval[] {\n    const ids = new Set(run.selectedEvalIds);\n    return all.filter((e) => ids.has(e.id));\n}\n"]}

package/dist/runner/eval-selection.d.cts

@@ -1,10 +1,10 @@
-import type { ResolvedModelSlots } from "../model-slots.cjs";
+import type { ResolvedModelRoles } from "../model-roles.cjs";
 import type { AgentRun, DiscoveredEval, EvalDescriptor, ExperimentAuthorFields } from "./types.cjs";
 /** `DiscoveredEval` → 用户谓词可见的显式白名单投影;不透传内部路径/执行字段。 */
 export declare function evalDescriptorOf(evalDef: DiscoveredEval): EvalDescriptor;
 export interface ResolveExperimentEvalsInput {
     experimentId: string;
-    models?: ResolvedModelSlots;
+    models?: ResolvedModelRoles;
     tag?: string;
     validateModelRequirements?: boolean;
     selector: ExperimentAuthorFields["evals"];

package/dist/runner/eval-selection.d.mts

@@ -1,10 +1,10 @@
-import type { ResolvedModelSlots } from "../model-slots.mjs";
+import type { ResolvedModelRoles } from "../model-roles.mjs";
 import type { AgentRun, DiscoveredEval, EvalDescriptor, ExperimentAuthorFields } from "./types.mjs";
 /** `DiscoveredEval` → 用户谓词可见的显式白名单投影;不透传内部路径/执行字段。 */
 export declare function evalDescriptorOf(evalDef: DiscoveredEval): EvalDescriptor;
 export interface ResolveExperimentEvalsInput {
     experimentId: string;
-    models?: ResolvedModelSlots;
+    models?: ResolvedModelRoles;
     tag?: string;
     validateModelRequirements?: boolean;
     selector: ExperimentAuthorFields["evals"];

package/dist/runner/eval-selection.d.ts

@@ -1,10 +1,10 @@
-import type { ResolvedModelSlots } from "../model-slots.js";
+import type { ResolvedModelRoles } from "../model-roles.js";
 import type { AgentRun, DiscoveredEval, EvalDescriptor, ExperimentAuthorFields } from "./types.js";
 /** `DiscoveredEval` → 用户谓词可见的显式白名单投影;不透传内部路径/执行字段。 */
 export declare function evalDescriptorOf(evalDef: DiscoveredEval): EvalDescriptor;
 export interface ResolveExperimentEvalsInput {
     experimentId: string;
-    models?: ResolvedModelSlots;
+    models?: ResolvedModelRoles;
     tag?: string;
     validateModelRequirements?: boolean;
     selector: ExperimentAuthorFields["evals"];

package/dist/runner/feedback/human.cjs

@@ -494,7 +494,7 @@ function buildSummaryLines(event, state, panel) {
         { kind: "line", text: formatSummaryCostLine(state) },
         {
             kind: "line",
-            text: `Setup prefixes: ${summary.setupPrefixes.hit} hit · ${summary.setupPrefixes.prepared} prepared · ${summary.setupPrefixes.failed} failed (${summary.setupPrefixes.total} total)`,
+            text: `Before caches: ${summary.beforeCaches.hit} hit · ${summary.beforeCaches.prepared} prepared · ${summary.beforeCaches.failed} failed (${summary.beforeCaches.total} total)`,
         },
     ];
     const blocks = [