@aipack-ai/eval 1.1.7 → 1.1.8

package/dist/src/core/import.js

@@ -0,0 +1,111 @@
+/**
+ * packages/eval/src/core/import.ts - trace 回流用例入库（M4）
+ *
+ * 把 observability-server `POST /api/v1/export-eval` 导出的 EvalCase JSON
+ * 写入用例目录（入库 = 纳入日常 eval 回归）。入库前走 validateEvalCase
+ * （fail fast），支持单用例 / 数组 / { cases: [...] } 三种文件形态，
+ * 以及 suite / origin / id 前缀的人工修正（bad case 固化纪律）。
+ */
+import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { validateEvalCase } from './validate.js';
+import { defaultCasesDir } from './loader.js';
+function normalizeId(id) {
+    // 文件名 = id 的最后一段（'trace-export/t-1' → 't-1.json'），非法字符收敛
+    const last = id.split('/').pop() ?? id;
+    return last.replace(/[^a-zA-Z0-9_-]/g, '_');
+}
+export async function importCases(opts) {
+    const outDir = resolve(opts.outDir ?? defaultCasesDir());
+    const raw = await readFile(opts.file, 'utf-8');
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (e) {
+        return { written: [], errors: [`JSON 解析失败: ${e.message}`] };
+    }
+    let list;
+    if (Array.isArray(parsed))
+        list = parsed;
+    else if (parsed && typeof parsed === 'object' && Array.isArray(parsed.cases)) {
+        list = parsed.cases;
+    }
+    else
+        list = [parsed];
+    if (list.length === 0)
+        return { written: [], errors: ['文件中没有用例'] };
+    const written = [];
+    const errors = [];
+    /** 已占用文件名：suite → Set<fileName> */
+    const taken = new Map();
+    const takeFor = (suiteName) => {
+        let set = taken.get(suiteName);
+        if (!set) {
+            set = new Set();
+            taken.set(suiteName, set);
+        }
+        return set;
+    };
+    if (!opts.dryRun) {
+        // 同目录已存在的文件也占位，避免覆盖人工已修正的用例
+        try {
+            const suiteDirGuess = opts.suite ?? guessSuite(list);
+            if (suiteDirGuess) {
+                const existing = await readdir(resolve(outDir, suiteDirGuess)).catch(() => []);
+                for (const f of existing)
+                    takeFor(suiteDirGuess).add(f);
+            }
+        }
+        catch {
+            // outDir 不存在时忽略
+        }
+    }
+    for (const [i, item] of list.entries()) {
+        const c = item;
+        const label = `#${i} ${c.id ?? '<missing id>'}`;
+        if (!c || typeof c !== 'object') {
+            errors.push(`${label}: 不是对象`);
+            continue;
+        }
+        if (opts.suite)
+            c.suite = opts.suite;
+        if (opts.origin)
+            c.origin = opts.origin;
+        if (opts.prefix)
+            c.id = `${opts.prefix}/${String(c.id ?? `case-${i}`)}`;
+        const validation = validateEvalCase(c);
+        if (validation.length > 0) {
+            errors.push(`${label}: ${validation.join('; ')}`);
+            continue;
+        }
+        const suite = String(c.suite);
+        const takenSet = takeFor(suite);
+        let fileName = `${normalizeId(String(c.id))}.json`;
+        let n = 1;
+        while (takenSet.has(fileName)) {
+            fileName = `${normalizeId(String(c.id))}-${n}.json`;
+            n += 1;
+        }
+        takenSet.add(fileName);
+        if (opts.dryRun) {
+            written.push(`${suite}/${fileName}`);
+            continue;
+        }
+        const target = resolve(outDir, suite, fileName);
+        try {
+            await mkdir(resolve(outDir, suite), { recursive: true });
+            await writeFile(target, `${JSON.stringify(c, null, 2)}\n`, 'utf-8');
+            written.push(`${suite}/${fileName}`);
+        }
+        catch (e) {
+            errors.push(`${label}: 写入失败 - ${e.message}`);
+        }
+    }
+    return { written, errors };
+}
+function guessSuite(list) {
+    const first = list[0];
+    return typeof first?.suite === 'string' && first.suite ? first.suite : undefined;
+}
+//# sourceMappingURL=import.js.map

package/dist/src/core/import.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"import.js","sourceRoot":"","sources":["../../../src/core/import.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAwB3C,SAAS,WAAW,CAAC,EAAU;IAC7B,yDAAyD;IACzD,MAAM,IAAI,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;IACvC,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAmB;IACnD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC,CAAC;IACzD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,cAAe,CAAW,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;IACzE,CAAC;IAED,IAAI,IAAe,CAAC;IACpB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,IAAI,GAAG,MAAM,CAAC;SACpC,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAE,MAAgC,CAAC,KAAK,CAAC,EAAE,CAAC;QACxG,IAAI,GAAI,MAA+B,CAAC,KAAK,CAAC;IAChD,CAAC;;QAAM,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAEvB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;IAEnE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,mCAAmC;IACnC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC7C,MAAM,OAAO,GAAG,CAAC,SAAiB,EAAe,EAAE;QACjD,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;YACxB,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IACF,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,4BAA4B;QAC5B,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAc,CAAC,CAAC;gBAC3F,KAAK,MAAM,CAAC,IAAI,QAAQ;oBAAE,OAAO,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,gBAAgB;QAClB,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,IAA+B,CAAC;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,IAAK,CAAC,CAAC,EAAa,IAAI,cAAc,EAAE,CAAC;QAC5D,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,KAAK;YAAE,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM;YAAE,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACxC,IAAI,IAAI,CAAC,MAAM;YAAE,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;QAExE,MAAM,UAAU,GAAG,gBAAgB,CAAC,CAAU,CAAC,CAAC;QAChD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClD,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,QAAQ,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC;QACnD,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OAAO,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,QAAQ,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;YACpD,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,QAAQ,EAAE,CAAC,CAAC;YACrC,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACpE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,QAAQ,EAAE,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,YAAa,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,UAAU,CAAC,IAAe;IACjC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAoC,CAAC;IACzD,OAAO,OAAO,KAAK,EAAE,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACnF,CAAC"}

package/dist/src/core/judge-llm.d.ts

@@ -0,0 +1,13 @@
+/**
+ * packages/eval/src/core/judge-llm.ts - judge 模型装配（M5）
+ *
+ * 把 StreamFn + 框架 Model 装配成 llm-judge 需要的 complete(prompt)。
+ * 与 runner 的 live 装配对称：judge 模型也走 createLiveLlm / 注入 StreamFn，
+ * 单次非流式调用（内部仍消费事件流，聚合 text delta）。
+ */
+import type { Model, StreamFn } from '@aipack-ai/agent';
+import type { JudgeDeps } from './scorer';
+/** 单条消息 → complete(prompt) 的适配（temperature 0，judge 要确定性） */
+export declare function createCompleteFromStreamFn(streamFn: StreamFn, model: Model | undefined): (prompt: string) => Promise<string>;
+/** JudgeDeps 快捷装配 */
+export declare function createJudgeDeps(streamFn: StreamFn, model: Model | undefined, modelLabel?: string): JudgeDeps;

package/dist/src/core/judge-llm.js

@@ -0,0 +1,47 @@
+/**
+ * packages/eval/src/core/judge-llm.ts - judge 模型装配（M5）
+ *
+ * 把 StreamFn + 框架 Model 装配成 llm-judge 需要的 complete(prompt)。
+ * 与 runner 的 live 装配对称：judge 模型也走 createLiveLlm / 注入 StreamFn，
+ * 单次非流式调用（内部仍消费事件流，聚合 text delta）。
+ */
+/** 单条消息 → complete(prompt) 的适配（temperature 0，judge 要确定性） */
+export function createCompleteFromStreamFn(streamFn, model) {
+    return async (prompt) => {
+        const context = {
+            systemPrompt: '',
+            messages: [{ role: 'user', content: prompt, timestamp: Date.now() }],
+        };
+        let text = '';
+        const result = streamFn(model, context);
+        for await (const event of result) {
+            if (event.type === 'text_delta')
+                text += event.delta;
+            else if (event.type === 'done') {
+                // 兜底：done 消息里的 text（个别 provider 不发 text_delta）
+                if (!text) {
+                    const content = event.message.content;
+                    if (typeof content === 'string')
+                        text = content;
+                    else
+                        for (const block of content)
+                            if (block.type === 'text')
+                                text += block.text;
+                }
+                break;
+            }
+            else if (event.type === 'error') {
+                throw new Error('judge 流式调用错误');
+            }
+        }
+        return text;
+    };
+}
+/** JudgeDeps 快捷装配 */
+export function createJudgeDeps(streamFn, model, modelLabel) {
+    return {
+        complete: createCompleteFromStreamFn(streamFn, model),
+        ...(modelLabel ? { modelLabel } : {}),
+    };
+}
+//# sourceMappingURL=judge-llm.js.map

package/dist/src/core/judge-llm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-llm.js","sourceRoot":"","sources":["../../../src/core/judge-llm.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,4DAA4D;AAC5D,MAAM,UAAU,0BAA0B,CACxC,QAAkB,EAClB,KAAwB;IAExB,OAAO,KAAK,EAAE,MAAc,EAAmB,EAAE;QAC/C,MAAM,OAAO,GAAG;YACd,YAAY,EAAE,EAAE;YAChB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SAC9E,CAAC;QACF,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAc,EAAE,OAAO,CAAC,CAAC;QACjD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY;gBAAE,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC;iBAChD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC/B,+CAA+C;gBAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;oBACtC,IAAI,OAAO,OAAO,KAAK,QAAQ;wBAAE,IAAI,GAAG,OAAO,CAAC;;wBAC3C,KAAK,MAAM,KAAK,IAAI,OAAO;4BAAE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;gCAAE,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC;gBAClF,CAAC;gBACD,MAAM;YACR,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;AACJ,CAAC;AAED,qBAAqB;AACrB,MAAM,UAAU,eAAe,CAC7B,QAAkB,EAClB,KAAwB,EACxB,UAAmB;IAEnB,OAAO;QACL,QAAQ,EAAE,0BAA0B,CAAC,QAAQ,EAAE,KAAK,CAAC;QACrD,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtC,CAAC;AACJ,CAAC"}

package/dist/src/core/live.d.ts

@@ -75,3 +75,37 @@ export declare function resolveLiveLlm(opts?: ResolveLiveOptions): LiveLlm;
 export declare function describeLiveLlm(spec: LiveLlmSpec): string;
 /** live 模式前置检查：缺 Key 时给出可操作的错误信息 */
 export declare function assertLiveReady(spec: LiveLlmSpec): void;
+/** judge 模型的 env 覆盖键（与被测模型的键分离，避免混用） */
+export declare const JUDGE_ENV_KEYS: {
+    readonly model: "AIPACK_EVAL_JUDGE_MODEL";
+    readonly apiKey: "AIPACK_EVAL_JUDGE_API_KEY";
+    readonly baseUrl: "AIPACK_EVAL_JUDGE_BASE_URL";
+};
+/**
+ * 解析 judge 模型 spec：显式参数优先，缺省读 AIPACK_EVAL_JUDGE_MODEL。
+ * 复用 parseModelSpec（'provider/modelId' 或探测已配置 Key 的 provider）。
+ */
+export declare function resolveJudgeSpec(opts: {
+    model?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    timeoutMs?: number;
+}): LiveLlmSpec;
+/**
+ * 异源硬性约束（EVAL_PLAN.md 4.2）：judge 与被测模型完全相同 → 拒绝；
+ * 同 provider 不同模型 → 允许但告警（同源偏置风险低于同模型，仍建议跨 provider）。
+ */
+export declare function assertJudgeDistinct(target: {
+    provider: string;
+    modelId: string;
+} | undefined, judge: {
+    provider: string;
+    modelId: string;
+}): void;
+/** 从 CLI/库参数一步装配 judge LLM（complete 由调用方基于 streamFn 包装） */
+export declare function resolveJudgeLlm(opts: {
+    model?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    timeoutMs?: number;
+}): LiveLlm;

package/dist/src/core/live.js

@@ -139,4 +139,49 @@ export function assertLiveReady(spec) {
     const envName = BUILTIN_PROVIDERS.find((p) => p.id === spec.provider)?.envVar;
     throw new Error(`provider '${spec.provider}' 未配置 API Key：--api-key <key>、${LIVE_ENV_KEYS.apiKey} 或 ${envName ?? '<PROVIDER>_API_KEY'}`);
 }
+// ─── judge 装配（M5：LLM-as-judge）────────────────────────────────
+/** judge 模型的 env 覆盖键（与被测模型的键分离，避免混用） */
+export const JUDGE_ENV_KEYS = {
+    model: 'AIPACK_EVAL_JUDGE_MODEL',
+    apiKey: 'AIPACK_EVAL_JUDGE_API_KEY',
+    baseUrl: 'AIPACK_EVAL_JUDGE_BASE_URL',
+};
+/**
+ * 解析 judge 模型 spec：显式参数优先，缺省读 AIPACK_EVAL_JUDGE_MODEL。
+ * 复用 parseModelSpec（'provider/modelId' 或探测已配置 Key 的 provider）。
+ */
+export function resolveJudgeSpec(opts) {
+    const env = process.env;
+    const raw = opts.model ?? env[JUDGE_ENV_KEYS.model];
+    if (!raw) {
+        throw new Error(`llm-judge 需要 judge 模型：--judge-model provider/modelId 或 ${JUDGE_ENV_KEYS.model}（须与被测模型异源）`);
+    }
+    const { provider, modelId } = parseModelSpec(raw);
+    return {
+        provider,
+        modelId,
+        apiKey: opts.apiKey ?? env[JUDGE_ENV_KEYS.apiKey] ?? getEnvApiKey(provider),
+        baseUrl: opts.baseUrl ?? env[JUDGE_ENV_KEYS.baseUrl],
+        temperature: 0, // judge 要确定性
+        timeoutMs: opts.timeoutMs,
+    };
+}
+/**
+ * 异源硬性约束（EVAL_PLAN.md 4.2）：judge 与被测模型完全相同 → 拒绝；
+ * 同 provider 不同模型 → 允许但告警（同源偏置风险低于同模型，仍建议跨 provider）。
+ */
+export function assertJudgeDistinct(target, judge) {
+    if (!target)
+        return;
+    if (target.provider === judge.provider && target.modelId === judge.modelId) {
+        throw new Error(`judge 模型 (${judge.provider}/${judge.modelId}) 与被测模型相同：LLM-as-judge 要求异源（换 provider 或换 modelId）`);
+    }
+    if (target.provider === judge.provider) {
+        console.warn(`[eval] judge 与被测模型同 provider (${judge.provider})：存在同源偏置风险，建议跨 provider 选取 judge`);
+    }
+}
+/** 从 CLI/库参数一步装配 judge LLM（complete 由调用方基于 streamFn 包装） */
+export function resolveJudgeLlm(opts) {
+    return createLiveLlm(resolveJudgeSpec(opts));
+}
 //# sourceMappingURL=live.js.map

package/dist/src/core/live.js.map

@@ -1 +1 @@
-{"version":3,"file":"live.js","sourceRoot":"","sources":["../../../src/core/live.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EACL,iBAAiB,EACjB,YAAY,EACZ,oBAAoB,EACpB,eAAe,EACf,YAAY,GACb,MAAM,kBAAkB,CAAC;AAgC1B,4CAA4C;AAC5C,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,KAAK,EAAE,mBAAmB;IAC1B,QAAQ,EAAE,sBAAsB;IAChC,MAAM,EAAE,qBAAqB;IAC7B,OAAO,EAAE,sBAAsB;IAC/B,WAAW,EAAE,yBAAyB;CAC9B,CAAC;AAEX,gDAAgD;AAChD,MAAM,WAAW,GAAG;IAClB,UAAU;IACV,QAAQ;IACR,WAAW;IACX,MAAM;IACN,UAAU;IACV,QAAQ;IACR,YAAY;IACZ,SAAS;IACT,KAAK;IACL,UAAU;IACV,UAAU;IACV,WAAW;IACX,QAAQ;CACT,CAAC;AAEF,mEAAmE;AAEnE;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,OAAO;YACL,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;YACjC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;SAClC,CAAC;IACJ,CAAC;IACD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACxD,IAAI,WAAW;QAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IACpE,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,IAAI,UAAU;QAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAClE,MAAM,IAAI,KAAK,CACb,QAAQ,IAAI,4CAA4C,aAAa,CAAC,QAAQ,uBAAuB,CACtG,CAAC;AACJ,CAAC;AAWD,sCAAsC;AACtC,MAAM,UAAU,eAAe,CAAC,OAA2B,EAAE;IAC3D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IACxB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,4CAA4C,aAAa,CAAC,KAAK,4BAA4B,CAC5F,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAElD,MAAM,OAAO,GAAG,GAAG,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;IAC/C,MAAM,WAAW,GACf,IAAI,CAAC,WAAW,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAE9F,MAAM,IAAI,GAAgB;QACxB,QAAQ;QACR,OAAO;QACP,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC;QAC1E,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC;QACnD,WAAW,EAAE,WAAW,IAAI,CAAC;KAC9B,CAAC;IACF,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;QAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,mEAAmE;AAEnE,uCAAuC;AACvC,SAAS,gBAAgB,CAAC,QAAgB,EAAE,OAAe;IACzD,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,IAAI,EAAE,OAAO,CAAC;IAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,gBAAgB,QAAQ,YAAY,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpF,CAAC;IACJ,CAAC;IACD,OAAO;QACL,EAAE,EAAE,OAAO;QACX,IAAI,EAAE,OAAO;QACb,GAAG,EAAE,QAAQ,KAAK,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,oBAAoB;QAC3E,QAAQ;QACR,OAAO;QACP,SAAS,EAAE,KAAK;QAChB,KAAK,EAAE,CAAC,MAAM,CAAC;QACf,aAAa,EAAE,OAAO;QACtB,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,IAAiB;IAC7C,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY,IAAI;QAC3B,CAAC,CAAC,IAAI,CAAC,OAAO;YACZ,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;YACpC,CAAC,CAAC,IAAI;QACR,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAElD,MAAM,OAAO,GAAwB,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;IAC5E,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC9C,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;QAAE,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAErE,OAAO;QACL,IAAI;QACJ,OAAO;QACP,KAAK,EAAE,YAAY,CAAC,OAAO,CAAC;QAC5B,QAAQ,EAAE,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,cAAc,CAAC,OAA2B,EAAE;IAC1D,OAAO,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,eAAe,CAAC,IAAiB;IAC/C,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;AAC5C,CAAC;AAED,oCAAoC;AACpC,MAAM,UAAU,eAAe,CAAC,IAAiB;IAC/C,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO;IACxB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC9E,MAAM,IAAI,KAAK,CACb,aAAa,IAAI,CAAC,QAAQ,iCAAiC,aAAa,CAAC,MAAM,MAAM,OAAO,IAAI,oBAAoB,EAAE,CACvH,CAAC;AACJ,CAAC"}
+{"version":3,"file":"live.js","sourceRoot":"","sources":["../../../src/core/live.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EACL,iBAAiB,EACjB,YAAY,EACZ,oBAAoB,EACpB,eAAe,EACf,YAAY,GACb,MAAM,kBAAkB,CAAC;AAgC1B,4CAA4C;AAC5C,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,KAAK,EAAE,mBAAmB;IAC1B,QAAQ,EAAE,sBAAsB;IAChC,MAAM,EAAE,qBAAqB;IAC7B,OAAO,EAAE,sBAAsB;IAC/B,WAAW,EAAE,yBAAyB;CAC9B,CAAC;AAEX,gDAAgD;AAChD,MAAM,WAAW,GAAG;IAClB,UAAU;IACV,QAAQ;IACR,WAAW;IACX,MAAM;IACN,UAAU;IACV,QAAQ;IACR,YAAY;IACZ,SAAS;IACT,KAAK;IACL,UAAU;IACV,UAAU;IACV,WAAW;IACX,QAAQ;CACT,CAAC;AAEF,mEAAmE;AAEnE;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,OAAO;YACL,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;YACjC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;SAClC,CAAC;IACJ,CAAC;IACD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACxD,IAAI,WAAW;QAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IACpE,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,IAAI,UAAU;QAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAClE,MAAM,IAAI,KAAK,CACb,QAAQ,IAAI,4CAA4C,aAAa,CAAC,QAAQ,uBAAuB,CACtG,CAAC;AACJ,CAAC;AAWD,sCAAsC;AACtC,MAAM,UAAU,eAAe,CAAC,OAA2B,EAAE;IAC3D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IACxB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,4CAA4C,aAAa,CAAC,KAAK,4BAA4B,CAC5F,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAElD,MAAM,OAAO,GAAG,GAAG,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;IAC/C,MAAM,WAAW,GACf,IAAI,CAAC,WAAW,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAE9F,MAAM,IAAI,GAAgB;QACxB,QAAQ;QACR,OAAO;QACP,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC;QAC1E,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC;QACnD,WAAW,EAAE,WAAW,IAAI,CAAC;KAC9B,CAAC;IACF,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;QAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,mEAAmE;AAEnE,uCAAuC;AACvC,SAAS,gBAAgB,CAAC,QAAgB,EAAE,OAAe;IACzD,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,IAAI,EAAE,OAAO,CAAC;IAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,gBAAgB,QAAQ,YAAY,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpF,CAAC;IACJ,CAAC;IACD,OAAO;QACL,EAAE,EAAE,OAAO;QACX,IAAI,EAAE,OAAO;QACb,GAAG,EAAE,QAAQ,KAAK,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,oBAAoB;QAC3E,QAAQ;QACR,OAAO;QACP,SAAS,EAAE,KAAK;QAChB,KAAK,EAAE,CAAC,MAAM,CAAC;QACf,aAAa,EAAE,OAAO;QACtB,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,IAAiB;IAC7C,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY,IAAI;QAC3B,CAAC,CAAC,IAAI,CAAC,OAAO;YACZ,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;YACpC,CAAC,CAAC,IAAI;QACR,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAElD,MAAM,OAAO,GAAwB,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;IAC5E,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC9C,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;QAAE,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAErE,OAAO;QACL,IAAI;QACJ,OAAO;QACP,KAAK,EAAE,YAAY,CAAC,OAAO,CAAC;QAC5B,QAAQ,EAAE,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,cAAc,CAAC,OAA2B,EAAE;IAC1D,OAAO,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,eAAe,CAAC,IAAiB;IAC/C,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;AAC5C,CAAC;AAED,oCAAoC;AACpC,MAAM,UAAU,eAAe,CAAC,IAAiB;IAC/C,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO;IACxB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC9E,MAAM,IAAI,KAAK,CACb,aAAa,IAAI,CAAC,QAAQ,iCAAiC,aAAa,CAAC,MAAM,MAAM,OAAO,IAAI,oBAAoB,EAAE,CACvH,CAAC;AACJ,CAAC;AAED,gEAAgE;AAEhE,wCAAwC;AACxC,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,KAAK,EAAE,yBAAyB;IAChC,MAAM,EAAE,2BAA2B;IACnC,OAAO,EAAE,4BAA4B;CAC7B,CAAC;AAEX;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAKhC;IACC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IACxB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IACpD,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,0DAA0D,cAAc,CAAC,KAAK,YAAY,CAC3F,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAClD,OAAO;QACL,QAAQ;QACR,OAAO;QACP,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC;QAC3E,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,cAAc,CAAC,OAAO,CAAC;QACpD,WAAW,EAAE,CAAC,EAAE,aAAa;QAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAyD,EACzD,KAA4C;IAE5C,IAAI,CAAC,MAAM;QAAE,OAAO;IACpB,IAAI,MAAM,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CACb,aAAa,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,oDAAoD,CACjG,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,EAAE,CAAC;QACvC,OAAO,CAAC,IAAI,CACV,iCAAiC,KAAK,CAAC,QAAQ,kCAAkC,CAClF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,eAAe,CAAC,IAK/B;IACC,OAAO,aAAa,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;AAC/C,CAAC"}

package/dist/src/core/runner.js

@@ -18,7 +18,9 @@ import { createMockStreamFn } from './mock-stream.js';
 import { createMockTools } from './mock-tools.js';
 import { countTurns, extractTrajectory } from './trajectory.js';
 import { resolveScorers, scoreTrace } from './scorer/index.js';
-import { assertLiveReady, describeLiveLlm, resolveLiveLlm, } from './live.js';
+import { createOpenAiCompatibleEmbedFn } from './embedding.js';
+import { createJudgeDeps } from './judge-llm.js';
+import { assertJudgeDistinct, assertLiveReady, describeLiveLlm, parseModelSpec, resolveJudgeLlm, resolveLiveLlm, } from './live.js';
 const DEFAULT_TIMEOUT_MS = 30_000;
 const DEFAULT_MAX_STEPS = 50;
 const DEFAULT_CONCURRENCY = 8;
@@ -28,6 +30,15 @@ const LIVE_DEFAULT_REPEATS = 3;
 function sessionKeyFor(caseId, repeat) {
     return `eval:${caseId.replace(/[^a-zA-Z0-9_-]/g, '_')}#r${repeat}`;
 }
+/** bundle.label（'provider/modelId'）→ spec；解析失败不阻断（mock / 注入 streamFn 场景） */
+function parseModelSpecSafe(label) {
+    try {
+        return parseModelSpec(label);
+    }
+    catch {
+        return undefined;
+    }
+}
 /**
  * 模式不匹配 → 跳过（不计入通过率，避免出现"必然失败"的噪音）。
  * 返回 undefined 表示该用例在当前模式下可跑。
@@ -135,8 +146,38 @@ async function runCaseOnce(c, config, repeat, bundle) {
         await runtime.close().catch(() => undefined);
     }
 }
+/**
+ * M5：按需装配评分器依赖（lazy 检测——只有用例真的用了 semantic /
+ * llm-judge 才装配，配置缺失时抛可操作错误，由 CLI 转为用法错误）。
+ */
+function resolveScorerDeps(config, needs, targetSpec) {
+    const deps = {};
+    if (needs.judge) {
+        const judge = config.judgeStreamFn
+            ? { streamFn: config.judgeStreamFn, model: config.judgeFrameworkModel }
+            : (() => {
+                const llm = resolveJudgeLlm({
+                    model: config.judgeModel,
+                    apiKey: config.judgeApiKey,
+                    baseUrl: config.judgeBaseUrl,
+                    timeoutMs: config.requestTimeoutMs,
+                });
+                assertJudgeDistinct(targetSpec, llm.spec);
+                return llm;
+            })();
+        deps.judge = createJudgeDeps(judge.streamFn, judge.model, config.judgeStreamFn ? 'injected-judge' : undefined);
+    }
+    if (needs.embed) {
+        deps.embed = createOpenAiCompatibleEmbedFn({
+            model: config.embedModel,
+            apiKey: config.embedApiKey,
+            baseUrl: config.embedBaseUrl,
+        });
+    }
+    return deps;
+}
 /** 单用例（含 repeats） */
-async function runCase(c, config, bundle, budget) {
+async function runCase(c, config, bundle, budget, scorerDeps) {
     const isLive = config.mode === 'live';
     const repeats = c.metadata?.repeats ?? config.repeats ?? (isLive ? LIVE_DEFAULT_REPEATS : 1);
     const scorers = resolveScorers(c.expected, c.scorers);
@@ -178,7 +219,7 @@ async function runCase(c, config, bundle, budget) {
         lastTraceUsage = trace.usageTotal;
         usageTotal += trace.usageTotal;
         budget.spent += trace.usageTotal;
-        const caseScore = scoreTrace(trace, scorers);
+        const caseScore = await scoreTrace(trace, scorers, scorerDeps);
         scoreSum += caseScore.score;
         scoreCount += 1;
         if (caseScore.passed)
@@ -275,7 +316,10 @@ export async function runEval(cases, config) {
     });
     const concurrency = config.concurrency ?? DEFAULT_CONCURRENCY;
     const budget = { spent: 0 };
-    const results = await runAllConcurrent(runnable, concurrency, (c) => runCase(c, config, bundle, budget));
+    // M5：只有用例真的使用 semantic / llm-judge 才装配对应依赖（配置缺失抛错 → CLI 用法错误）
+    const usesScorer = (type) => runnable.some((c) => resolveScorers(c.expected, c.scorers).some((s) => s.type === type));
+    const scorerDeps = resolveScorerDeps(config, { judge: usesScorer('llm-judge'), embed: usesScorer('semantic') }, bundle?.label ? parseModelSpecSafe(bundle.label) : undefined);
+    const results = await runAllConcurrent(runnable, concurrency, (c) => runCase(c, config, bundle, budget, scorerDeps));
     const casesPassed = results.filter((r) => r.passed).length;
     return {
         runId: `run_${new Date().toISOString().replace(/[:.]/g, '-')}`,

package/dist/src/core/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/core/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEhE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtD,OAAO,EACL,eAAe,EACf,eAAe,EACf,cAAc,GACf,MAAM,QAAQ,CAAC;AAWhB,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,0CAA0C;AAC1C,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAe/B,yCAAyC;AACzC,SAAS,aAAa,CAAC,MAAc,EAAE,MAAc;IACnD,OAAO,QAAQ,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,KAAK,MAAM,EAAE,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,CAAW,EAAE,IAAqB;IAC3D,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,yBAAyB,CAAC;IAC3E,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,yBAAyB,CAAC;IAC3E,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI;QAAE,OAAO,8BAA8B,CAAC;IAC3E,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,iDAAiD;AACjD,SAAS,oBAAoB,CAAC,MAAiB;IAC7C,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,SAAS,CAAC;IAC7C,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;IACrE,CAAC;IACD,MAAM,GAAG,GAAG,cAAc,CAAC;QACzB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,SAAS,EAAE,MAAM,CAAC,gBAAgB;KACnC,CAAC,CAAC;IACH,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1B,OAAO;QACL,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,oCAAoC;AACpC,KAAK,UAAU,WAAW,CACxB,CAAW,EACX,MAAiB,EACjB,MAAc,EACd,MAAsB;IAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;IACtC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,OAAO,EAAE,KAAK,EAAE,+CAA+C,EAAE,CAAC;IACpE,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAClF,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,IAAI,MAAM,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IAE9E,MAAM,KAAK,GAAW,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACvE,MAAM,OAAO,GAAG,aAAa,CAAC;QAC5B,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE;QACxC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,MAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,IAAK,CAAC;QACvE,GAAG,CAAC,MAAM,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,KAAK;QACL,QAAQ,EAAE,QAAQ;QAClB,gBAAgB,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC;KACpD,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAa7B,IAAI,GAAwB,CAAC;IAE7B,IAAI,CAAC;QACH,GAAG,GAAG,OAAO;aACV,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;aACnD,IAAI,CAAC,CAAC,MAAM,EAAc,EAAE;YAC3B,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACnC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAC1C,MAAM,UAAU,GACd,OAAO,MAAM,CAAC,KAAK,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnE,OAAO;gBACL,EAAE,EAAE,IAAI;gBACR,MAAM;gBACN,QAAQ;gBACR,UAAU;gBACV,KAAK;gBACL,UAAU;gBACV,UAAU;aACX,CAAC;QACJ,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,CAAU,EAAc,EAAE,CAAC,CAAC;YAClC,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,SAAU,CAAW,CAAC,OAAO,EAAE;SACvC,CAAC,CAAC,CAAC;QAEN,kCAAkC;QAClC,MAAM,cAAc,GAAG,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,EAAE;YACzD,MAAM,CAAC,GAAG,UAAU,CAClB,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,KAAK,EAAE,CAAC,EACzD,SAAS,CACV,CAAC;YACD,CAA4B,CAAC,KAAK,EAAE,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,OAAO,CAAC,EAAE;YAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;QAEjD,MAAM,KAAK,GAAa;YACtB,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC;QAEF,qBAAqB;QACrB,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC;QACxC,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;YAC5D,OAAO,EAAE,KAAK,EAAE,oBAAoB,KAAK,CAAC,UAAU,gBAAgB,SAAS,EAAE,EAAE,CAAC;QACpF,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,CAAC;IACnB,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,qBAAqB;AACrB,KAAK,UAAU,OAAO,CACpB,CAAW,EACX,MAAiB,EACjB,MAAiC,EACjC,MAAmB;IAEnB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;IACtC,MAAM,OAAO,GACX,CAAC,CAAC,QAAQ,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAEtD,4BAA4B;IAC5B,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;QAChF,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,EAAE;YACV,KAAK,EAAE,wBAAwB,MAAM,CAAC,KAAK,qBAAqB,MAAM,CAAC,cAAc,EAAE;YACvF,UAAU,EAAE,CAAC;YACb,KAAK,EAAE,CAAC;YACR,UAAU,EAAE,CAAC;YACb,OAAO;YACP,SAAS,EAAE,CAAC;SACb,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,YAAY,GAAkB,EAAE,CAAC;IACvC,IAAI,QAA4B,CAAC;IACjC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QACjC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACjE,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;QAE9B,IAAI,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;YACpB,QAAQ,GAAG,QAAQ,IAAI,KAAK,CAAC;YAC7B,SAAS;QACX,CAAC;QAED,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;QAChC,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC;QAClC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;QAC/B,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;QAEjC,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7C,QAAQ,IAAI,SAAS,CAAC,KAAK,CAAC;QAC5B,UAAU,IAAI,CAAC,CAAC;QAChB,IAAI,SAAS,CAAC,MAAM;YAAE,SAAS,IAAI,CAAC,CAAC;;YAChC,YAAY,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,UAAU,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;QACjC,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE;gBACN,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,QAAQ,EAAE,EAAE;aAC5E;YACD,KAAK,EAAE,QAAQ;YACf,UAAU;YACV,KAAK;YACL,UAAU;YACV,OAAO;YACP,SAAS,EAAE,CAAC;SACb,CAAC;IACJ,CAAC;IAED,qCAAqC;IACrC,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,KAAK,UAAU,IAAI,UAAU,GAAG,CAAC,CAAC;IACnF,IAAI,UAAU,GAAG,OAAO,EAAE,CAAC;QACzB,QAAQ,GAAG,QAAQ,IAAI,GAAG,OAAO,GAAG,UAAU,IAAI,OAAO,QAAQ,CAAC;IACpE,CAAC;IAED,OAAO;QACL,MAAM,EAAE,CAAC,CAAC,EAAE;QACZ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,MAAM;QACN,KAAK,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,UAAU;QACnD,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY;QAClC,KAAK,EAAE,QAAQ;QACf,UAAU;QACV,KAAK;QACL,UAAU,EAAE,UAAU,IAAI,cAAc;QACxC,OAAO;QACP,SAAS;KACV,CAAC;AACJ,CAAC;AAED,qBAAqB;AACrB,KAAK,UAAU,gBAAgB,CAC7B,KAAU,EACV,WAAmB,EACnB,EAA2B;IAE3B,MAAM,OAAO,GAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACrF,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,OAAqB;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,MAAM,GAAG,GAAiC,EAAE,CAAC;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACnD,GAAG,CAAC,GAAG,CAAC,GAAG;YACT,KAAK;YACL,MAAM;YACN,QAAQ,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK;YAC1C,QAAQ,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,KAAK;SAC1E,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,aAAa;AACb,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAiB,EACjB,MAAiB;IAEjB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAE5C,MAAM,OAAO,GACX,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QACvC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACvD,CAAC,CAAC,KAAK,CAAC;IAEZ,MAAM,OAAO,GAAuC,EAAE,CAAC;IACvD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,mBAAmB,CAAC;IAC9D,MAAM,MAAM,GAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAClE,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CACnC,CAAC;IAEF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,OAAO;QACL,KAAK,EAAE,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE;QAC9D,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;QAC5C,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3E,MAAM,EAAE;YACN,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM,EAAE,WAAW;YACnB,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM;YACjE,QAAQ,EACN,OAAO,CAAC,MAAM,KAAK,CAAC;gBAClB,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;YAC/D,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC;QAC3B,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACpE,OAAO;QACP,OAAO;KACR,CAAC;AACJ,CAAC"}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/core/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEhE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAmB,MAAM,UAAU,CAAC;AACvE,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,eAAe,EACf,cAAc,EACd,eAAe,EACf,cAAc,GACf,MAAM,QAAQ,CAAC;AAWhB,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,0CAA0C;AAC1C,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAe/B,yCAAyC;AACzC,SAAS,aAAa,CAAC,MAAc,EAAE,MAAc;IACnD,OAAO,QAAQ,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,KAAK,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,4EAA4E;AAC5E,SAAS,kBAAkB,CAAC,KAAa;IACvC,IAAI,CAAC;QACH,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,CAAW,EAAE,IAAqB;IAC3D,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,yBAAyB,CAAC;IAC3E,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,yBAAyB,CAAC;IAC3E,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI;QAAE,OAAO,8BAA8B,CAAC;IAC3E,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,iDAAiD;AACjD,SAAS,oBAAoB,CAAC,MAAiB;IAC7C,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,SAAS,CAAC;IAC7C,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;IACrE,CAAC;IACD,MAAM,GAAG,GAAG,cAAc,CAAC;QACzB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,SAAS,EAAE,MAAM,CAAC,gBAAgB;KACnC,CAAC,CAAC;IACH,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1B,OAAO;QACL,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,oCAAoC;AACpC,KAAK,UAAU,WAAW,CACxB,CAAW,EACX,MAAiB,EACjB,MAAc,EACd,MAAsB;IAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;IACtC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,OAAO,EAAE,KAAK,EAAE,+CAA+C,EAAE,CAAC;IACpE,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAClF,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,IAAI,MAAM,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IAE9E,MAAM,KAAK,GAAW,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACvE,MAAM,OAAO,GAAG,aAAa,CAAC;QAC5B,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE;QACxC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,MAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,IAAK,CAAC;QACvE,GAAG,CAAC,MAAM,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,KAAK;QACL,QAAQ,EAAE,QAAQ;QAClB,gBAAgB,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC;KACpD,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAa7B,IAAI,GAAwB,CAAC;IAE7B,IAAI,CAAC;QACH,GAAG,GAAG,OAAO;aACV,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;aACnD,IAAI,CAAC,CAAC,MAAM,EAAc,EAAE;YAC3B,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACnC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAC1C,MAAM,UAAU,GACd,OAAO,MAAM,CAAC,KAAK,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnE,OAAO;gBACL,EAAE,EAAE,IAAI;gBACR,MAAM;gBACN,QAAQ;gBACR,UAAU;gBACV,KAAK;gBACL,UAAU;gBACV,UAAU;aACX,CAAC;QACJ,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,CAAU,EAAc,EAAE,CAAC,CAAC;YAClC,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,SAAU,CAAW,CAAC,OAAO,EAAE;SACvC,CAAC,CAAC,CAAC;QAEN,kCAAkC;QAClC,MAAM,cAAc,GAAG,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,EAAE;YACzD,MAAM,CAAC,GAAG,UAAU,CAClB,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,KAAK,EAAE,CAAC,EACzD,SAAS,CACV,CAAC;YACD,CAA4B,CAAC,KAAK,EAAE,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,OAAO,CAAC,EAAE;YAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;QAEjD,MAAM,KAAK,GAAa;YACtB,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC;QAEF,qBAAqB;QACrB,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC;QACxC,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;YAC5D,OAAO,EAAE,KAAK,EAAE,oBAAoB,KAAK,CAAC,UAAU,gBAAgB,SAAS,EAAE,EAAE,CAAC;QACpF,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,CAAC;IACnB,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CACxB,MAAiB,EACjB,KAAyC,EACzC,UAAkD;IAElD,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa;YAChC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,aAAa,EAAE,KAAK,EAAE,MAAM,CAAC,mBAAmB,EAAE;YACvE,CAAC,CAAC,CAAC,GAAG,EAAE;gBACJ,MAAM,GAAG,GAAG,eAAe,CAAC;oBAC1B,KAAK,EAAE,MAAM,CAAC,UAAU;oBACxB,MAAM,EAAE,MAAM,CAAC,WAAW;oBAC1B,OAAO,EAAE,MAAM,CAAC,YAAY;oBAC5B,SAAS,EAAE,MAAM,CAAC,gBAAgB;iBACnC,CAAC,CAAC;gBACH,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC1C,OAAO,GAAG,CAAC;YACb,CAAC,CAAC,EAAE,CAAC;QACT,IAAI,CAAC,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACjH,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,IAAI,CAAC,KAAK,GAAG,6BAA6B,CAAC;YACzC,KAAK,EAAE,MAAM,CAAC,UAAU;YACxB,MAAM,EAAE,MAAM,CAAC,WAAW;YAC1B,OAAO,EAAE,MAAM,CAAC,YAAY;SAC7B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,qBAAqB;AACrB,KAAK,UAAU,OAAO,CACpB,CAAW,EACX,MAAiB,EACjB,MAAiC,EACjC,MAAmB,EACnB,UAAsB;IAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;IACtC,MAAM,OAAO,GACX,CAAC,CAAC,QAAQ,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAEtD,4BAA4B;IAC5B,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;QAChF,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,EAAE;YACV,KAAK,EAAE,wBAAwB,MAAM,CAAC,KAAK,qBAAqB,MAAM,CAAC,cAAc,EAAE;YACvF,UAAU,EAAE,CAAC;YACb,KAAK,EAAE,CAAC;YACR,UAAU,EAAE,CAAC;YACb,OAAO;YACP,SAAS,EAAE,CAAC;SACb,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,YAAY,GAAkB,EAAE,CAAC;IACvC,IAAI,QAA4B,CAAC;IACjC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QACjC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACjE,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;QAE9B,IAAI,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;YACpB,QAAQ,GAAG,QAAQ,IAAI,KAAK,CAAC;YAC7B,SAAS;QACX,CAAC;QAED,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;QAChC,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC;QAClC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;QAC/B,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;QAEjC,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;QAC/D,QAAQ,IAAI,SAAS,CAAC,KAAK,CAAC;QAC5B,UAAU,IAAI,CAAC,CAAC;QAChB,IAAI,SAAS,CAAC,MAAM;YAAE,SAAS,IAAI,CAAC,CAAC;;YAChC,YAAY,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,UAAU,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;QACjC,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE;gBACN,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,QAAQ,EAAE,EAAE;aAC5E;YACD,KAAK,EAAE,QAAQ;YACf,UAAU;YACV,KAAK;YACL,UAAU;YACV,OAAO;YACP,SAAS,EAAE,CAAC;SACb,CAAC;IACJ,CAAC;IAED,qCAAqC;IACrC,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,KAAK,UAAU,IAAI,UAAU,GAAG,CAAC,CAAC;IACnF,IAAI,UAAU,GAAG,OAAO,EAAE,CAAC;QACzB,QAAQ,GAAG,QAAQ,IAAI,GAAG,OAAO,GAAG,UAAU,IAAI,OAAO,QAAQ,CAAC;IACpE,CAAC;IAED,OAAO;QACL,MAAM,EAAE,CAAC,CAAC,EAAE;QACZ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,MAAM;QACN,KAAK,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,UAAU;QACnD,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY;QAClC,KAAK,EAAE,QAAQ;QACf,UAAU;QACV,KAAK;QACL,UAAU,EAAE,UAAU,IAAI,cAAc;QACxC,OAAO;QACP,SAAS;KACV,CAAC;AACJ,CAAC;AAED,qBAAqB;AACrB,KAAK,UAAU,gBAAgB,CAC7B,KAAU,EACV,WAAmB,EACnB,EAA2B;IAE3B,MAAM,OAAO,GAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACrF,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,OAAqB;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,MAAM,GAAG,GAAiC,EAAE,CAAC;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACnD,GAAG,CAAC,GAAG,CAAC,GAAG;YACT,KAAK;YACL,MAAM;YACN,QAAQ,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK;YAC1C,QAAQ,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,KAAK;SAC1E,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,aAAa;AACb,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAiB,EACjB,MAAiB;IAEjB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAE5C,MAAM,OAAO,GACX,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QACvC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACvD,CAAC,CAAC,KAAK,CAAC;IAEZ,MAAM,OAAO,GAAuC,EAAE,CAAC;IACvD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,mBAAmB,CAAC;IAC9D,MAAM,MAAM,GAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAEzC,8DAA8D;IAC9D,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,EAAE,CAClC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;IAC3F,MAAM,UAAU,GAAG,iBAAiB,CAClC,MAAM,EACN,EAAE,KAAK,EAAE,UAAU,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,EACjE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAC7D,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAClE,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAC/C,CAAC;IAEF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,OAAO;QACL,KAAK,EAAE,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE;QAC9D,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;QAC5C,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3E,MAAM,EAAE;YACN,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM,EAAE,WAAW;YACnB,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM;YACjE,QAAQ,EACN,OAAO,CAAC,MAAM,KAAK,CAAC;gBAClB,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;YAC/D,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC;QAC3B,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACpE,OAAO;QACP,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/src/core/scorer/index.d.ts

@@ -2,10 +2,26 @@
  * packages/eval/src/core/scorer/index.ts - 评分入口与 expected 归一化
  *
  * - expected（声明式）→ ScorerConfig[]（与显式 scorers 合并）
- * - scoreCase：依次执行规则评分器，加权合成用例总分
+ * - scoreCase：按类型分发评分器（rule 同步 / semantic & llm-judge 异步），
+ *   加权合成用例总分
+ * - M5：semantic / llm-judge 需要外部装配（embed / judge），缺装配时
+ *   评分器跳过（不判用例失败，见各实现文件）
  */
 import type { ExpectedResult, RunTrace, ScoreResult, ScorerConfig } from '../types';
+import { type EmbedFn } from './semantic';
+import { type JudgeDeps } from './llm-judge';
 export { RULE_SCORER_TYPES, runRuleScorer, partialMatch } from './rule';
+export { cosineSimilarity, runSemanticScorer, type EmbedFn, type SemanticParams } from './semantic';
+export { buildJudgePrompt, parseJudgeResponse, runLlmJudge, JUDGE_PROMPT_VERSION, type JudgeDeps, type JudgeParams, type JudgeVerdict, } from './llm-judge';
+/** 全量评分器类型（rule 8 种 + semantic + llm-judge） */
+export declare const ALL_SCORER_TYPES: readonly [...string[], "semantic", "llm-judge"];
+/** 评分器外部装配（M5）：缺省时 semantic / llm-judge 跳过 */
+export interface ScorerDeps {
+    /** semantic 评分器的 embedding 函数 */
+    embed?: EmbedFn;
+    /** llm-judge 的异源 judge 装配 */
+    judge?: JudgeDeps;
+}
 /** 声明式 expected → 评分器配置 */
 export declare function expectedToScorers(expected: ExpectedResult): ScorerConfig[];
 /** 合并用例的 expected + scorers */
@@ -14,6 +30,8 @@ export interface CaseScore {
     passed: boolean;
     score: number;
     scores: ScoreResult[];
+    /** 因缺装配 / judge 解析失败而跳过的评分器（不判用例失败） */
+    skippedScorers: string[];
 }
 /** 对一次运行轨迹执行全部评分器 */
-export declare function scoreTrace(trace: RunTrace, scorers: ScorerConfig[]): CaseScore;
+export declare function scoreTrace(trace: RunTrace, scorers: ScorerConfig[], deps?: ScorerDeps): Promise<CaseScore>;

package/dist/src/core/scorer/index.js

@@ -2,10 +2,19 @@
  * packages/eval/src/core/scorer/index.ts - 评分入口与 expected 归一化
  *
  * - expected（声明式）→ ScorerConfig[]（与显式 scorers 合并）
- * - scoreCase：依次执行规则评分器，加权合成用例总分
+ * - scoreCase：按类型分发评分器（rule 同步 / semantic & llm-judge 异步），
+ *   加权合成用例总分
+ * - M5：semantic / llm-judge 需要外部装配（embed / judge），缺装配时
+ *   评分器跳过（不判用例失败，见各实现文件）
  */
-import { runRuleScorer } from './rule.js';
+import { RULE_SCORER_TYPES, runRuleScorer } from './rule.js';
+import { runSemanticScorer } from './semantic.js';
+import { runLlmJudge } from './llm-judge.js';
 export { RULE_SCORER_TYPES, runRuleScorer, partialMatch } from './rule.js';
+export { cosineSimilarity, runSemanticScorer } from './semantic.js';
+export { buildJudgePrompt, parseJudgeResponse, runLlmJudge, JUDGE_PROMPT_VERSION, } from './llm-judge.js';
+/** 全量评分器类型（rule 8 种 + semantic + llm-judge） */
+export const ALL_SCORER_TYPES = [...RULE_SCORER_TYPES, 'semantic', 'llm-judge'];
 /** 声明式 expected → 评分器配置 */
 export function expectedToScorers(expected) {
     switch (expected.type) {
@@ -39,18 +48,36 @@ export function expectedToScorers(expected) {
 export function resolveScorers(expected, explicit) {
     return [...(expected ? expectedToScorers(expected) : []), ...(explicit ?? [])];
 }
+/** 执行单个评分器（按类型分发）；返回 undefined = 跳过 */
+async function runScorer(trace, s, deps) {
+    if (s.type === 'semantic') {
+        return runSemanticScorer(trace, s.params, deps.embed);
+    }
+    if (s.type === 'llm-judge') {
+        return runLlmJudge(trace, s.params, deps.judge);
+    }
+    return runRuleScorer(trace, s.type, s.params);
+}
 /** 对一次运行轨迹执行全部评分器 */
-export function scoreTrace(trace, scorers) {
-    const scores = scorers.map((s) => runRuleScorer(trace, s.type, s.params));
-    const totalWeight = scorers.reduce((acc, s) => acc + (s.weight ?? 1), 0);
+export async function scoreTrace(trace, scorers, deps = {}) {
+    const results = await Promise.all(scorers.map((s) => runScorer(trace, s, deps)));
+    // 跳过的评分器（缺装配 / scorer 自身错误）不参与判定
+    const skippedScorers = scorers
+        .filter((_, i) => results[i] === undefined)
+        .map((s) => s.type);
+    const effective = scorers
+        .map((s, i) => ({ s, r: results[i] }))
+        .filter((x) => x.r !== undefined);
+    const totalWeight = effective.reduce((acc, { s }) => acc + (s.weight ?? 1), 0);
     const weighted = totalWeight === 0
         ? 0
-        : scores.reduce((acc, r, i) => acc + r.score * (scorers[i].weight ?? 1), 0) /
+        : effective.reduce((acc, { s, r }) => acc + r.score * (s.weight ?? 1), 0) /
             totalWeight;
     return {
-        passed: scores.length > 0 && scores.every((r) => r.passed),
-        score: scorers.length === 0 ? 0 : weighted,
-        scores,
+        passed: effective.length > 0 && effective.every(({ r }) => r.passed),
+        score: effective.length === 0 ? 0 : weighted,
+        scores: effective.map(({ r }) => r),
+        skippedScorers,
     };
 }
 //# sourceMappingURL=index.js.map

package/dist/src/core/scorer/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/core/scorer/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,OAAO,EAAqB,aAAa,EAAE,MAAM,QAAQ,CAAC;AAE1D,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAExE,2BAA2B;AAC3B,MAAM,UAAU,iBAAiB,CAAC,QAAwB;IACxD,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,OAAO,CAAC;QACb,KAAK,UAAU;YACb,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtE,KAAK,OAAO;YACV,OAAO;gBACL,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE;aAClF,CAAC;QACJ,KAAK,WAAW;YACd,OAAO;gBACL;oBACE,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,QAAQ,EAAE;iBACrE;aACF,CAAC;QACJ,KAAK,YAAY;YACf,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrE,KAAK,YAAY;YACf,OAAO,OAAO,IAAI,QAAQ;gBACxB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;gBAClF,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAChE,KAAK,SAAS;YACZ,OAAO,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1E,KAAK,aAAa;YAChB,OAAO,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,+BAA+B;AAC/B,MAAM,UAAU,cAAc,CAC5B,QAAyB,EACzB,QAAyB;IAEzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AACjF,CAAC;AAQD,qBAAqB;AACrB,MAAM,UAAU,UAAU,CAAC,KAAe,EAAE,OAAuB;IACjE,MAAM,MAAM,GAAkB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9C,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CACvC,CAAC;IACF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,QAAQ,GACZ,WAAW,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YACzE,WAAW,CAAC;IAClB,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1D,KAAK,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ;QAC1C,MAAM;KACP,CAAC;AACJ,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/core/scorer/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAgB,MAAM,YAAY,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAkB,MAAM,aAAa,CAAC;AAE1D,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAqC,MAAM,YAAY,CAAC;AACpG,OAAO,EACL,gBAAgB,EAChB,kBAAkB,EAClB,WAAW,EACX,oBAAoB,GAIrB,MAAM,aAAa,CAAC;AAErB,+CAA+C;AAC/C,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,EAAE,UAAU,EAAE,WAAW,CAAU,CAAC;AAUzF,2BAA2B;AAC3B,MAAM,UAAU,iBAAiB,CAAC,QAAwB;IACxD,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,OAAO,CAAC;QACb,KAAK,UAAU;YACb,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtE,KAAK,OAAO;YACV,OAAO;gBACL,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE;aAClF,CAAC;QACJ,KAAK,WAAW;YACd,OAAO;gBACL;oBACE,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,QAAQ,EAAE;iBACrE;aACF,CAAC;QACJ,KAAK,YAAY;YACf,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrE,KAAK,YAAY;YACf,OAAO,OAAO,IAAI,QAAQ;gBACxB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;gBAClF,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAChE,KAAK,SAAS;YACZ,OAAO,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1E,KAAK,aAAa;YAChB,OAAO,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,+BAA+B;AAC/B,MAAM,UAAU,cAAc,CAC5B,QAAyB,EACzB,QAAyB;IAEzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AACjF,CAAC;AAUD,uCAAuC;AACvC,KAAK,UAAU,SAAS,CACtB,KAAe,EACf,CAAe,EACf,IAAgB;IAEhB,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC1B,OAAO,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,MAAe,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC3B,OAAO,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC,MAAe,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;AAChD,CAAC;AAED,qBAAqB;AACrB,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAe,EACf,OAAuB,EACvB,OAAmB,EAAE;IAErB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAEjF,iCAAiC;IACjC,MAAM,cAAc,GAAG,OAAO;SAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC;SAC1C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtB,MAAM,SAAS,GAAG,OAAO;SACtB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SACrC,MAAM,CAAC,CAAC,CAAC,EAA4C,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAE9E,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/E,MAAM,QAAQ,GACZ,WAAW,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YACvE,WAAW,CAAC;IAClB,OAAO;QACL,MAAM,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QACpE,KAAK,EAAE,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ;QAC5C,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QACnC,cAAc;KACf,CAAC;AACJ,CAAC"}

package/dist/src/core/scorer/llm-judge.d.ts

@@ -0,0 +1,43 @@
+/**
+ * packages/eval/src/core/scorer/llm-judge.ts - LLM-as-judge 评分器（M5）
+ *
+ * 开放性回答的终审（EVAL_PLAN.md 4.2 硬性约束）：
+ *   - judge 模型与被测模型异源（装配层校验，见 live.ts assertJudgeDistinct）
+ *   - 输出强制 JSON（score + reason + evidence），解析失败视为 scorer
+ *     自身错误 → 跳过该评分器而非判用例失败
+ *   - judge prompt 版本纳入元数据（JUDGE_PROMPT_VERSION / params.promptVersion），
+ *     改 rubric 不 invalidate 全部历史结果
+ */
+import type { RunTrace, ScoreResult, ToolCallRecord } from '../types';
+export interface JudgeParams {
+    /** rubric 条目（逐条评判） */
+    criteria: string[];
+    /** rubric 版本（缺省 JUDGE_PROMPT_VERSION） */
+    promptVersion?: number;
+}
+export interface JudgeDeps {
+    /** prompt → assistant 文本（由 live judge 模型装配，见 core/judge-llm.ts） */
+    complete(prompt: string): Promise<string>;
+    /** 报告展示的 judge 模型标识 */
+    modelLabel?: string;
+}
+export declare const JUDGE_PROMPT_VERSION = 1;
+/** 单条 rubric 评判的 JSON 结构 */
+export interface JudgeVerdict {
+    score: number;
+    reason: string;
+    evidence?: string;
+}
+/** 组装 judge prompt（版本化：改 rubric / 格式不影响历史结果的含义） */
+export declare function buildJudgePrompt(params: JudgeParams, output: string, trajectory: ToolCallRecord[]): string;
+/**
+ * 解析 judge 输出。宽容处理：截取首个 '{' 到最后一个 '}'。
+ * 解析失败返回 undefined（scorer 自身错误 → 跳过，不判用例失败）。
+ */
+export declare function parseJudgeResponse(raw: string): JudgeVerdict | undefined;
+/**
+ * 执行 LLM-as-judge。
+ * 返回 undefined = 评分器跳过（缺 judge 装配 / judge 输出解析失败 / judge 调用失败），
+ * 不判用例失败（EVAL_PLAN.md 4.2）。
+ */
+export declare function runLlmJudge(trace: RunTrace, params: JudgeParams, deps?: JudgeDeps): Promise<ScoreResult | undefined>;