@aipack-ai/eval 1.1.6 → 1.1.8

package/dist/src/core/runner.js

@@ -1,41 +1,91 @@
 /**
  * packages/eval/src/core/runner.ts - Eval Runner
  *
- * mock 模式：每个用例构造独立 Runtime（createMockStreamFn 注入
- * streamFn + createMockTools 提供工具），run() 后经 getMessages()
- * 重建完整轨迹，再交给规则评分器。
+ * 两种模式共用同一条执行链路（Runtime → 轨迹 → 评分器）：
+ *   mock —— fixture replay：createMockStreamFn 注入脚本化 LLM 响应
+ *   live —— 真实 LLM：createLiveLlm 装配 provider streamFn（M3）
  *
  * 工程保障：
  *   - 用例间完全隔离（独立 runtime / 独立 fs 副本 / 独立 sessionKey）
  *   - 墙钟超时（Promise.race）
  *   - 步数上限（映射 RuntimeOptions.maxTurns）
- *   - 预算熔断（usage.total 超 case.metadata.maxTokens 判失败）
- *   - repeats 消随机性（live 模式用；mock 模式缺省 1）
+ *   - 预算熔断（case.metadata.maxTokens + RunConfig.maxTotalTokens 全局）
+ *   - repeats 消随机性：mock 1；live 默认 3 并取 pass@k（适配层不支持 seed）
  *   - 并发受控（简单队列实现）
  */
-import { createRuntime, createRequest, } from '@aipack-ai/agent';
+import { createRuntime, createRequest } from '@aipack-ai/agent';
 import { createMockStreamFn } from './mock-stream.js';
 import { createMockTools } from './mock-tools.js';
 import { countTurns, extractTrajectory } from './trajectory.js';
 import { resolveScorers, scoreTrace } from './scorer/index.js';
+import { createOpenAiCompatibleEmbedFn } from './embedding.js';
+import { createJudgeDeps } from './judge-llm.js';
+import { assertJudgeDistinct, assertLiveReady, describeLiveLlm, parseModelSpec, resolveJudgeLlm, resolveLiveLlm, } from './live.js';
 const DEFAULT_TIMEOUT_MS = 30_000;
 const DEFAULT_MAX_STEPS = 50;
 const DEFAULT_CONCURRENCY = 8;
+/** live 模式缺省 repeats：无 seed 可用，靠多次采样消噪 */
+const LIVE_DEFAULT_REPEATS = 3;
 /** sanitize sessionKey（用例 id 可能含 '/'） */
 function sessionKeyFor(caseId, repeat) {
     return `eval:${caseId.replace(/[^a-zA-Z0-9_-]/g, '_')}#r${repeat}`;
 }
+/** bundle.label（'provider/modelId'）→ spec；解析失败不阻断（mock / 注入 streamFn 场景） */
+function parseModelSpecSafe(label) {
+    try {
+        return parseModelSpec(label);
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * 模式不匹配 → 跳过（不计入通过率，避免出现"必然失败"的噪音）。
+ * 返回 undefined 表示该用例在当前模式下可跑。
+ */
+export function skipReason(c, mode) {
+    if (mode === 'mock' && c.mode === 'live')
+        return 'live-only 用例（mock 模式跳过）';
+    if (mode === 'live' && c.mode === 'mock')
+        return 'mock-only 用例（live 模式跳过）';
+    if (mode === 'live' && c.input.mock)
+        return 'fixture replay 用例（live 模式跳过）';
+    return undefined;
+}
+/** live 模式：装配真实 LLM（缺配置直接抛错，由 CLI 转 usage 错误） */
+function resolveRuntimeBundle(config) {
+    if (config.mode !== 'live')
+        return undefined;
+    if (config.streamFn) {
+        return { streamFn: config.streamFn, model: config.frameworkModel };
+    }
+    const llm = resolveLiveLlm({
+        model: config.model,
+        apiKey: config.apiKey,
+        baseUrl: config.baseUrl,
+        temperature: config.temperature,
+        timeoutMs: config.requestTimeoutMs,
+    });
+    assertLiveReady(llm.spec);
+    return {
+        streamFn: llm.streamFn,
+        model: llm.model,
+        label: describeLiveLlm(llm.spec),
+    };
+}
 /** 单用例单次运行：返回轨迹（失败时返回 error 字符串） */
-async function runCaseOnce(c, config, repeat) {
-    if (!c.input.mock) {
-        return { error: 'mock 模式要求 input.mock 脚本（live 模式 M3 接入）' };
+async function runCaseOnce(c, config, repeat, bundle) {
+    const isLive = config.mode === 'live';
+    if (!isLive && !c.input.mock) {
+        return { error: 'mock 模式要求 input.mock 脚本（live 模式用 --mode live）' };
     }
     const timeoutMs = c.metadata?.timeoutMs ?? config.timeoutMs ?? DEFAULT_TIMEOUT_MS;
     const maxSteps = c.metadata?.maxSteps ?? config.maxSteps ?? DEFAULT_MAX_STEPS;
     const tools = createMockTools(c.input.fs ?? {}, c.input.tools);
     const runtime = createRuntime({
         systemPrompt: c.input.systemPrompt ?? '',
-        streamFn: createMockStreamFn(c.input.mock),
+        streamFn: isLive ? bundle.streamFn : createMockStreamFn(c.input.mock),
+        ...(isLive && bundle?.model ? { model: bundle.model } : {}),
         tools,
         maxTurns: maxSteps,
         traceIdGenerator: () => sessionKeyFor(c.id, repeat),
@@ -96,57 +146,123 @@ async function runCaseOnce(c, config, repeat) {
         await runtime.close().catch(() => undefined);
     }
 }
+/**
+ * M5：按需装配评分器依赖（lazy 检测——只有用例真的用了 semantic /
+ * llm-judge 才装配，配置缺失时抛可操作错误，由 CLI 转为用法错误）。
+ */
+function resolveScorerDeps(config, needs, targetSpec) {
+    const deps = {};
+    if (needs.judge) {
+        const judge = config.judgeStreamFn
+            ? { streamFn: config.judgeStreamFn, model: config.judgeFrameworkModel }
+            : (() => {
+                const llm = resolveJudgeLlm({
+                    model: config.judgeModel,
+                    apiKey: config.judgeApiKey,
+                    baseUrl: config.judgeBaseUrl,
+                    timeoutMs: config.requestTimeoutMs,
+                });
+                assertJudgeDistinct(targetSpec, llm.spec);
+                return llm;
+            })();
+        deps.judge = createJudgeDeps(judge.streamFn, judge.model, config.judgeStreamFn ? 'injected-judge' : undefined);
+    }
+    if (needs.embed) {
+        deps.embed = createOpenAiCompatibleEmbedFn({
+            model: config.embedModel,
+            apiKey: config.embedApiKey,
+            baseUrl: config.embedBaseUrl,
+        });
+    }
+    return deps;
+}
 /** 单用例（含 repeats） */
-async function runCase(c, config) {
-    const repeats = config.mode === 'mock' ? (config.repeats ?? 1) : (config.repeats ?? 1);
+async function runCase(c, config, bundle, budget, scorerDeps) {
+    const isLive = config.mode === 'live';
+    const repeats = c.metadata?.repeats ?? config.repeats ?? (isLive ? LIVE_DEFAULT_REPEATS : 1);
     const scorers = resolveScorers(c.expected, c.scorers);
-    const startedAt = Date.now();
-    let passed = true;
-    let score = 0;
+    // 全局预算熔断：已超限时剩余用例直接判失败，不再烧钱
+    if (config.maxTotalTokens !== undefined && budget.spent > config.maxTotalTokens) {
+        return {
+            caseId: c.id,
+            suite: c.suite,
+            origin: c.origin,
+            passed: false,
+            score: 0,
+            scores: [],
+            error: `总预算熔断：累计 usage.total=${budget.spent} > maxTotalTokens=${config.maxTotalTokens}`,
+            durationMs: 0,
+            steps: 0,
+            usageTotal: 0,
+            repeats,
+            passCount: 0,
+        };
+    }
+    let scoreSum = 0;
     let scoreCount = 0;
-    const scoreResults = [];
+    let passCount = 0;
+    const failedScores = [];
     let runError;
     let steps = 0;
     let usageTotal = 0;
     let durationMs = 0;
+    let lastTraceUsage = 0;
     for (let r = 0; r < repeats; r++) {
-        const { trace, error } = await runCaseOnce(c, config, r);
-        durationMs += Date.now() - startedAt;
+        const t0 = Date.now();
+        const { trace, error } = await runCaseOnce(c, config, r, bundle);
+        durationMs += Date.now() - t0;
         if (error || !trace) {
-            passed = false;
-            runError = error;
-            scoreResults.push({
-                scorer: 'success',
-                score: 0,
-                passed: false,
-                reason: `运行失败: ${error ?? '未知错误'}`,
-            });
-            break;
+            runError = runError ?? error;
+            continue;
         }
         steps = trace.trajectory.length;
+        lastTraceUsage = trace.usageTotal;
         usageTotal += trace.usageTotal;
-        const caseScore = scoreTrace(trace, scorers);
-        passed = passed && caseScore.passed;
-        score += caseScore.score;
+        budget.spent += trace.usageTotal;
+        const caseScore = await scoreTrace(trace, scorers, scorerDeps);
+        scoreSum += caseScore.score;
         scoreCount += 1;
-        if (!caseScore.passed) {
-            scoreResults.push(...caseScore.scores.filter((s) => !s.passed).map((s) => ({
-                ...s,
-                reason: s.reason,
-            })));
-        }
+        if (caseScore.passed)
+            passCount += 1;
+        else
+            failedScores.push(...caseScore.scores.filter((s) => !s.passed));
+    }
+    if (scoreCount === 0 && runError) {
+        return {
+            caseId: c.id,
+            suite: c.suite,
+            origin: c.origin,
+            passed: false,
+            score: 0,
+            scores: [
+                { scorer: 'success', score: 0, passed: false, reason: `运行失败: ${runError}` },
+            ],
+            error: runError,
+            durationMs,
+            steps,
+            usageTotal,
+            repeats,
+            passCount: 0,
+        };
+    }
+    // live：pass@k（k 次里至少过一次）；mock：每次都必须过
+    const passed = isLive ? passCount > 0 : passCount === scoreCount && scoreCount > 0;
+    if (scoreCount < repeats) {
+        runError = runError ?? `${repeats - scoreCount}/${repeats} 次运行异常`;
     }
     return {
         caseId: c.id,
         suite: c.suite,
         origin: c.origin,
         passed,
-        score: scoreCount === 0 ? 0 : score / scoreCount,
-        scores: scoreResults,
+        score: scoreCount === 0 ? 0 : scoreSum / scoreCount,
+        scores: passed ? [] : failedScores,
         error: runError,
         durationMs,
         steps,
-        usageTotal,
+        usageTotal: usageTotal || lastTraceUsage,
+        repeats,
+        passCount,
     };
 }
 /** 简单并发队列（避免引入依赖） */
@@ -185,20 +301,36 @@ function summarize(results) {
 /** 执行一次评测 */
 export async function runEval(cases, config) {
     const startedAt = Date.now();
-    const filtered = config.suites && config.suites.length > 0
+    const bundle = resolveRuntimeBundle(config);
+    const bySuite = config.suites && config.suites.length > 0
         ? cases.filter((c) => config.suites.includes(c.suite))
         : cases;
+    const skipped = [];
+    const runnable = bySuite.filter((c) => {
+        const reason = skipReason(c, config.mode);
+        if (reason) {
+            skipped.push({ caseId: c.id, suite: c.suite, reason });
+            return false;
+        }
+        return true;
+    });
     const concurrency = config.concurrency ?? DEFAULT_CONCURRENCY;
-    const results = await runAllConcurrent(filtered, concurrency, (c) => runCase(c, config));
+    const budget = { spent: 0 };
+    // M5：只有用例真的使用 semantic / llm-judge 才装配对应依赖（配置缺失抛错 → CLI 用法错误）
+    const usesScorer = (type) => runnable.some((c) => resolveScorers(c.expected, c.scorers).some((s) => s.type === type));
+    const scorerDeps = resolveScorerDeps(config, { judge: usesScorer('llm-judge'), embed: usesScorer('semantic') }, bundle?.label ? parseModelSpecSafe(bundle.label) : undefined);
+    const results = await runAllConcurrent(runnable, concurrency, (c) => runCase(c, config, bundle, budget, scorerDeps));
     const casesPassed = results.filter((r) => r.passed).length;
     return {
         runId: `run_${new Date().toISOString().replace(/[:.]/g, '-')}`,
         startedAt: new Date(startedAt).toISOString(),
         durationMs: Date.now() - startedAt,
         mode: config.mode,
+        model: bundle?.label ?? (config.mode === 'live' ? config.model : undefined),
         totals: {
             cases: results.length,
             passed: casesPassed,
+            skipped: skipped.length,
             passRate: results.length === 0 ? 0 : casesPassed / results.length,
             avgScore: results.length === 0
                 ? 0
@@ -208,6 +340,7 @@ export async function runEval(cases, config) {
         bySuite: summarize(results),
         byOrigin: summarize(results.map((r) => ({ ...r, suite: r.origin }))),
         results,
+        skipped,
     };
 }
 //# sourceMappingURL=runner.js.map

package/dist/src/core/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/core/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EACL,aAAa,EACb,aAAa,GACd,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAUtD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAE9B,yCAAyC;AACzC,SAAS,aAAa,CAAC,MAAc,EAAE,MAAc;IACnD,OAAO,QAAQ,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,KAAK,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,oCAAoC;AACpC,KAAK,UAAU,WAAW,CACxB,CAAW,EACX,MAAiB,EACjB,MAAc;IAEd,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,KAAK,EAAE,wCAAwC,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAClF,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,IAAI,MAAM,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IAE9E,MAAM,KAAK,GAAW,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACvE,MAAM,OAAO,GAAG,aAAa,CAAC;QAC5B,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE;QACxC,QAAQ,EAAE,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QAC1C,KAAK;QACL,QAAQ,EAAE,QAAQ;QAClB,gBAAgB,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC;KACpD,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAa7B,IAAI,GAAwB,CAAC;IAE7B,IAAI,CAAC;QACH,GAAG,GAAG,OAAO;aACV,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;aACnD,IAAI,CAAC,CAAC,MAAM,EAAc,EAAE;YAC3B,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACnC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAC1C,MAAM,UAAU,GACd,OAAO,MAAM,CAAC,KAAK,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnE,OAAO;gBACL,EAAE,EAAE,IAAI;gBACR,MAAM;gBACN,QAAQ;gBACR,UAAU;gBACV,KAAK;gBACL,UAAU;gBACV,UAAU;aACX,CAAC;QACJ,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,CAAU,EAAc,EAAE,CAAC,CAAC;YAClC,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,SAAU,CAAW,CAAC,OAAO,EAAE;SACvC,CAAC,CAAC,CAAC;QAEN,kCAAkC;QAClC,MAAM,cAAc,GAAG,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,EAAE;YACzD,MAAM,CAAC,GAAG,UAAU,CAClB,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,KAAK,EAAE,CAAC,EACzD,SAAS,CACV,CAAC;YACD,CAA4B,CAAC,KAAK,EAAE,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,OAAO,CAAC,EAAE;YAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;QAEjD,MAAM,KAAK,GAAa;YACtB,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC;QAEF,qBAAqB;QACrB,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC;QACxC,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;YAC5D,OAAO,EAAE,KAAK,EAAE,oBAAoB,KAAK,CAAC,UAAU,gBAAgB,SAAS,EAAE,EAAE,CAAC;QACpF,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,CAAC;IACnB,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,qBAAqB;AACrB,KAAK,UAAU,OAAO,CACpB,CAAW,EACX,MAAiB;IAEjB,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;IACvF,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,YAAY,GAAyB,EAAE,CAAC;IAC9C,IAAI,QAA4B,CAAC;IACjC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QACjC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACzD,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACrC,IAAI,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,GAAG,KAAK,CAAC;YACf,QAAQ,GAAG,KAAK,CAAC;YACjB,YAAY,CAAC,IAAI,CAAC;gBAChB,MAAM,EAAE,SAAS;gBACjB,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,SAAS,KAAK,IAAI,MAAM,EAAE;aACnC,CAAC,CAAC;YACH,MAAM;QACR,CAAC;QACD,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;QAChC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;QAC/B,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7C,MAAM,GAAG,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC;QACpC,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC;QACzB,UAAU,IAAI,CAAC,CAAC;QAChB,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YACtB,YAAY,CAAC,IAAI,CACf,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACvD,GAAG,CAAC;gBACJ,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB,CAAC,CAAC,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE,CAAC,CAAC,EAAE;QACZ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,MAAM;QACN,KAAK,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,UAAU;QAChD,MAAM,EAAE,YAAY;QACpB,KAAK,EAAE,QAAQ;QACf,UAAU;QACV,KAAK;QACL,UAAU;KACX,CAAC;AACJ,CAAC;AAED,qBAAqB;AACrB,KAAK,UAAU,gBAAgB,CAC7B,KAAU,EACV,WAAmB,EACnB,EAA2B;IAE3B,MAAM,OAAO,GAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACrF,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,OAAqB;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,MAAM,GAAG,GAAiC,EAAE,CAAC;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACnD,GAAG,CAAC,GAAG,CAAC,GAAG;YACT,KAAK;YACL,MAAM;YACN,QAAQ,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK;YAC1C,QAAQ,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,KAAK;SAC1E,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,aAAa;AACb,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAiB,EACjB,MAAiB;IAEjB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QACvC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACvD,CAAC,CAAC,KAAK,CAAC;IAEZ,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,mBAAmB,CAAC;IAC9D,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAClE,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CACnB,CAAC;IAEF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,OAAO;QACL,KAAK,EAAE,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE;QAC9D,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;QAC5C,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE;YACN,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM,EAAE,WAAW;YACnB,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM;YACjE,QAAQ,EACN,OAAO,CAAC,MAAM,KAAK,CAAC;gBAClB,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;YAC/D,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC;QAC3B,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACpE,OAAO;KACR,CAAC;AACJ,CAAC"}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/core/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEhE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAmB,MAAM,UAAU,CAAC;AACvE,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,eAAe,EACf,cAAc,EACd,eAAe,EACf,cAAc,GACf,MAAM,QAAQ,CAAC;AAWhB,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,0CAA0C;AAC1C,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAe/B,yCAAyC;AACzC,SAAS,aAAa,CAAC,MAAc,EAAE,MAAc;IACnD,OAAO,QAAQ,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,KAAK,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,4EAA4E;AAC5E,SAAS,kBAAkB,CAAC,KAAa;IACvC,IAAI,CAAC;QACH,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,CAAW,EAAE,IAAqB;IAC3D,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,yBAAyB,CAAC;IAC3E,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,yBAAyB,CAAC;IAC3E,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI;QAAE,OAAO,8BAA8B,CAAC;IAC3E,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,iDAAiD;AACjD,SAAS,oBAAoB,CAAC,MAAiB;IAC7C,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,SAAS,CAAC;IAC7C,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;IACrE,CAAC;IACD,MAAM,GAAG,GAAG,cAAc,CAAC;QACzB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,SAAS,EAAE,MAAM,CAAC,gBAAgB;KACnC,CAAC,CAAC;IACH,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1B,OAAO;QACL,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,oCAAoC;AACpC,KAAK,UAAU,WAAW,CACxB,CAAW,EACX,MAAiB,EACjB,MAAc,EACd,MAAsB;IAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;IACtC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,OAAO,EAAE,KAAK,EAAE,+CAA+C,EAAE,CAAC;IACpE,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAClF,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,IAAI,MAAM,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IAE9E,MAAM,KAAK,GAAW,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACvE,MAAM,OAAO,GAAG,aAAa,CAAC;QAC5B,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE;QACxC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,MAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,IAAK,CAAC;QACvE,GAAG,CAAC,MAAM,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,KAAK;QACL,QAAQ,EAAE,QAAQ;QAClB,gBAAgB,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC;KACpD,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAa7B,IAAI,GAAwB,CAAC;IAE7B,IAAI,CAAC;QACH,GAAG,GAAG,OAAO;aACV,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;aACnD,IAAI,CAAC,CAAC,MAAM,EAAc,EAAE;YAC3B,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACnC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAC1C,MAAM,UAAU,GACd,OAAO,MAAM,CAAC,KAAK,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnE,OAAO;gBACL,EAAE,EAAE,IAAI;gBACR,MAAM;gBACN,QAAQ;gBACR,UAAU;gBACV,KAAK;gBACL,UAAU;gBACV,UAAU;aACX,CAAC;QACJ,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,CAAU,EAAc,EAAE,CAAC,CAAC;YAClC,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,SAAU,CAAW,CAAC,OAAO,EAAE;SACvC,CAAC,CAAC,CAAC;QAEN,kCAAkC;QAClC,MAAM,cAAc,GAAG,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,EAAE;YACzD,MAAM,CAAC,GAAG,UAAU,CAClB,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,KAAK,EAAE,CAAC,EACzD,SAAS,CACV,CAAC;YACD,CAA4B,CAAC,KAAK,EAAE,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,OAAO,CAAC,EAAE;YAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;QAEjD,MAAM,KAAK,GAAa;YACtB,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC;QAEF,qBAAqB;QACrB,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC;QACxC,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;YAC5D,OAAO,EAAE,KAAK,EAAE,oBAAoB,KAAK,CAAC,UAAU,gBAAgB,SAAS,EAAE,EAAE,CAAC;QACpF,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,CAAC;IACnB,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CACxB,MAAiB,EACjB,KAAyC,EACzC,UAAkD;IAElD,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa;YAChC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,aAAa,EAAE,KAAK,EAAE,MAAM,CAAC,mBAAmB,EAAE;YACvE,CAAC,CAAC,CAAC,GAAG,EAAE;gBACJ,MAAM,GAAG,GAAG,eAAe,CAAC;oBAC1B,KAAK,EAAE,MAAM,CAAC,UAAU;oBACxB,MAAM,EAAE,MAAM,CAAC,WAAW;oBAC1B,OAAO,EAAE,MAAM,CAAC,YAAY;oBAC5B,SAAS,EAAE,MAAM,CAAC,gBAAgB;iBACnC,CAAC,CAAC;gBACH,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC1C,OAAO,GAAG,CAAC;YACb,CAAC,CAAC,EAAE,CAAC;QACT,IAAI,CAAC,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACjH,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,IAAI,CAAC,KAAK,GAAG,6BAA6B,CAAC;YACzC,KAAK,EAAE,MAAM,CAAC,UAAU;YACxB,MAAM,EAAE,MAAM,CAAC,WAAW;YAC1B,OAAO,EAAE,MAAM,CAAC,YAAY;SAC7B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,qBAAqB;AACrB,KAAK,UAAU,OAAO,CACpB,CAAW,EACX,MAAiB,EACjB,MAAiC,EACjC,MAAmB,EACnB,UAAsB;IAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;IACtC,MAAM,OAAO,GACX,CAAC,CAAC,QAAQ,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAEtD,4BAA4B;IAC5B,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;QAChF,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,EAAE;YACV,KAAK,EAAE,wBAAwB,MAAM,CAAC,KAAK,qBAAqB,MAAM,CAAC,cAAc,EAAE;YACvF,UAAU,EAAE,CAAC;YACb,KAAK,EAAE,CAAC;YACR,UAAU,EAAE,CAAC;YACb,OAAO;YACP,SAAS,EAAE,CAAC;SACb,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,YAAY,GAAkB,EAAE,CAAC;IACvC,IAAI,QAA4B,CAAC;IACjC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QACjC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACjE,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;QAE9B,IAAI,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;YACpB,QAAQ,GAAG,QAAQ,IAAI,KAAK,CAAC;YAC7B,SAAS;QACX,CAAC;QAED,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;QAChC,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC;QAClC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;QAC/B,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;QAEjC,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;QAC/D,QAAQ,IAAI,SAAS,CAAC,KAAK,CAAC;QAC5B,UAAU,IAAI,CAAC,CAAC;QAChB,IAAI,SAAS,CAAC,MAAM;YAAE,SAAS,IAAI,CAAC,CAAC;;YAChC,YAAY,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,UAAU,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;QACjC,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE;gBACN,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,QAAQ,EAAE,EAAE;aAC5E;YACD,KAAK,EAAE,QAAQ;YACf,UAAU;YACV,KAAK;YACL,UAAU;YACV,OAAO;YACP,SAAS,EAAE,CAAC;SACb,CAAC;IACJ,CAAC;IAED,qCAAqC;IACrC,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,KAAK,UAAU,IAAI,UAAU,GAAG,CAAC,CAAC;IACnF,IAAI,UAAU,GAAG,OAAO,EAAE,CAAC;QACzB,QAAQ,GAAG,QAAQ,IAAI,GAAG,OAAO,GAAG,UAAU,IAAI,OAAO,QAAQ,CAAC;IACpE,CAAC;IAED,OAAO;QACL,MAAM,EAAE,CAAC,CAAC,EAAE;QACZ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,MAAM;QACN,KAAK,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,UAAU;QACnD,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY;QAClC,KAAK,EAAE,QAAQ;QACf,UAAU;QACV,KAAK;QACL,UAAU,EAAE,UAAU,IAAI,cAAc;QACxC,OAAO;QACP,SAAS;KACV,CAAC;AACJ,CAAC;AAED,qBAAqB;AACrB,KAAK,UAAU,gBAAgB,CAC7B,KAAU,EACV,WAAmB,EACnB,EAA2B;IAE3B,MAAM,OAAO,GAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACrF,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,OAAqB;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,MAAM,GAAG,GAAiC,EAAE,CAAC;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACnD,GAAG,CAAC,GAAG,CAAC,GAAG;YACT,KAAK;YACL,MAAM;YACN,QAAQ,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK;YAC1C,QAAQ,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,KAAK;SAC1E,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,aAAa;AACb,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAiB,EACjB,MAAiB;IAEjB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAE5C,MAAM,OAAO,GACX,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QACvC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACvD,CAAC,CAAC,KAAK,CAAC;IAEZ,MAAM,OAAO,GAAuC,EAAE,CAAC;IACvD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,mBAAmB,CAAC;IAC9D,MAAM,MAAM,GAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAEzC,8DAA8D;IAC9D,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,EAAE,CAClC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;IAC3F,MAAM,UAAU,GAAG,iBAAiB,CAClC,MAAM,EACN,EAAE,KAAK,EAAE,UAAU,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,EACjE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAC7D,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAClE,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAC/C,CAAC;IAEF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,OAAO;QACL,KAAK,EAAE,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE;QAC9D,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;QAC5C,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3E,MAAM,EAAE;YACN,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM,EAAE,WAAW;YACnB,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM;YACjE,QAAQ,EACN,OAAO,CAAC,MAAM,KAAK,CAAC;gBAClB,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;YAC/D,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC;QAC3B,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACpE,OAAO;QACP,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/src/core/scorer/index.d.ts

@@ -2,10 +2,26 @@
  * packages/eval/src/core/scorer/index.ts - 评分入口与 expected 归一化
  *
  * - expected（声明式）→ ScorerConfig[]（与显式 scorers 合并）
- * - scoreCase：依次执行规则评分器，加权合成用例总分
+ * - scoreCase：按类型分发评分器（rule 同步 / semantic & llm-judge 异步），
+ *   加权合成用例总分
+ * - M5：semantic / llm-judge 需要外部装配（embed / judge），缺装配时
+ *   评分器跳过（不判用例失败，见各实现文件）
  */
 import type { ExpectedResult, RunTrace, ScoreResult, ScorerConfig } from '../types';
+import { type EmbedFn } from './semantic';
+import { type JudgeDeps } from './llm-judge';
 export { RULE_SCORER_TYPES, runRuleScorer, partialMatch } from './rule';
+export { cosineSimilarity, runSemanticScorer, type EmbedFn, type SemanticParams } from './semantic';
+export { buildJudgePrompt, parseJudgeResponse, runLlmJudge, JUDGE_PROMPT_VERSION, type JudgeDeps, type JudgeParams, type JudgeVerdict, } from './llm-judge';
+/** 全量评分器类型（rule 8 种 + semantic + llm-judge） */
+export declare const ALL_SCORER_TYPES: readonly [...string[], "semantic", "llm-judge"];
+/** 评分器外部装配（M5）：缺省时 semantic / llm-judge 跳过 */
+export interface ScorerDeps {
+    /** semantic 评分器的 embedding 函数 */
+    embed?: EmbedFn;
+    /** llm-judge 的异源 judge 装配 */
+    judge?: JudgeDeps;
+}
 /** 声明式 expected → 评分器配置 */
 export declare function expectedToScorers(expected: ExpectedResult): ScorerConfig[];
 /** 合并用例的 expected + scorers */
@@ -14,6 +30,8 @@ export interface CaseScore {
     passed: boolean;
     score: number;
     scores: ScoreResult[];
+    /** 因缺装配 / judge 解析失败而跳过的评分器（不判用例失败） */
+    skippedScorers: string[];
 }
 /** 对一次运行轨迹执行全部评分器 */
-export declare function scoreTrace(trace: RunTrace, scorers: ScorerConfig[]): CaseScore;
+export declare function scoreTrace(trace: RunTrace, scorers: ScorerConfig[], deps?: ScorerDeps): Promise<CaseScore>;

package/dist/src/core/scorer/index.js

@@ -2,10 +2,19 @@
  * packages/eval/src/core/scorer/index.ts - 评分入口与 expected 归一化
  *
  * - expected（声明式）→ ScorerConfig[]（与显式 scorers 合并）
- * - scoreCase：依次执行规则评分器，加权合成用例总分
+ * - scoreCase：按类型分发评分器（rule 同步 / semantic & llm-judge 异步），
+ *   加权合成用例总分
+ * - M5：semantic / llm-judge 需要外部装配（embed / judge），缺装配时
+ *   评分器跳过（不判用例失败，见各实现文件）
  */
-import { runRuleScorer } from './rule.js';
+import { RULE_SCORER_TYPES, runRuleScorer } from './rule.js';
+import { runSemanticScorer } from './semantic.js';
+import { runLlmJudge } from './llm-judge.js';
 export { RULE_SCORER_TYPES, runRuleScorer, partialMatch } from './rule.js';
+export { cosineSimilarity, runSemanticScorer } from './semantic.js';
+export { buildJudgePrompt, parseJudgeResponse, runLlmJudge, JUDGE_PROMPT_VERSION, } from './llm-judge.js';
+/** 全量评分器类型（rule 8 种 + semantic + llm-judge） */
+export const ALL_SCORER_TYPES = [...RULE_SCORER_TYPES, 'semantic', 'llm-judge'];
 /** 声明式 expected → 评分器配置 */
 export function expectedToScorers(expected) {
     switch (expected.type) {
@@ -39,18 +48,36 @@ export function expectedToScorers(expected) {
 export function resolveScorers(expected, explicit) {
     return [...(expected ? expectedToScorers(expected) : []), ...(explicit ?? [])];
 }
+/** 执行单个评分器（按类型分发）；返回 undefined = 跳过 */
+async function runScorer(trace, s, deps) {
+    if (s.type === 'semantic') {
+        return runSemanticScorer(trace, s.params, deps.embed);
+    }
+    if (s.type === 'llm-judge') {
+        return runLlmJudge(trace, s.params, deps.judge);
+    }
+    return runRuleScorer(trace, s.type, s.params);
+}
 /** 对一次运行轨迹执行全部评分器 */
-export function scoreTrace(trace, scorers) {
-    const scores = scorers.map((s) => runRuleScorer(trace, s.type, s.params));
-    const totalWeight = scorers.reduce((acc, s) => acc + (s.weight ?? 1), 0);
+export async function scoreTrace(trace, scorers, deps = {}) {
+    const results = await Promise.all(scorers.map((s) => runScorer(trace, s, deps)));
+    // 跳过的评分器（缺装配 / scorer 自身错误）不参与判定
+    const skippedScorers = scorers
+        .filter((_, i) => results[i] === undefined)
+        .map((s) => s.type);
+    const effective = scorers
+        .map((s, i) => ({ s, r: results[i] }))
+        .filter((x) => x.r !== undefined);
+    const totalWeight = effective.reduce((acc, { s }) => acc + (s.weight ?? 1), 0);
     const weighted = totalWeight === 0
         ? 0
-        : scores.reduce((acc, r, i) => acc + r.score * (scorers[i].weight ?? 1), 0) /
+        : effective.reduce((acc, { s, r }) => acc + r.score * (s.weight ?? 1), 0) /
             totalWeight;
     return {
-        passed: scores.length > 0 && scores.every((r) => r.passed),
-        score: scorers.length === 0 ? 0 : weighted,
-        scores,
+        passed: effective.length > 0 && effective.every(({ r }) => r.passed),
+        score: effective.length === 0 ? 0 : weighted,
+        scores: effective.map(({ r }) => r),
+        skippedScorers,
     };
 }
 //# sourceMappingURL=index.js.map

package/dist/src/core/scorer/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/core/scorer/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,OAAO,EAAqB,aAAa,EAAE,MAAM,QAAQ,CAAC;AAE1D,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAExE,2BAA2B;AAC3B,MAAM,UAAU,iBAAiB,CAAC,QAAwB;IACxD,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,OAAO,CAAC;QACb,KAAK,UAAU;YACb,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtE,KAAK,OAAO;YACV,OAAO;gBACL,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE;aAClF,CAAC;QACJ,KAAK,WAAW;YACd,OAAO;gBACL;oBACE,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,QAAQ,EAAE;iBACrE;aACF,CAAC;QACJ,KAAK,YAAY;YACf,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrE,KAAK,YAAY;YACf,OAAO,OAAO,IAAI,QAAQ;gBACxB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;gBAClF,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAChE,KAAK,SAAS;YACZ,OAAO,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1E,KAAK,aAAa;YAChB,OAAO,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,+BAA+B;AAC/B,MAAM,UAAU,cAAc,CAC5B,QAAyB,EACzB,QAAyB;IAEzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AACjF,CAAC;AAQD,qBAAqB;AACrB,MAAM,UAAU,UAAU,CAAC,KAAe,EAAE,OAAuB;IACjE,MAAM,MAAM,GAAkB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9C,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CACvC,CAAC;IACF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,QAAQ,GACZ,WAAW,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YACzE,WAAW,CAAC;IAClB,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1D,KAAK,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ;QAC1C,MAAM;KACP,CAAC;AACJ,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/core/scorer/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAgB,MAAM,YAAY,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAkB,MAAM,aAAa,CAAC;AAE1D,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAqC,MAAM,YAAY,CAAC;AACpG,OAAO,EACL,gBAAgB,EAChB,kBAAkB,EAClB,WAAW,EACX,oBAAoB,GAIrB,MAAM,aAAa,CAAC;AAErB,+CAA+C;AAC/C,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,EAAE,UAAU,EAAE,WAAW,CAAU,CAAC;AAUzF,2BAA2B;AAC3B,MAAM,UAAU,iBAAiB,CAAC,QAAwB;IACxD,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,OAAO,CAAC;QACb,KAAK,UAAU;YACb,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtE,KAAK,OAAO;YACV,OAAO;gBACL,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE;aAClF,CAAC;QACJ,KAAK,WAAW;YACd,OAAO;gBACL;oBACE,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,QAAQ,EAAE;iBACrE;aACF,CAAC;QACJ,KAAK,YAAY;YACf,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrE,KAAK,YAAY;YACf,OAAO,OAAO,IAAI,QAAQ;gBACxB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;gBAClF,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAChE,KAAK,SAAS;YACZ,OAAO,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1E,KAAK,aAAa;YAChB,OAAO,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,+BAA+B;AAC/B,MAAM,UAAU,cAAc,CAC5B,QAAyB,EACzB,QAAyB;IAEzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AACjF,CAAC;AAUD,uCAAuC;AACvC,KAAK,UAAU,SAAS,CACtB,KAAe,EACf,CAAe,EACf,IAAgB;IAEhB,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC1B,OAAO,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,MAAe,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC3B,OAAO,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC,MAAe,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;AAChD,CAAC;AAED,qBAAqB;AACrB,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAe,EACf,OAAuB,EACvB,OAAmB,EAAE;IAErB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAEjF,iCAAiC;IACjC,MAAM,cAAc,GAAG,OAAO;SAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC;SAC1C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtB,MAAM,SAAS,GAAG,OAAO;SACtB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SACrC,MAAM,CAAC,CAAC,CAAC,EAA4C,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAE9E,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/E,MAAM,QAAQ,GACZ,WAAW,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YACvE,WAAW,CAAC;IAClB,OAAO;QACL,MAAM,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QACpE,KAAK,EAAE,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ;QAC5C,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QACnC,cAAc;KACf,CAAC;AACJ,CAAC"}

package/dist/src/core/scorer/llm-judge.d.ts

@@ -0,0 +1,43 @@
+/**
+ * packages/eval/src/core/scorer/llm-judge.ts - LLM-as-judge 评分器（M5）
+ *
+ * 开放性回答的终审（EVAL_PLAN.md 4.2 硬性约束）：
+ *   - judge 模型与被测模型异源（装配层校验，见 live.ts assertJudgeDistinct）
+ *   - 输出强制 JSON（score + reason + evidence），解析失败视为 scorer
+ *     自身错误 → 跳过该评分器而非判用例失败
+ *   - judge prompt 版本纳入元数据（JUDGE_PROMPT_VERSION / params.promptVersion），
+ *     改 rubric 不 invalidate 全部历史结果
+ */
+import type { RunTrace, ScoreResult, ToolCallRecord } from '../types';
+export interface JudgeParams {
+    /** rubric 条目（逐条评判） */
+    criteria: string[];
+    /** rubric 版本（缺省 JUDGE_PROMPT_VERSION） */
+    promptVersion?: number;
+}
+export interface JudgeDeps {
+    /** prompt → assistant 文本（由 live judge 模型装配，见 core/judge-llm.ts） */
+    complete(prompt: string): Promise<string>;
+    /** 报告展示的 judge 模型标识 */
+    modelLabel?: string;
+}
+export declare const JUDGE_PROMPT_VERSION = 1;
+/** 单条 rubric 评判的 JSON 结构 */
+export interface JudgeVerdict {
+    score: number;
+    reason: string;
+    evidence?: string;
+}
+/** 组装 judge prompt（版本化：改 rubric / 格式不影响历史结果的含义） */
+export declare function buildJudgePrompt(params: JudgeParams, output: string, trajectory: ToolCallRecord[]): string;
+/**
+ * 解析 judge 输出。宽容处理：截取首个 '{' 到最后一个 '}'。
+ * 解析失败返回 undefined（scorer 自身错误 → 跳过，不判用例失败）。
+ */
+export declare function parseJudgeResponse(raw: string): JudgeVerdict | undefined;
+/**
+ * 执行 LLM-as-judge。
+ * 返回 undefined = 评分器跳过（缺 judge 装配 / judge 输出解析失败 / judge 调用失败），
+ * 不判用例失败（EVAL_PLAN.md 4.2）。
+ */
+export declare function runLlmJudge(trace: RunTrace, params: JudgeParams, deps?: JudgeDeps): Promise<ScoreResult | undefined>;

package/dist/src/core/scorer/llm-judge.js

@@ -0,0 +1,111 @@
+/**
+ * packages/eval/src/core/scorer/llm-judge.ts - LLM-as-judge 评分器（M5）
+ *
+ * 开放性回答的终审（EVAL_PLAN.md 4.2 硬性约束）：
+ *   - judge 模型与被测模型异源（装配层校验，见 live.ts assertJudgeDistinct）
+ *   - 输出强制 JSON（score + reason + evidence），解析失败视为 scorer
+ *     自身错误 → 跳过该评分器而非判用例失败
+ *   - judge prompt 版本纳入元数据（JUDGE_PROMPT_VERSION / params.promptVersion），
+ *     改 rubric 不 invalidate 全部历史结果
+ */
+export const JUDGE_PROMPT_VERSION = 1;
+function truncate(s, n) {
+    return s.length <= n ? s : `${s.slice(0, n)}…`;
+}
+function formatTrajectory(calls) {
+    if (calls.length === 0)
+        return '(无工具调用)';
+    return calls
+        .map((c, i) => `${i + 1}. ${c.name}(${JSON.stringify(c.args ?? {})})${c.isError ? ' [error]' : ''}`)
+        .join('\n');
+}
+/** 组装 judge prompt（版本化：改 rubric / 格式不影响历史结果的含义） */
+export function buildJudgePrompt(params, output, trajectory) {
+    const version = params.promptVersion ?? JUDGE_PROMPT_VERSION;
+    const criteria = params.criteria.map((c, i) => `${i + 1}. ${c}`).join('\n');
+    return [
+        `你是严格的评测 judge（prompt v${version}）。请逐条依据以下 rubric 评判 agent 回答的质量：`,
+        '',
+        criteria,
+        '',
+        '── Agent 最终输出 ──',
+        output,
+        '',
+        '── 工具调用轨迹（时间序） ──',
+        formatTrajectory(trajectory),
+        '',
+        '要求：',
+        '- score 为 0~1 的数字（全部 rubric 满足 = 1，完全不满足 = 0）',
+        '- reason 必须给出理由并引用证据',
+        '- 只输出一个 JSON 对象，不要输出其他任何文本：',
+        '  {"score": <0~1>, "reason": "<理由>", "evidence": "<命中的原文片段或调用序号>"}',
+    ].join('\n');
+}
+/**
+ * 解析 judge 输出。宽容处理：截取首个 '{' 到最后一个 '}'。
+ * 解析失败返回 undefined（scorer 自身错误 → 跳过，不判用例失败）。
+ */
+export function parseJudgeResponse(raw) {
+    const start = raw.indexOf('{');
+    const end = raw.lastIndexOf('}');
+    if (start < 0 || end <= start)
+        return undefined;
+    let parsed;
+    try {
+        parsed = JSON.parse(raw.slice(start, end + 1));
+    }
+    catch {
+        return undefined;
+    }
+    if (!parsed || typeof parsed !== 'object')
+        return undefined;
+    const { score, reason, evidence } = parsed;
+    if (typeof score !== 'number' || !Number.isFinite(score))
+        return undefined;
+    const clamped = Math.min(Math.max(score, 0), 1);
+    return {
+        score: clamped,
+        reason: typeof reason === 'string' && reason ? reason : '(judge 未给出理由)',
+        evidence: typeof evidence === 'string' ? evidence : undefined,
+    };
+}
+/**
+ * 执行 LLM-as-judge。
+ * 返回 undefined = 评分器跳过（缺 judge 装配 / judge 输出解析失败 / judge 调用失败），
+ * 不判用例失败（EVAL_PLAN.md 4.2）。
+ */
+export async function runLlmJudge(trace, params, deps) {
+    if (!deps)
+        return undefined;
+    if (!Array.isArray(params.criteria) || params.criteria.length === 0) {
+        return {
+            scorer: 'llm-judge',
+            score: 0,
+            passed: false,
+            reason: "评分器参数缺少非空数组字段 'criteria'",
+        };
+    }
+    const prompt = buildJudgePrompt(params, trace.result.content, trace.trajectory);
+    let raw;
+    try {
+        raw = await deps.complete(prompt);
+    }
+    catch (e) {
+        console.error('[eval] llm-judge 调用失败（跳过）:', e.message);
+        return undefined;
+    }
+    const verdict = parseJudgeResponse(raw);
+    if (!verdict) {
+        console.error('[eval] llm-judge 输出解析失败（跳过）:', truncate(raw, 120));
+        return undefined;
+    }
+    const passed = verdict.score >= 0.5;
+    return {
+        scorer: 'llm-judge',
+        score: verdict.score,
+        passed,
+        reason: `[${deps.modelLabel ?? 'judge'} v${params.promptVersion ?? JUDGE_PROMPT_VERSION}] ${verdict.reason}`,
+        evidence: verdict.evidence,
+    };
+}
+//# sourceMappingURL=llm-judge.js.map

package/dist/src/core/scorer/llm-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../../../src/core/scorer/llm-judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAkBH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAStC,SAAS,QAAQ,CAAC,CAAS,EAAE,CAAS;IACpC,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC;AACjD,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAuB;IAC/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzC,OAAO,KAAK;SACT,GAAG,CACF,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CACvF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,gBAAgB,CAC9B,MAAmB,EACnB,MAAc,EACd,UAA4B;IAE5B,MAAM,OAAO,GAAG,MAAM,CAAC,aAAa,IAAI,oBAAoB,CAAC;IAC7D,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5E,OAAO;QACL,yBAAyB,OAAO,kCAAkC;QAClE,EAAE;QACF,QAAQ;QACR,EAAE;QACF,kBAAkB;QAClB,MAAM;QACN,EAAE;QACF,mBAAmB;QACnB,gBAAgB,CAAC,UAAU,CAAC;QAC5B,EAAE;QACF,KAAK;QACL,+CAA+C;QAC/C,sBAAsB;QACtB,6BAA6B;QAC7B,oEAAoE;KACrE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO,SAAS,CAAC;IAChD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC5D,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAiC,CAAC;IACtE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChD,OAAO;QACL,KAAK,EAAE,OAAO;QACd,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe;QACvE,QAAQ,EAAE,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;KAC9D,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAe,EACf,MAAmB,EACnB,IAAgB;IAEhB,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpE,OAAO;YACL,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,0BAA0B;SACnC,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAChF,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAG,CAAW,CAAC,OAAO,CAAC,CAAC;QAClE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAClE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC;IACpC,OAAO;QACL,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM;QACN,MAAM,EAAE,IAAI,IAAI,CAAC,UAAU,IAAI,OAAO,KAAK,MAAM,CAAC,aAAa,IAAI,oBAAoB,KAAK,OAAO,CAAC,MAAM,EAAE;QAC5G,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC;AACJ,CAAC"}