@aipack-ai/eval 1.1.7 → 1.1.8

package/dist/src/core/scorer/llm-judge.js

@@ -0,0 +1,111 @@
+/**
+ * packages/eval/src/core/scorer/llm-judge.ts - LLM-as-judge 评分器（M5）
+ *
+ * 开放性回答的终审（EVAL_PLAN.md 4.2 硬性约束）：
+ *   - judge 模型与被测模型异源（装配层校验，见 live.ts assertJudgeDistinct）
+ *   - 输出强制 JSON（score + reason + evidence），解析失败视为 scorer
+ *     自身错误 → 跳过该评分器而非判用例失败
+ *   - judge prompt 版本纳入元数据（JUDGE_PROMPT_VERSION / params.promptVersion），
+ *     改 rubric 不 invalidate 全部历史结果
+ */
+export const JUDGE_PROMPT_VERSION = 1;
+function truncate(s, n) {
+    return s.length <= n ? s : `${s.slice(0, n)}…`;
+}
+function formatTrajectory(calls) {
+    if (calls.length === 0)
+        return '(无工具调用)';
+    return calls
+        .map((c, i) => `${i + 1}. ${c.name}(${JSON.stringify(c.args ?? {})})${c.isError ? ' [error]' : ''}`)
+        .join('\n');
+}
+/** 组装 judge prompt（版本化：改 rubric / 格式不影响历史结果的含义） */
+export function buildJudgePrompt(params, output, trajectory) {
+    const version = params.promptVersion ?? JUDGE_PROMPT_VERSION;
+    const criteria = params.criteria.map((c, i) => `${i + 1}. ${c}`).join('\n');
+    return [
+        `你是严格的评测 judge（prompt v${version}）。请逐条依据以下 rubric 评判 agent 回答的质量：`,
+        '',
+        criteria,
+        '',
+        '── Agent 最终输出 ──',
+        output,
+        '',
+        '── 工具调用轨迹（时间序） ──',
+        formatTrajectory(trajectory),
+        '',
+        '要求：',
+        '- score 为 0~1 的数字（全部 rubric 满足 = 1，完全不满足 = 0）',
+        '- reason 必须给出理由并引用证据',
+        '- 只输出一个 JSON 对象，不要输出其他任何文本：',
+        '  {"score": <0~1>, "reason": "<理由>", "evidence": "<命中的原文片段或调用序号>"}',
+    ].join('\n');
+}
+/**
+ * 解析 judge 输出。宽容处理：截取首个 '{' 到最后一个 '}'。
+ * 解析失败返回 undefined（scorer 自身错误 → 跳过，不判用例失败）。
+ */
+export function parseJudgeResponse(raw) {
+    const start = raw.indexOf('{');
+    const end = raw.lastIndexOf('}');
+    if (start < 0 || end <= start)
+        return undefined;
+    let parsed;
+    try {
+        parsed = JSON.parse(raw.slice(start, end + 1));
+    }
+    catch {
+        return undefined;
+    }
+    if (!parsed || typeof parsed !== 'object')
+        return undefined;
+    const { score, reason, evidence } = parsed;
+    if (typeof score !== 'number' || !Number.isFinite(score))
+        return undefined;
+    const clamped = Math.min(Math.max(score, 0), 1);
+    return {
+        score: clamped,
+        reason: typeof reason === 'string' && reason ? reason : '(judge 未给出理由)',
+        evidence: typeof evidence === 'string' ? evidence : undefined,
+    };
+}
+/**
+ * 执行 LLM-as-judge。
+ * 返回 undefined = 评分器跳过（缺 judge 装配 / judge 输出解析失败 / judge 调用失败），
+ * 不判用例失败（EVAL_PLAN.md 4.2）。
+ */
+export async function runLlmJudge(trace, params, deps) {
+    if (!deps)
+        return undefined;
+    if (!Array.isArray(params.criteria) || params.criteria.length === 0) {
+        return {
+            scorer: 'llm-judge',
+            score: 0,
+            passed: false,
+            reason: "评分器参数缺少非空数组字段 'criteria'",
+        };
+    }
+    const prompt = buildJudgePrompt(params, trace.result.content, trace.trajectory);
+    let raw;
+    try {
+        raw = await deps.complete(prompt);
+    }
+    catch (e) {
+        console.error('[eval] llm-judge 调用失败（跳过）:', e.message);
+        return undefined;
+    }
+    const verdict = parseJudgeResponse(raw);
+    if (!verdict) {
+        console.error('[eval] llm-judge 输出解析失败（跳过）:', truncate(raw, 120));
+        return undefined;
+    }
+    const passed = verdict.score >= 0.5;
+    return {
+        scorer: 'llm-judge',
+        score: verdict.score,
+        passed,
+        reason: `[${deps.modelLabel ?? 'judge'} v${params.promptVersion ?? JUDGE_PROMPT_VERSION}] ${verdict.reason}`,
+        evidence: verdict.evidence,
+    };
+}
+//# sourceMappingURL=llm-judge.js.map

package/dist/src/core/scorer/llm-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../../../src/core/scorer/llm-judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAkBH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAStC,SAAS,QAAQ,CAAC,CAAS,EAAE,CAAS;IACpC,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC;AACjD,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAuB;IAC/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzC,OAAO,KAAK;SACT,GAAG,CACF,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CACvF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,gBAAgB,CAC9B,MAAmB,EACnB,MAAc,EACd,UAA4B;IAE5B,MAAM,OAAO,GAAG,MAAM,CAAC,aAAa,IAAI,oBAAoB,CAAC;IAC7D,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5E,OAAO;QACL,yBAAyB,OAAO,kCAAkC;QAClE,EAAE;QACF,QAAQ;QACR,EAAE;QACF,kBAAkB;QAClB,MAAM;QACN,EAAE;QACF,mBAAmB;QACnB,gBAAgB,CAAC,UAAU,CAAC;QAC5B,EAAE;QACF,KAAK;QACL,+CAA+C;QAC/C,sBAAsB;QACtB,6BAA6B;QAC7B,oEAAoE;KACrE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO,SAAS,CAAC;IAChD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC5D,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAiC,CAAC;IACtE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChD,OAAO;QACL,KAAK,EAAE,OAAO;QACd,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe;QACvE,QAAQ,EAAE,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;KAC9D,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAe,EACf,MAAmB,EACnB,IAAgB;IAEhB,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpE,OAAO;YACL,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,0BAA0B;SACnC,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAChF,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAG,CAAW,CAAC,OAAO,CAAC,CAAC;QAClE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAClE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC;IACpC,OAAO;QACL,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM;QACN,MAAM,EAAE,IAAI,IAAI,CAAC,UAAU,IAAI,OAAO,KAAK,MAAM,CAAC,aAAa,IAAI,oBAAoB,KAAK,OAAO,CAAC,MAAM,EAAE;QAC5G,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC;AACJ,CAAC"}

package/dist/src/core/scorer/semantic.d.ts

@@ -0,0 +1,27 @@
+/**
+ * packages/eval/src/core/scorer/semantic.ts - 语义评分器（M5）
+ *
+ * embedding 相似度粗筛开放性回答：期望文本与最终输出各算一次 embedding，
+ * 余弦相似度 ≥ threshold 即通过。低成本低精度，只做开放性回答的粗筛
+ * （EVAL_PLAN.md 4.2：语义评分器）。
+ *
+ * embedding 注入点（EmbedFn）由调用方装配（见 core/embedding.ts 的
+ * OpenAI 兼容实现），单测可注入 stub；缺 embed 时评分器跳过（不判失败）。
+ */
+import type { RunTrace, ScoreResult } from '../types';
+/** 文本 → 向量（由调用方注入：真实 provider / 测试 stub） */
+export type EmbedFn = (text: string) => Promise<number[]>;
+export interface SemanticParams {
+    /** 期望语义的参考文本 */
+    expected: string;
+    /** 通过阈值（缺省 0.75，0~1） */
+    threshold?: number;
+}
+export declare const SEMANTIC_DEFAULT_THRESHOLD = 0.75;
+/** 余弦相似度（零向量返回 0） */
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+/**
+ * 执行语义评分。
+ * 返回 undefined = 评分器跳过（缺 embed 注入点），不判用例失败。
+ */
+export declare function runSemanticScorer(trace: RunTrace, params: SemanticParams, embed?: EmbedFn): Promise<ScoreResult | undefined>;

package/dist/src/core/scorer/semantic.js

@@ -0,0 +1,74 @@
+/**
+ * packages/eval/src/core/scorer/semantic.ts - 语义评分器（M5）
+ *
+ * embedding 相似度粗筛开放性回答：期望文本与最终输出各算一次 embedding，
+ * 余弦相似度 ≥ threshold 即通过。低成本低精度，只做开放性回答的粗筛
+ * （EVAL_PLAN.md 4.2：语义评分器）。
+ *
+ * embedding 注入点（EmbedFn）由调用方装配（见 core/embedding.ts 的
+ * OpenAI 兼容实现），单测可注入 stub；缺 embed 时评分器跳过（不判失败）。
+ */
+export const SEMANTIC_DEFAULT_THRESHOLD = 0.75;
+/** 余弦相似度（零向量返回 0） */
+export function cosineSimilarity(a, b) {
+    if (a.length === 0 || a.length !== b.length)
+        return 0;
+    let dot = 0;
+    let na = 0;
+    let nb = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        na += a[i] * a[i];
+        nb += b[i] * b[i];
+    }
+    if (na === 0 || nb === 0)
+        return 0;
+    return dot / Math.sqrt(na * nb);
+}
+function truncate(s, n) {
+    return s.length <= n ? s : `${s.slice(0, n)}…`;
+}
+/**
+ * 执行语义评分。
+ * 返回 undefined = 评分器跳过（缺 embed 注入点），不判用例失败。
+ */
+export async function runSemanticScorer(trace, params, embed) {
+    if (!embed) {
+        return undefined;
+    }
+    const expected = params.expected;
+    if (typeof expected !== 'string' || !expected) {
+        return {
+            scorer: 'semantic',
+            score: 0,
+            passed: false,
+            reason: "评分器参数缺少字符串字段 'expected'",
+        };
+    }
+    const threshold = typeof params.threshold === 'number' && params.threshold > 0 && params.threshold <= 1
+        ? params.threshold
+        : SEMANTIC_DEFAULT_THRESHOLD;
+    try {
+        const [expVec, actVec] = await Promise.all([
+            embed(expected),
+            embed(trace.result.content),
+        ]);
+        const sim = cosineSimilarity(expVec, actVec);
+        const passed = sim >= threshold;
+        return {
+            scorer: 'semantic',
+            score: sim,
+            passed,
+            reason: passed
+                ? `语义相似度 ${sim.toFixed(3)} ≥ 阈值 ${threshold}`
+                : `语义相似度 ${sim.toFixed(3)} < 阈值 ${threshold}`,
+            evidence: `expected: ${truncate(expected, 80)} / actual: ${truncate(trace.result.content, 80)}`,
+        };
+    }
+    catch (e) {
+        // embedding 调用失败视为 scorer 自身错误，不判用例失败（跳过）
+        console.error('[eval] semantic scorer embedding 失败（跳过）:', e.message);
+        return undefined;
+    }
+}
+//# sourceMappingURL=semantic.js.map

package/dist/src/core/scorer/semantic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic.js","sourceRoot":"","sources":["../../../../src/core/scorer/semantic.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAcH,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC;AAE/C,qBAAqB;AACrB,MAAM,UAAU,gBAAgB,CAAC,CAAW,EAAE,CAAW;IACvD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACtD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IACD,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnC,OAAO,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,CAAS;IACpC,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC;AACjD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAe,EACf,MAAsB,EACtB,KAAe;IAEf,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IACjC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9C,OAAO;YACL,MAAM,EAAE,UAAU;YAClB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,yBAAyB;SAClC,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GACb,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,IAAI,MAAM,CAAC,SAAS,IAAI,CAAC;QACnF,CAAC,CAAC,MAAM,CAAC,SAAS;QAClB,CAAC,CAAC,0BAA0B,CAAC;IAEjC,IAAI,CAAC;QACH,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACzC,KAAK,CAAC,QAAQ,CAAC;YACf,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;SAC5B,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,GAAG,IAAI,SAAS,CAAC;QAChC,OAAO;YACL,MAAM,EAAE,UAAU;YAClB,KAAK,EAAE,GAAG;YACV,MAAM;YACN,MAAM,EAAE,MAAM;gBACZ,CAAC,CAAC,SAAS,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,SAAS,EAAE;gBAC7C,CAAC,CAAC,SAAS,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,SAAS,EAAE;YAC/C,QAAQ,EAAE,aAAa,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,cAAc,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE;SAChG,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,0CAA0C;QAC1C,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAG,CAAW,CAAC,OAAO,CAAC,CAAC;QAChF,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}

package/dist/src/core/types.d.ts

@@ -265,6 +265,25 @@ export interface RunConfig {
     maxTotalTokens?: number;
     /** live 模式单请求超时 ms */
     requestTimeoutMs?: number;
+    /**
+     * LLM-as-judge 的异源模型（'provider/modelId'）；缺省读 AIPACK_EVAL_JUDGE_MODEL。
+     * 用例含 llm-judge 评分器时必填；与被测模型相同则拒绝（同源偏置）。
+     */
+    judgeModel?: string;
+    /** judge API Key（缺省 AIPACK_EVAL_JUDGE_API_KEY / <PROVIDER>_API_KEY） */
+    judgeApiKey?: string;
+    /** judge 端点覆盖 */
+    judgeBaseUrl?: string;
+    /** semantic 评分器的 embedding 模型；缺省读 AIPACK_EVAL_EMBEDDING_MODEL */
+    embedModel?: string;
+    /** embedding API Key（缺省 AIPACK_EVAL_EMBEDDING_API_KEY / OPENAI_API_KEY） */
+    embedApiKey?: string;
+    /** embedding 端点覆盖（OpenAI 兼容 /embeddings） */
+    embedBaseUrl?: string;
+    /** 直接注入 judge StreamFn（自托管 / 测试用），配合 judgeFrameworkModel */
+    judgeStreamFn?: StreamFn;
+    /** 与 judgeStreamFn 配套的框架 Model */
+    judgeFrameworkModel?: Model;
 }
 export interface BaselineFile {
     /** 生成 baseline 的 runId */

package/dist/src/core/validate.js

@@ -3,7 +3,7 @@
  *
  * loader 加载 JSON 后立即校验，坏用例在运行前暴露（fail fast）。
  */
-import { RULE_SCORER_TYPES } from './scorer/rule.js';
+import { ALL_SCORER_TYPES } from './scorer/index.js';
 const ORIGINS = [
     'handwritten',
     'trace',
@@ -65,8 +65,8 @@ export function validateEvalCase(c) {
         }
     }
     for (const s of c.scorers ?? []) {
-        if (!RULE_SCORER_TYPES.includes(s.type)) {
-            errors.push(at(`scorers 含未知类型 '${s.type}'（支持: ${RULE_SCORER_TYPES.join(', ')}）`));
+        if (!ALL_SCORER_TYPES.includes(s.type)) {
+            errors.push(at(`scorers 含未知类型 '${s.type}'（支持: ${ALL_SCORER_TYPES.join(', ')}）`));
         }
     }
     const hasScorers = Boolean(c.expected) || (c.scorers?.length ?? 0) > 0;

package/dist/src/core/validate.js.map

@@ -1 +1 @@
-{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../../src/core/validate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,MAAM,OAAO,GAAiB;IAC5B,aAAa;IACb,OAAO;IACP,WAAW;IACX,SAAS;IACT,QAAQ;CACT,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,OAAO;IACP,UAAU;IACV,OAAO;IACP,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,SAAS;IACT,aAAa;CACd,CAAC;AAEF,MAAM,UAAU,gBAAgB,CAAC,CAAW;IAC1C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,EAAE,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,cAAc,KAAK,GAAG,EAAE,CAAC;IAEjE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SACvD,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;IACzE,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC5D,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACjB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,0BAA0B,CAAC,CAAC,CAAC;QAC9C,CAAC;aAAM,CAAC;YACN,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBAClC,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBACzC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,wBAAwB,CAAC,CAAC,CAAC;gBAC3D,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACnE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,0BAA0B,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,uCAAuC,CAAC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,IAAI,GAAI,CAAC,CAAC,QAA8B,CAAC,IAAI,CAAC;QACpD,IAAI,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,sBAAsB,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,IAAI,SAAS,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACvE,IAAI,CAAC,UAAU;QAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,CAAC,CAAC;IAE9D,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../../src/core/validate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAE5C,MAAM,OAAO,GAAiB;IAC5B,aAAa;IACb,OAAO;IACP,WAAW;IACX,SAAS;IACT,QAAQ;CACT,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,OAAO;IACP,UAAU;IACV,OAAO;IACP,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,SAAS;IACT,aAAa;CACd,CAAC;AAEF,MAAM,UAAU,gBAAgB,CAAC,CAAW;IAC1C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,EAAE,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,cAAc,KAAK,GAAG,EAAE,CAAC;IAEjE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SACvD,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;IACzE,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC5D,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACjB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,0BAA0B,CAAC,CAAC,CAAC;QAC9C,CAAC;aAAM,CAAC;YACN,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBAClC,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBACzC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,wBAAwB,CAAC,CAAC,CAAC;gBAC3D,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACnE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,0BAA0B,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,uCAAuC,CAAC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,IAAI,GAAI,CAAC,CAAC,QAA8B,CAAC,IAAI,CAAC;QACpD,IAAI,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,sBAAsB,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAkB,CAAC,EAAE,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,IAAI,SAAS,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACvE,IAAI,CAAC,UAAU;QAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,CAAC,CAAC;IAE9D,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aipack-ai/eval",
-  "version": "1.1.7",
+  "version": "1.1.8",
   "description": "aipack Eval 体系：EvalCase 格式 + 规则评分器 + Runner（mock fixture replay）+ 报告与 baseline 门禁",
   "type": "module",
   "main": "./dist/index.js",
@@ -35,7 +35,7 @@
     "node": ">=18.0.0"
   },
   "dependencies": {
-    "@aipack-ai/agent": "1.1.7"
+    "@aipack-ai/agent": "1.1.8"
   },
   "devDependencies": {
     "@types/node": "^20.12.0",