@aipack-ai/eval 1.1.6 → 1.1.8

package/dist/src/core/live.d.ts

@@ -0,0 +1,111 @@
+/**
+ * packages/eval/src/core/live.ts - live 模式（真实 LLM）装配
+ *
+ * 职责：把「provider/modelId + apiKey + baseUrl」装配成 Runtime 可直接消费的
+ * `{ model, streamFn }`，与 mock 模式的 fixture replay 对称。
+ *
+ * 消随机性（EVAL_PLAN.md 4.3）：
+ *   - temperature 缺省 0（provider 支持即生效）
+ *   - 本仓库适配层不支持 seed → 靠 repeats + pass@k 消噪，见 runner.ts
+ *
+ * 装配路径：getBuiltinModel（内置目录）→ 兜底构造 custom model（代理 / 兼容网关）
+ *   → adaptAiModel（框架 Model）+ createStreamFnFromAi（StreamFn）
+ */
+import type { Model as AiModel } from '@aipack-ai/agent/ai';
+import type { Model, StreamFn } from '@aipack-ai/agent';
+export interface LiveLlmSpec {
+    /** 提供商 id：deepseek / openai / anthropic / ... */
+    provider: string;
+    /** 模型 id：deepseek-chat / gpt-4o-mini / ...（不在内置目录时按 custom 兜底） */
+    modelId: string;
+    /** API Key；缺省取 <PROVIDER>_API_KEY 环境变量 */
+    apiKey?: string;
+    /** 覆盖内置端点（代理 / 兼容网关） */
+    baseUrl?: string;
+    /** 缺省 0（消随机性） */
+    temperature?: number;
+    /** 单次请求超时 ms */
+    timeoutMs?: number;
+}
+export interface LiveLlm {
+    spec: LiveLlmSpec;
+    /** AI 层模型（含 api / baseUrl） */
+    aiModel: AiModel;
+    /** 框架层 Model，放进 RuntimeOptions.model */
+    model: Model;
+    /** 真实流式函数，放进 RuntimeOptions.streamFn */
+    streamFn: StreamFn;
+}
+/** env 覆盖键（优先级低于显式参数，便于 CI 用 secrets 注入） */
+export declare const LIVE_ENV_KEYS: {
+    readonly model: "AIPACK_EVAL_MODEL";
+    readonly provider: "AIPACK_EVAL_PROVIDER";
+    readonly apiKey: "AIPACK_EVAL_API_KEY";
+    readonly baseUrl: "AIPACK_EVAL_BASE_URL";
+    readonly temperature: "AIPACK_EVAL_TEMPERATURE";
+};
+/**
+ * 解析 'provider/modelId' 或 'modelId'。
+ * 仅给 modelId 时：优先 AIPACK_EVAL_PROVIDER，否则探测第一个已配置 Key 的 provider。
+ */
+export declare function parseModelSpec(spec: string): {
+    provider: string;
+    modelId: string;
+};
+export interface ResolveLiveOptions {
+    /** 'provider/modelId' 或 'modelId'；缺省读 AIPACK_EVAL_MODEL */
+    model?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    temperature?: number;
+    timeoutMs?: number;
+}
+/** 合并显式参数与环境变量，得到最终生效的 LiveLlmSpec */
+export declare function resolveLiveSpec(opts?: ResolveLiveOptions): LiveLlmSpec;
+/**
+ * 装配真实 LLM。不发起任何网络请求（只是闭包），可安全单测。
+ *
+ * 注意：createStreamFnFromAi 优先使用 runtime 传入的 model，
+ * 因此必须把返回的 model 一起交给 createRuntime，否则 api 分派会不一致。
+ */
+export declare function createLiveLlm(spec: LiveLlmSpec): LiveLlm;
+/** 从 CLI/库参数一步到位：解析 spec → 装配 LLM */
+export declare function resolveLiveLlm(opts?: ResolveLiveOptions): LiveLlm;
+/** 报告 / 日志用的模型标识（'deepseek/deepseek-chat'） */
+export declare function describeLiveLlm(spec: LiveLlmSpec): string;
+/** live 模式前置检查：缺 Key 时给出可操作的错误信息 */
+export declare function assertLiveReady(spec: LiveLlmSpec): void;
+/** judge 模型的 env 覆盖键（与被测模型的键分离，避免混用） */
+export declare const JUDGE_ENV_KEYS: {
+    readonly model: "AIPACK_EVAL_JUDGE_MODEL";
+    readonly apiKey: "AIPACK_EVAL_JUDGE_API_KEY";
+    readonly baseUrl: "AIPACK_EVAL_JUDGE_BASE_URL";
+};
+/**
+ * 解析 judge 模型 spec：显式参数优先，缺省读 AIPACK_EVAL_JUDGE_MODEL。
+ * 复用 parseModelSpec（'provider/modelId' 或探测已配置 Key 的 provider）。
+ */
+export declare function resolveJudgeSpec(opts: {
+    model?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    timeoutMs?: number;
+}): LiveLlmSpec;
+/**
+ * 异源硬性约束（EVAL_PLAN.md 4.2）：judge 与被测模型完全相同 → 拒绝；
+ * 同 provider 不同模型 → 允许但告警（同源偏置风险低于同模型，仍建议跨 provider）。
+ */
+export declare function assertJudgeDistinct(target: {
+    provider: string;
+    modelId: string;
+} | undefined, judge: {
+    provider: string;
+    modelId: string;
+}): void;
+/** 从 CLI/库参数一步装配 judge LLM（complete 由调用方基于 streamFn 包装） */
+export declare function resolveJudgeLlm(opts: {
+    model?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    timeoutMs?: number;
+}): LiveLlm;

package/dist/src/core/live.js

@@ -0,0 +1,187 @@
+/**
+ * packages/eval/src/core/live.ts - live 模式（真实 LLM）装配
+ *
+ * 职责：把「provider/modelId + apiKey + baseUrl」装配成 Runtime 可直接消费的
+ * `{ model, streamFn }`，与 mock 模式的 fixture replay 对称。
+ *
+ * 消随机性（EVAL_PLAN.md 4.3）：
+ *   - temperature 缺省 0（provider 支持即生效）
+ *   - 本仓库适配层不支持 seed → 靠 repeats + pass@k 消噪，见 runner.ts
+ *
+ * 装配路径：getBuiltinModel（内置目录）→ 兜底构造 custom model（代理 / 兼容网关）
+ *   → adaptAiModel（框架 Model）+ createStreamFnFromAi（StreamFn）
+ */
+import { BUILTIN_PROVIDERS, adaptAiModel, createStreamFnFromAi, getBuiltinModel, getEnvApiKey, } from '@aipack-ai/agent';
+/** env 覆盖键（优先级低于显式参数，便于 CI 用 secrets 注入） */
+export const LIVE_ENV_KEYS = {
+    model: 'AIPACK_EVAL_MODEL',
+    provider: 'AIPACK_EVAL_PROVIDER',
+    apiKey: 'AIPACK_EVAL_API_KEY',
+    baseUrl: 'AIPACK_EVAL_BASE_URL',
+    temperature: 'AIPACK_EVAL_TEMPERATURE',
+};
+/** 未显式指定 provider 时的探测顺序（与 CLI 默认策略一致：低成本在前） */
+const PROBE_ORDER = [
+    'deepseek',
+    'openai',
+    'anthropic',
+    'groq',
+    'moonshot',
+    'google',
+    'openrouter',
+    'mistral',
+    'xai',
+    'cerebras',
+    'together',
+    'fireworks',
+    'nvidia',
+];
+// ─── 解析 ─────────────────────────────────────────────────────────
+/**
+ * 解析 'provider/modelId' 或 'modelId'。
+ * 仅给 modelId 时：优先 AIPACK_EVAL_PROVIDER，否则探测第一个已配置 Key 的 provider。
+ */
+export function parseModelSpec(spec) {
+    const trimmed = spec.trim();
+    const slash = trimmed.indexOf('/');
+    if (slash > 0) {
+        return {
+            provider: trimmed.slice(0, slash),
+            modelId: trimmed.slice(slash + 1),
+        };
+    }
+    const envProvider = process.env[LIVE_ENV_KEYS.provider];
+    if (envProvider)
+        return { provider: envProvider, modelId: trimmed };
+    const configured = PROBE_ORDER.find((p) => Boolean(getEnvApiKey(p)));
+    if (configured)
+        return { provider: configured, modelId: trimmed };
+    throw new Error(`无法从 '${spec}' 推断 provider：请写全 'provider/modelId'，或设置 ${LIVE_ENV_KEYS.provider} / <PROVIDER>_API_KEY`);
+}
+/** 合并显式参数与环境变量，得到最终生效的 LiveLlmSpec */
+export function resolveLiveSpec(opts = {}) {
+    const env = process.env;
+    const raw = opts.model ?? env[LIVE_ENV_KEYS.model];
+    if (!raw) {
+        throw new Error(`live 模式需要指定模型：--model provider/modelId 或 ${LIVE_ENV_KEYS.model}（如 deepseek/deepseek-chat）`);
+    }
+    const { provider, modelId } = parseModelSpec(raw);
+    const envTemp = env[LIVE_ENV_KEYS.temperature];
+    const temperature = opts.temperature ?? (envTemp !== undefined && envTemp !== '' ? Number(envTemp) : undefined);
+    const spec = {
+        provider,
+        modelId,
+        apiKey: opts.apiKey ?? env[LIVE_ENV_KEYS.apiKey] ?? getEnvApiKey(provider),
+        baseUrl: opts.baseUrl ?? env[LIVE_ENV_KEYS.baseUrl],
+        temperature: temperature ?? 0,
+    };
+    if (opts.timeoutMs !== undefined)
+        spec.timeoutMs = opts.timeoutMs;
+    return spec;
+}
+// ─── 装配 ─────────────────────────────────────────────────────────
+/** 内置目录缺失时的兜底：按 provider 推断 api 与端点 */
+function buildCustomModel(provider, modelId) {
+    const meta = BUILTIN_PROVIDERS.find((p) => p.id === provider);
+    const baseUrl = meta?.baseUrl;
+    if (!baseUrl) {
+        throw new Error(`未知 provider '${provider}'：内置提供商为 ${BUILTIN_PROVIDERS.map((p) => p.id).join(', ')}`);
+    }
+    return {
+        id: modelId,
+        name: modelId,
+        api: provider === 'anthropic' ? 'anthropic-messages' : 'openai-completions',
+        provider,
+        baseUrl,
+        reasoning: false,
+        input: ['text'],
+        contextWindow: 128_000,
+        maxTokens: 8_192,
+    };
+}
+/**
+ * 装配真实 LLM。不发起任何网络请求（只是闭包），可安全单测。
+ *
+ * 注意：createStreamFnFromAi 优先使用 runtime 传入的 model，
+ * 因此必须把返回的 model 一起交给 createRuntime，否则 api 分派会不一致。
+ */
+export function createLiveLlm(spec) {
+    const base = getBuiltinModel(spec.provider, spec.modelId);
+    const aiModel = base
+        ? spec.baseUrl
+            ? { ...base, baseUrl: spec.baseUrl }
+            : base
+        : buildCustomModel(spec.provider, spec.modelId);
+    const options = { temperature: spec.temperature ?? 0 };
+    if (spec.apiKey)
+        options.apiKey = spec.apiKey;
+    if (spec.timeoutMs !== undefined)
+        options.timeoutMs = spec.timeoutMs;
+    return {
+        spec,
+        aiModel,
+        model: adaptAiModel(aiModel),
+        streamFn: createStreamFnFromAi(aiModel, options),
+    };
+}
+/** 从 CLI/库参数一步到位：解析 spec → 装配 LLM */
+export function resolveLiveLlm(opts = {}) {
+    return createLiveLlm(resolveLiveSpec(opts));
+}
+/** 报告 / 日志用的模型标识（'deepseek/deepseek-chat'） */
+export function describeLiveLlm(spec) {
+    return `${spec.provider}/${spec.modelId}`;
+}
+/** live 模式前置检查：缺 Key 时给出可操作的错误信息 */
+export function assertLiveReady(spec) {
+    if (spec.apiKey)
+        return;
+    const envName = BUILTIN_PROVIDERS.find((p) => p.id === spec.provider)?.envVar;
+    throw new Error(`provider '${spec.provider}' 未配置 API Key：--api-key <key>、${LIVE_ENV_KEYS.apiKey} 或 ${envName ?? '<PROVIDER>_API_KEY'}`);
+}
+// ─── judge 装配（M5：LLM-as-judge）────────────────────────────────
+/** judge 模型的 env 覆盖键（与被测模型的键分离，避免混用） */
+export const JUDGE_ENV_KEYS = {
+    model: 'AIPACK_EVAL_JUDGE_MODEL',
+    apiKey: 'AIPACK_EVAL_JUDGE_API_KEY',
+    baseUrl: 'AIPACK_EVAL_JUDGE_BASE_URL',
+};
+/**
+ * 解析 judge 模型 spec：显式参数优先，缺省读 AIPACK_EVAL_JUDGE_MODEL。
+ * 复用 parseModelSpec（'provider/modelId' 或探测已配置 Key 的 provider）。
+ */
+export function resolveJudgeSpec(opts) {
+    const env = process.env;
+    const raw = opts.model ?? env[JUDGE_ENV_KEYS.model];
+    if (!raw) {
+        throw new Error(`llm-judge 需要 judge 模型：--judge-model provider/modelId 或 ${JUDGE_ENV_KEYS.model}（须与被测模型异源）`);
+    }
+    const { provider, modelId } = parseModelSpec(raw);
+    return {
+        provider,
+        modelId,
+        apiKey: opts.apiKey ?? env[JUDGE_ENV_KEYS.apiKey] ?? getEnvApiKey(provider),
+        baseUrl: opts.baseUrl ?? env[JUDGE_ENV_KEYS.baseUrl],
+        temperature: 0, // judge 要确定性
+        timeoutMs: opts.timeoutMs,
+    };
+}
+/**
+ * 异源硬性约束（EVAL_PLAN.md 4.2）：judge 与被测模型完全相同 → 拒绝；
+ * 同 provider 不同模型 → 允许但告警（同源偏置风险低于同模型，仍建议跨 provider）。
+ */
+export function assertJudgeDistinct(target, judge) {
+    if (!target)
+        return;
+    if (target.provider === judge.provider && target.modelId === judge.modelId) {
+        throw new Error(`judge 模型 (${judge.provider}/${judge.modelId}) 与被测模型相同：LLM-as-judge 要求异源（换 provider 或换 modelId）`);
+    }
+    if (target.provider === judge.provider) {
+        console.warn(`[eval] judge 与被测模型同 provider (${judge.provider})：存在同源偏置风险，建议跨 provider 选取 judge`);
+    }
+}
+/** 从 CLI/库参数一步装配 judge LLM（complete 由调用方基于 streamFn 包装） */
+export function resolveJudgeLlm(opts) {
+    return createLiveLlm(resolveJudgeSpec(opts));
+}
+//# sourceMappingURL=live.js.map

package/dist/src/core/live.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"live.js","sourceRoot":"","sources":["../../../src/core/live.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EACL,iBAAiB,EACjB,YAAY,EACZ,oBAAoB,EACpB,eAAe,EACf,YAAY,GACb,MAAM,kBAAkB,CAAC;AAgC1B,4CAA4C;AAC5C,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,KAAK,EAAE,mBAAmB;IAC1B,QAAQ,EAAE,sBAAsB;IAChC,MAAM,EAAE,qBAAqB;IAC7B,OAAO,EAAE,sBAAsB;IAC/B,WAAW,EAAE,yBAAyB;CAC9B,CAAC;AAEX,gDAAgD;AAChD,MAAM,WAAW,GAAG;IAClB,UAAU;IACV,QAAQ;IACR,WAAW;IACX,MAAM;IACN,UAAU;IACV,QAAQ;IACR,YAAY;IACZ,SAAS;IACT,KAAK;IACL,UAAU;IACV,UAAU;IACV,WAAW;IACX,QAAQ;CACT,CAAC;AAEF,mEAAmE;AAEnE;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,OAAO;YACL,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;YACjC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;SAClC,CAAC;IACJ,CAAC;IACD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACxD,IAAI,WAAW;QAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IACpE,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,IAAI,UAAU;QAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAClE,MAAM,IAAI,KAAK,CACb,QAAQ,IAAI,4CAA4C,aAAa,CAAC,QAAQ,uBAAuB,CACtG,CAAC;AACJ,CAAC;AAWD,sCAAsC;AACtC,MAAM,UAAU,eAAe,CAAC,OAA2B,EAAE;IAC3D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IACxB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,4CAA4C,aAAa,CAAC,KAAK,4BAA4B,CAC5F,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAElD,MAAM,OAAO,GAAG,GAAG,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;IAC/C,MAAM,WAAW,GACf,IAAI,CAAC,WAAW,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAE9F,MAAM,IAAI,GAAgB;QACxB,QAAQ;QACR,OAAO;QACP,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC;QAC1E,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC;QACnD,WAAW,EAAE,WAAW,IAAI,CAAC;KAC9B,CAAC;IACF,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;QAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,mEAAmE;AAEnE,uCAAuC;AACvC,SAAS,gBAAgB,CAAC,QAAgB,EAAE,OAAe;IACzD,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,IAAI,EAAE,OAAO,CAAC;IAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,gBAAgB,QAAQ,YAAY,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpF,CAAC;IACJ,CAAC;IACD,OAAO;QACL,EAAE,EAAE,OAAO;QACX,IAAI,EAAE,OAAO;QACb,GAAG,EAAE,QAAQ,KAAK,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,oBAAoB;QAC3E,QAAQ;QACR,OAAO;QACP,SAAS,EAAE,KAAK;QAChB,KAAK,EAAE,CAAC,MAAM,CAAC;QACf,aAAa,EAAE,OAAO;QACtB,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,IAAiB;IAC7C,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY,IAAI;QAC3B,CAAC,CAAC,IAAI,CAAC,OAAO;YACZ,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;YACpC,CAAC,CAAC,IAAI;QACR,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAElD,MAAM,OAAO,GAAwB,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;IAC5E,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC9C,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;QAAE,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAErE,OAAO;QACL,IAAI;QACJ,OAAO;QACP,KAAK,EAAE,YAAY,CAAC,OAAO,CAAC;QAC5B,QAAQ,EAAE,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,cAAc,CAAC,OAA2B,EAAE;IAC1D,OAAO,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,eAAe,CAAC,IAAiB;IAC/C,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;AAC5C,CAAC;AAED,oCAAoC;AACpC,MAAM,UAAU,eAAe,CAAC,IAAiB;IAC/C,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO;IACxB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC9E,MAAM,IAAI,KAAK,CACb,aAAa,IAAI,CAAC,QAAQ,iCAAiC,aAAa,CAAC,MAAM,MAAM,OAAO,IAAI,oBAAoB,EAAE,CACvH,CAAC;AACJ,CAAC;AAED,gEAAgE;AAEhE,wCAAwC;AACxC,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,KAAK,EAAE,yBAAyB;IAChC,MAAM,EAAE,2BAA2B;IACnC,OAAO,EAAE,4BAA4B;CAC7B,CAAC;AAEX;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAKhC;IACC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IACxB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IACpD,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,0DAA0D,cAAc,CAAC,KAAK,YAAY,CAC3F,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAClD,OAAO;QACL,QAAQ;QACR,OAAO;QACP,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC;QAC3E,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,cAAc,CAAC,OAAO,CAAC;QACpD,WAAW,EAAE,CAAC,EAAE,aAAa;QAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAyD,EACzD,KAA4C;IAE5C,IAAI,CAAC,MAAM;QAAE,OAAO;IACpB,IAAI,MAAM,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CACb,aAAa,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,oDAAoD,CACjG,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,EAAE,CAAC;QACvC,OAAO,CAAC,IAAI,CACV,iCAAiC,KAAK,CAAC,QAAQ,kCAAkC,CAClF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,eAAe,CAAC,IAK/B;IACC,OAAO,aAAa,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;AAC/C,CAAC"}

package/dist/src/core/report.d.ts

@@ -4,9 +4,14 @@
  * - JSON + Markdown 双格式输出到报告目录（缺省 ./eval-results）
  * - baseline：对比整体与分套件通过率，回归超阈值判失败
  */
+import type { HistoryEntry } from './history';
 import type { BaselineComparison, BaselineFile, EvalReport, RunConfig } from './types';
-export declare function renderMarkdown(report: EvalReport): string;
-export declare function writeReport(report: EvalReport, reportDir: string): Promise<{
+export interface RenderOptions {
+    /** 历史条目（含本次）→ 渲染通过率日环比趋势段 */
+    history?: HistoryEntry[];
+}
+export declare function renderMarkdown(report: EvalReport, opts?: RenderOptions): string;
+export declare function writeReport(report: EvalReport, reportDir: string, opts?: RenderOptions): Promise<{
     jsonPath: string;
     mdPath: string;
 }>;
@@ -23,4 +28,6 @@ export declare function finalizeReport(report: EvalReport, config: RunConfig): P
     mdPath?: string;
     baseline?: BaselineFile;
     comparison?: BaselineComparison;
+    historyPath?: string;
+    history?: HistoryEntry[];
 }>;

package/dist/src/core/report.js

@@ -6,6 +6,7 @@
  */
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname, join, resolve } from 'node:path';
+import { appendHistory, readHistory, renderHistoryTrend } from './history.js';
 // ─── Markdown 渲染 ────────────────────────────────────────────────
 function pct(x) {
     return `${(x * 100).toFixed(1)}%`;
@@ -20,12 +21,15 @@ function renderSummaryTable(byKey, keyHeader) {
         ...rows,
     ].join('\n');
 }
-export function renderMarkdown(report) {
+export function renderMarkdown(report, opts = {}) {
     const lines = [];
     lines.push(`# aipack Eval 报告`);
     lines.push('');
-    lines.push(`- Run: \`${report.runId}\`（mode: ${report.mode}）`);
+    lines.push(`- Run: \`${report.runId}\`（mode: ${report.mode}${report.model ? `, model: ${report.model}` : ''}）`);
     lines.push(`- 总览: **${report.totals.passed}/${report.totals.cases} 通过（${pct(report.totals.passRate)}）**，平均分 ${report.totals.avgScore.toFixed(2)}，耗时 ${(report.durationMs / 1000).toFixed(2)}s`);
+    if (report.totals.skipped > 0) {
+        lines.push(`- 跳过: ${report.totals.skipped} 个用例（模式不匹配，不计入通过率）`);
+    }
     lines.push('');
     lines.push(`## 按套件`);
     lines.push('');
@@ -35,6 +39,20 @@ export function renderMarkdown(report) {
     lines.push('');
     lines.push(renderSummaryTable(report.byOrigin, 'origin'));
     lines.push('');
+    if (opts.history && opts.history.length > 0) {
+        lines.push(`## 通过率趋势（环比）`);
+        lines.push('');
+        lines.push(renderHistoryTrend(opts.history));
+        lines.push('');
+    }
+    const skipped = report.skipped ?? [];
+    if (skipped.length > 0) {
+        lines.push(`## 跳过用例（${skipped.length}）`);
+        lines.push('');
+        for (const s of skipped)
+            lines.push(`- \`${s.caseId}\`: ${s.reason}`);
+        lines.push('');
+    }
     const failures = report.results.filter((r) => !r.passed);
     if (failures.length === 0) {
         lines.push(`## 失败用例`);
@@ -59,13 +77,13 @@ export function renderMarkdown(report) {
     return lines.join('\n');
 }
 // ─── 报告写盘 ─────────────────────────────────────────────────────
-export async function writeReport(report, reportDir) {
+export async function writeReport(report, reportDir, opts = {}) {
     const dir = resolve(reportDir);
     await mkdir(dir, { recursive: true });
     const jsonPath = join(dir, `${report.runId}.json`);
     const mdPath = join(dir, `${report.runId}.md`);
     await writeFile(jsonPath, JSON.stringify(report, null, 2), 'utf-8');
-    await writeFile(mdPath, renderMarkdown(report), 'utf-8');
+    await writeFile(mdPath, renderMarkdown(report, opts), 'utf-8');
     return { jsonPath, mdPath };
 }
 // ─── Baseline ─────────────────────────────────────────────────────
@@ -129,8 +147,15 @@ export async function readBaseline(baselinePath) {
 // ─── 便捷：跑完即出报告 ───────────────────────────────────────────
 export async function finalizeReport(report, config) {
     const out = {};
+    // 先写盘再读回：趋势段包含本次运行
+    if (config.historyPath) {
+        out.historyPath = await appendHistory(report, config.historyPath);
+        out.history = await readHistory(out.historyPath);
+    }
     if (config.reportDir) {
-        const { jsonPath, mdPath } = await writeReport(report, config.reportDir);
+        const { jsonPath, mdPath } = await writeReport(report, config.reportDir, {
+            history: out.history,
+        });
         out.jsonPath = jsonPath;
         out.mdPath = mdPath;
     }

package/dist/src/core/report.js.map

@@ -1 +1 @@
-{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/core/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASnD,mEAAmE;AAEnE,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAmC,EACnC,SAAiB;IAEjB,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;SAC/B,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SACtC,GAAG,CACF,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CACT,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACxF,CAAC;IACJ,OAAO;QACL,KAAK,SAAS,2BAA2B;QACzC,uBAAuB;QACvB,GAAG,IAAI;KACR,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAkB;IAC/C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,KAAK,YAAY,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;IAC/D,KAAK,CAAC,IAAI,CACR,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACpL,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC1D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,IAAI,CAAC,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC9C,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,CAAC,CAAC,CAAC,MAAM;oBAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3D,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,iEAAiE;AAEjE,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAkB,EAClB,SAAiB;IAEjB,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/B,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,OAAO,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC;IAC/C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACpE,MAAM,SAAS,CAAC,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;IACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC;AAED,qEAAqE;AAErE,MAAM,UAAU,gBAAgB,CAAC,MAAkB;IACjD,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE;YACN,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;YAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC5B,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ;SACjC;QACD,OAAO,EAAE,MAAM,CAAC,WAAW,CACzB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YAC7C,CAAC;YACD,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE;SAC3D,CAAC,CACH;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAkB,EAClB,YAAoB;IAEpB,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACnC,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CACb,IAAI,EACJ,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,EACjD,OAAO,CACR,CAAC;IACF,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAkB,EAClB,QAAsB,EACtB,SAAS,GAAG,IAAI;IAEhB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;IACvE,MAAM,WAAW,GAAsC,EAAE,CAAC;IAE1D,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1D,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,GAAG;YAAE,SAAS,CAAC,2BAA2B;QAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;QACxC,IAAI,KAAK,GAAG,CAAC,SAAS,EAAE,CAAC;YACvB,WAAW,CAAC,IAAI,CAAC;gBACf,KAAK;gBACL,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,OAAO,EAAE,GAAG,CAAC,QAAQ;gBACrB,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO;QACL,EAAE,EAAE,YAAY,IAAI,CAAC,SAAS,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAC1D,YAAY;QACZ,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,YAAoB;IAEpB,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;IACtD,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,CAAiB,CAAC;IACpF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,4DAA4D;AAE5D,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAkB,EAClB,MAAiB;IAEjB,MAAM,GAAG,GAKL,EAAE,CAAC;IAEP,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QACzE,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACxB,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;IACtB,CAAC;IAED,IAAI,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACjD,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IACnD,CAAC;SAAM,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACzD,IAAI,QAAQ,EAAE,CAAC;YACb,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACxB,GAAG,CAAC,UAAU,GAAG,eAAe,CAC9B,MAAM,EACN,QAAQ,EACR,MAAM,CAAC,mBAAmB,IAAI,IAAI,CACnC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/core/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAU3E,mEAAmE;AAEnE,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAmC,EACnC,SAAiB;IAEjB,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;SAC/B,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SACtC,GAAG,CACF,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CACT,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACxF,CAAC;IACJ,OAAO;QACL,KAAK,SAAS,2BAA2B;QACzC,uBAAuB;QACvB,GAAG,IAAI;KACR,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAOD,MAAM,UAAU,cAAc,CAAC,MAAkB,EAAE,OAAsB,EAAE;IACzE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CACR,YAAY,MAAM,CAAC,KAAK,YAAY,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CACpG,CAAC;IACF,KAAK,CAAC,IAAI,CACR,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACpL,CAAC;IACF,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,MAAM,CAAC,OAAO,oBAAoB,CAAC,CAAC;IACjE,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC1D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACrC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QACtE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,IAAI,CAAC,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC9C,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,CAAC,CAAC,CAAC,MAAM;oBAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3D,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,iEAAiE;AAEjE,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAkB,EAClB,SAAiB,EACjB,OAAsB,EAAE;IAExB,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/B,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,OAAO,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC;IAC/C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACpE,MAAM,SAAS,CAAC,MAAM,EAAE,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;IAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC;AAED,qEAAqE;AAErE,MAAM,UAAU,gBAAgB,CAAC,MAAkB;IACjD,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE;YACN,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;YAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC5B,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ;SACjC;QACD,OAAO,EAAE,MAAM,CAAC,WAAW,CACzB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YAC7C,CAAC;YACD,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE;SAC3D,CAAC,CACH;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAkB,EAClB,YAAoB;IAEpB,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACnC,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CACb,IAAI,EACJ,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,EACjD,OAAO,CACR,CAAC;IACF,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAkB,EAClB,QAAsB,EACtB,SAAS,GAAG,IAAI;IAEhB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;IACvE,MAAM,WAAW,GAAsC,EAAE,CAAC;IAE1D,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1D,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,GAAG;YAAE,SAAS,CAAC,2BAA2B;QAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;QACxC,IAAI,KAAK,GAAG,CAAC,SAAS,EAAE,CAAC;YACvB,WAAW,CAAC,IAAI,CAAC;gBACf,KAAK;gBACL,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,OAAO,EAAE,GAAG,CAAC,QAAQ;gBACrB,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO;QACL,EAAE,EAAE,YAAY,IAAI,CAAC,SAAS,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAC1D,YAAY;QACZ,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,YAAoB;IAEpB,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;IACtD,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,CAAiB,CAAC;IACpF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,4DAA4D;AAE5D,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAkB,EAClB,MAAiB;IASjB,MAAM,GAAG,GAOL,EAAE,CAAC;IAEP,mBAAmB;IACnB,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,GAAG,CAAC,WAAW,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;QAClE,GAAG,CAAC,OAAO,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE;YACvE,OAAO,EAAE,GAAG,CAAC,OAAO;SACrB,CAAC,CAAC;QACH,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACxB,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;IACtB,CAAC;IAED,IAAI,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACjD,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IACnD,CAAC;SAAM,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACzD,IAAI,QAAQ,EAAE,CAAC;YACb,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACxB,GAAG,CAAC,UAAU,GAAG,eAAe,CAC9B,MAAM,EACN,QAAQ,EACR,MAAM,CAAC,mBAAmB,IAAI,IAAI,CACnC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/src/core/runner.d.ts

@@ -1,18 +1,23 @@
 /**
  * packages/eval/src/core/runner.ts - Eval Runner
  *
- * mock 模式：每个用例构造独立 Runtime（createMockStreamFn 注入
- * streamFn + createMockTools 提供工具），run() 后经 getMessages()
- * 重建完整轨迹，再交给规则评分器。
+ * 两种模式共用同一条执行链路（Runtime → 轨迹 → 评分器）：
+ *   mock —— fixture replay：createMockStreamFn 注入脚本化 LLM 响应
+ *   live —— 真实 LLM：createLiveLlm 装配 provider streamFn（M3）
  *
  * 工程保障：
  *   - 用例间完全隔离（独立 runtime / 独立 fs 副本 / 独立 sessionKey）
  *   - 墙钟超时（Promise.race）
  *   - 步数上限（映射 RuntimeOptions.maxTurns）
- *   - 预算熔断（usage.total 超 case.metadata.maxTokens 判失败）
- *   - repeats 消随机性（live 模式用；mock 模式缺省 1）
+ *   - 预算熔断（case.metadata.maxTokens + RunConfig.maxTotalTokens 全局）
+ *   - repeats 消随机性：mock 1；live 默认 3 并取 pass@k（适配层不支持 seed）
  *   - 并发受控（简单队列实现）
  */
 import type { EvalCase, EvalReport, RunConfig } from './types';
+/**
+ * 模式不匹配 → 跳过（不计入通过率，避免出现"必然失败"的噪音）。
+ * 返回 undefined 表示该用例在当前模式下可跑。
+ */
+export declare function skipReason(c: EvalCase, mode: 'mock' | 'live'): string | undefined;
 /** 执行一次评测 */
 export declare function runEval(cases: EvalCase[], config: RunConfig): Promise<EvalReport>;