@aipack-ai/eval 1.1.6 → 1.1.8

package/dist/src/core/compare.js

@@ -0,0 +1,101 @@
+/**
+ * packages/eval/src/core/compare.ts - L4 模型 / Provider 对比（M5）
+ *
+ * 同一组用例在多个被测模型上各跑一遍 runEval（顺序执行，各算各的
+ * 预算与 repeats），汇总出通过率 / 平均分 / token 成本 / 延迟 /
+ * 分套件通过率矩阵。judge / embed 装配沿用 base config（judge 异源
+ * 校验对每个被测模型独立生效）。
+ */
+import { runEval } from './runner.js';
+/** 逐个模型跑同一组用例（顺序执行，避免跨模型并发互扰限流） */
+export async function compareModels(cases, base, models) {
+    if (models.length === 0)
+        throw new Error('compareModels 需要至少一个模型');
+    const startedAt = Date.now();
+    const entries = [];
+    for (const model of models) {
+        const report = await runEval(cases, { ...base, model });
+        entries.push({ model, report });
+    }
+    return {
+        startedAt: new Date(startedAt).toISOString(),
+        durationMs: Date.now() - startedAt,
+        mode: base.mode,
+        suites: base.suites ?? [],
+        entries,
+    };
+}
+// ─── Markdown 渲染 ────────────────────────────────────────────────
+function pct(x) {
+    return `${(x * 100).toFixed(1)}%`;
+}
+/** 单模型平均每 case 耗时（含 repeats） */
+function avgDurationMs(entry) {
+    const rs = entry.report.results;
+    if (rs.length === 0)
+        return 0;
+    return rs.reduce((a, r) => a + r.durationMs, 0) / rs.length;
+}
+function allSuites(entries) {
+    const set = new Set();
+    for (const e of entries)
+        for (const s of Object.keys(e.report.bySuite))
+            set.add(s);
+    return [...set].sort();
+}
+/**
+ * 对比表（报告按被测模型逐列对比；passRate 用 runEval 同一口径：
+ * 跳过用例不计入分母）。
+ */
+export function renderComparisonMarkdown(cmp) {
+    const lines = [];
+    lines.push('# L4 模型 / Provider 对比');
+    lines.push('');
+    lines.push(`- 模式: ${cmp.mode}`);
+    lines.push(`- 开始: ${cmp.startedAt}`);
+    lines.push(`- 总耗时: ${cmp.durationMs}ms`);
+    lines.push('');
+    // 总览
+    lines.push('## 总览');
+    lines.push('');
+    lines.push('| 模型 | 用例 | 通过率 | 平均分 | token 总耗 | 平均耗时/case |');
+    lines.push('|---|---|---|---|---|---|');
+    for (const e of cmp.entries) {
+        const t = e.report.totals;
+        lines.push(`| ${e.model} | ${t.cases} | ${pct(t.passRate)} | ${t.avgScore.toFixed(3)} | ${t.usageTotal} | ${Math.round(avgDurationMs(e))}ms |`);
+    }
+    lines.push('');
+    // 分套件矩阵
+    const suites = allSuites(cmp.entries);
+    if (suites.length > 0) {
+        lines.push('## 分套件通过率');
+        lines.push('');
+        lines.push(`| 套件 | ${cmp.entries.map((e) => e.model).join(' | ')} |`);
+        lines.push(`|---|${cmp.entries.map(() => '---').join('|')}|`);
+        for (const suite of suites) {
+            const cells = cmp.entries.map((e) => {
+                const s = e.report.bySuite[suite];
+                return s ? `${pct(s.passRate)} (${s.passed}/${s.total})` : '—';
+            });
+            lines.push(`| ${suite} | ${cells.join(' | ')} |`);
+        }
+        lines.push('');
+    }
+    // 失败明细（每模型最多列 5 条，防报告爆炸）
+    for (const e of cmp.entries) {
+        const failed = e.report.results.filter((r) => !r.passed);
+        if (failed.length === 0)
+            continue;
+        lines.push(`## 失败用例 — ${e.model}`);
+        lines.push('');
+        for (const r of failed.slice(0, 5)) {
+            const reason = r.error ?? r.scores.find((s) => !s.passed)?.reason ?? '';
+            lines.push(`- ${r.caseId}（score ${r.score.toFixed(3)}）: ${reason}`);
+        }
+        if (failed.length > 5)
+            lines.push(`- …共 ${failed.length} 条失败`);
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=compare.js.map

package/dist/src/core/compare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../../src/core/compare.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAiBnC,mCAAmC;AACnC,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAiB,EACjB,IAAe,EACf,MAAgB;IAEhB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;QAC5C,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;QACzB,OAAO;KACR,CAAC;AACJ,CAAC;AAED,mEAAmE;AAEnE,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,gCAAgC;AAChC,SAAS,aAAa,CAAC,KAA2B;IAChD,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;IAChC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;AAC9D,CAAC;AAED,SAAS,SAAS,CAAC,OAA+B;IAChD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,OAAO;QAAE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACnF,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,GAAqB;IAC5D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACpC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAChC,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;IACrC,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;IACzC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,KAAK;IACL,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAC7D,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACxC,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;QAC1B,KAAK,CAAC,IAAI,CACR,KAAK,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,UAAU,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CACpI,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,QAAQ;IACR,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtE,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBAClC,MAAM,CAAC,GAA6B,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC5D,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YACjE,CAAC,CAAC,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACpD,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,yBAAyB;IACzB,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACzD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAClC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;YACxE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC;QAC/D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/src/core/embedding.d.ts

@@ -0,0 +1,33 @@
+/**
+ * packages/eval/src/core/embedding.ts - OpenAI 兼容 embedding 客户端（M5）
+ *
+ * semantic 评分器的真实装配：POST {baseUrl}/embeddings（OpenAI 兼容协议，
+ * DeepSeek / OpenAI / Moonshot / 各兼容网关通用）。
+ *
+ * env 优先级低于显式参数（与 live.ts 的 spec 解析对称）：
+ *   AIPACK_EVAL_EMBEDDING_MODEL / AIPACK_EVAL_EMBEDDING_API_KEY /
+ *   AIPACK_EVAL_EMBEDDING_BASE_URL
+ */
+import type { EmbedFn } from './scorer';
+export interface EmbeddingConfig {
+    /** embedding 模型 id（如 text-embedding-3-small） */
+    model?: string;
+    apiKey?: string;
+    /** 缺省 https://api.openai.com/v1 */
+    baseUrl?: string;
+    /** 单请求超时 ms（缺省 30000） */
+    timeoutMs?: number;
+}
+export declare const EMBEDDING_ENV_KEYS: {
+    readonly model: "AIPACK_EVAL_EMBEDDING_MODEL";
+    readonly apiKey: "AIPACK_EVAL_EMBEDDING_API_KEY";
+    readonly baseUrl: "AIPACK_EVAL_EMBEDDING_BASE_URL";
+};
+/** 合并显式参数与环境变量，缺 model 抛可操作错误 */
+export declare function resolveEmbeddingConfig(opts?: EmbeddingConfig): EmbeddingConfig & {
+    model: string;
+    apiKey: string;
+    baseUrl: string;
+};
+/** 装配 OpenAI 兼容 embedding 函数 */
+export declare function createOpenAiCompatibleEmbedFn(opts?: EmbeddingConfig): EmbedFn;

package/dist/src/core/embedding.js

@@ -0,0 +1,68 @@
+/**
+ * packages/eval/src/core/embedding.ts - OpenAI 兼容 embedding 客户端（M5）
+ *
+ * semantic 评分器的真实装配：POST {baseUrl}/embeddings（OpenAI 兼容协议，
+ * DeepSeek / OpenAI / Moonshot / 各兼容网关通用）。
+ *
+ * env 优先级低于显式参数（与 live.ts 的 spec 解析对称）：
+ *   AIPACK_EVAL_EMBEDDING_MODEL / AIPACK_EVAL_EMBEDDING_API_KEY /
+ *   AIPACK_EVAL_EMBEDDING_BASE_URL
+ */
+export const EMBEDDING_ENV_KEYS = {
+    model: 'AIPACK_EVAL_EMBEDDING_MODEL',
+    apiKey: 'AIPACK_EVAL_EMBEDDING_API_KEY',
+    baseUrl: 'AIPACK_EVAL_EMBEDDING_BASE_URL',
+};
+const DEFAULT_BASE_URL = 'https://api.openai.com/v1';
+const DEFAULT_TIMEOUT_MS = 30_000;
+/** 合并显式参数与环境变量，缺 model 抛可操作错误 */
+export function resolveEmbeddingConfig(opts = {}) {
+    const env = process.env;
+    const model = opts.model ?? env[EMBEDDING_ENV_KEYS.model];
+    if (!model) {
+        throw new Error(`semantic 评分器需要 embedding 模型：--embed-model <id> 或 ${EMBEDDING_ENV_KEYS.model}`);
+    }
+    const apiKey = opts.apiKey ?? env[EMBEDDING_ENV_KEYS.apiKey] ?? env.OPENAI_API_KEY;
+    if (!apiKey) {
+        throw new Error(`embedding 缺少 API Key：--embed-api-key <key>、${EMBEDDING_ENV_KEYS.apiKey} 或 OPENAI_API_KEY`);
+    }
+    const baseUrl = (opts.baseUrl ?? env[EMBEDDING_ENV_KEYS.baseUrl] ?? DEFAULT_BASE_URL).replace(/\/+$/, '');
+    return {
+        model,
+        apiKey,
+        baseUrl,
+        timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+    };
+}
+/** 装配 OpenAI 兼容 embedding 函数 */
+export function createOpenAiCompatibleEmbedFn(opts = {}) {
+    const cfg = resolveEmbeddingConfig(opts);
+    return async (text) => {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), cfg.timeoutMs);
+        try {
+            const res = await fetch(`${cfg.baseUrl}/embeddings`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${cfg.apiKey}`,
+                },
+                body: JSON.stringify({ model: cfg.model, input: text }),
+                signal: controller.signal,
+            });
+            if (!res.ok) {
+                const body = await res.text().catch(() => '');
+                throw new Error(`embedding HTTP ${res.status}: ${body.slice(0, 200)}`);
+            }
+            const json = (await res.json());
+            const vec = json.data?.[0]?.embedding;
+            if (!Array.isArray(vec))
+                throw new Error('embedding 响应缺少 data[0].embedding');
+            return vec;
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    };
+}
+//# sourceMappingURL=embedding.js.map

package/dist/src/core/embedding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../../src/core/embedding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAcH,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,KAAK,EAAE,6BAA6B;IACpC,MAAM,EAAE,+BAA+B;IACvC,OAAO,EAAE,gCAAgC;CACjC,CAAC;AAEX,MAAM,gBAAgB,GAAG,2BAA2B,CAAC;AACrD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,iCAAiC;AACjC,MAAM,UAAU,sBAAsB,CAAC,OAAwB,EAAE;IAK/D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC1D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,oDAAoD,kBAAkB,CAAC,KAAK,EAAE,CAC/E,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC;IACnF,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,8CAA8C,kBAAkB,CAAC,MAAM,mBAAmB,CAC3F,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,kBAAkB,CAAC,OAAO,CAAC,IAAI,gBAAgB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC1G,OAAO;QACL,KAAK;QACL,MAAM;QACN,OAAO;QACP,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,kBAAkB;KAChD,CAAC;AACJ,CAAC;AAED,gCAAgC;AAChC,MAAM,UAAU,6BAA6B,CAAC,OAAwB,EAAE;IACtE,MAAM,GAAG,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO,KAAK,EAAE,IAAY,EAAqB,EAAE;QAC/C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;QAClE,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,GAAG,CAAC,OAAO,aAAa,EAAE;gBACnD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,GAAG,CAAC,MAAM,EAAE;iBACtC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;gBACvD,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACzE,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA+C,CAAC;YAC9E,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC;YACtC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YAC7E,OAAO,GAAG,CAAC;QACb,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/core/history.d.ts

@@ -0,0 +1,39 @@
+/**
+ * packages/eval/src/core/history.ts - 历史趋势（日环比可视化）
+ *
+ * 每次运行向 JSONL 追加一行（append-only，天然按时间序），
+ * 读回后渲染 Markdown：趋势 sparkline + 逐次通过率 + 环比 Δ。
+ *
+ * 为什么用 JSONL 而不是单文件 JSON：cron 场景下并发/追加写更安全，
+ * 且 diff 友好（一行一次运行）。
+ */
+import type { EvalReport, SuiteSummary } from './types';
+/** 每次运行落一行（JSONL） */
+export interface HistoryEntry {
+    runId: string;
+    /** ISO 时间 */
+    at: string;
+    mode: 'mock' | 'live';
+    model?: string;
+    cases: number;
+    passed: number;
+    skipped: number;
+    passRate: number;
+    avgScore: number;
+    durationMs: number;
+    usageTotal: number;
+    bySuite: Record<string, Pick<SuiteSummary, 'total' | 'passed' | 'passRate'>>;
+}
+/** 报告 → 历史条目 */
+export declare function reportToHistoryEntry(report: EvalReport): HistoryEntry;
+/** 追加一次运行记录（JSONL，一行一条） */
+export declare function appendHistory(report: EvalReport, historyPath: string): Promise<string>;
+/** 读取历史（最新在最后）；文件不存在返回空数组 */
+export declare function readHistory(historyPath: string, limit?: number): Promise<HistoryEntry[]>;
+/** 0~1 序列 → sparkline 字符串 */
+export declare function sparkline(values: number[], width?: number): string;
+/**
+ * 渲染通过率趋势（Markdown）。
+ * 环比定义：与上一次运行对比（cron 每日跑 → 即日环比）。
+ */
+export declare function renderHistoryTrend(entries: HistoryEntry[], limit?: number): string;

package/dist/src/core/history.js

@@ -0,0 +1,134 @@
+/**
+ * packages/eval/src/core/history.ts - 历史趋势（日环比可视化）
+ *
+ * 每次运行向 JSONL 追加一行（append-only，天然按时间序），
+ * 读回后渲染 Markdown：趋势 sparkline + 逐次通过率 + 环比 Δ。
+ *
+ * 为什么用 JSONL 而不是单文件 JSON：cron 场景下并发/追加写更安全，
+ * 且 diff 友好（一行一次运行）。
+ */
+import { appendFile, mkdir, readFile } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+/** 报告 → 历史条目 */
+export function reportToHistoryEntry(report) {
+    return {
+        runId: report.runId,
+        at: report.startedAt,
+        mode: report.mode,
+        model: report.model,
+        cases: report.totals.cases,
+        passed: report.totals.passed,
+        skipped: report.totals.skipped,
+        passRate: report.totals.passRate,
+        avgScore: report.totals.avgScore,
+        durationMs: report.durationMs,
+        usageTotal: report.totals.usageTotal,
+        bySuite: Object.fromEntries(Object.entries(report.bySuite).map(([k, s]) => [
+            k,
+            { total: s.total, passed: s.passed, passRate: s.passRate },
+        ])),
+    };
+}
+/** 追加一次运行记录（JSONL，一行一条） */
+export async function appendHistory(report, historyPath) {
+    const path = resolve(historyPath);
+    await mkdir(dirname(path), { recursive: true });
+    await appendFile(path, `${JSON.stringify(reportToHistoryEntry(report))}\n`, 'utf-8');
+    return path;
+}
+/** 读取历史（最新在最后）；文件不存在返回空数组 */
+export async function readHistory(historyPath, limit = 30) {
+    const path = resolve(historyPath);
+    let raw;
+    try {
+        raw = await readFile(path, 'utf-8');
+    }
+    catch {
+        return [];
+    }
+    const entries = [];
+    for (const line of raw.split('\n')) {
+        const s = line.trim();
+        if (!s)
+            continue;
+        try {
+            entries.push(JSON.parse(s));
+        }
+        catch {
+            // 坏行跳过（不因一行损坏丢掉整份历史）
+        }
+    }
+    return limit > 0 ? entries.slice(-limit) : entries;
+}
+// ─── 可视化 ───────────────────────────────────────────────────────
+const SPARK_CHARS = '▁▂▃▄▅▆▇█';
+/** 0~1 序列 → sparkline 字符串 */
+export function sparkline(values, width = 20) {
+    if (values.length === 0)
+        return '';
+    const seq = values.slice(-width);
+    return seq
+        .map((v) => {
+        const clamped = Math.min(1, Math.max(0, Number.isFinite(v) ? v : 0));
+        const idx = Math.round(clamped * (SPARK_CHARS.length - 1));
+        return SPARK_CHARS[idx];
+    })
+        .join('');
+}
+function pct(x) {
+    return `${(x * 100).toFixed(1)}%`;
+}
+function deltaCell(cur, prev) {
+    if (prev === undefined)
+        return '—';
+    const d = (cur - prev) * 100;
+    if (Math.abs(d) < 0.05)
+        return '±0.0%';
+    const sign = d > 0 ? '+' : '';
+    const icon = d < 0 ? '🔻' : '🔺';
+    return `${icon}${sign}${d.toFixed(1)}%`;
+}
+function shortAt(iso) {
+    // 2026-09-29T03:12:44.123Z → 09-29 03:12
+    return `${iso.slice(5, 10)} ${iso.slice(11, 16)}`;
+}
+/**
+ * 渲染通过率趋势（Markdown）。
+ * 环比定义：与上一次运行对比（cron 每日跑 → 即日环比）。
+ */
+export function renderHistoryTrend(entries, limit = 15) {
+    if (entries.length === 0)
+        return '（暂无历史记录）';
+    const rows = entries.slice(-limit);
+    const lines = [];
+    const rates = rows.map((e) => e.passRate);
+    const latest = rows[rows.length - 1];
+    const prev = rows.length > 1 ? rows[rows.length - 2] : undefined;
+    lines.push(`\`${sparkline(rates)}\`  最近 ${rows.length} 次：${pct(latest.passRate)}（环比 ${deltaCell(latest.passRate, prev?.passRate)}）`);
+    lines.push('');
+    lines.push(`| 时间(UTC) | 模式 | 模型 | 通过 | 通过率 | 环比 Δ | 平均分 | 耗时 |`);
+    lines.push(`|---|---|---|---|---|---|---|---|`);
+    rows.forEach((e, i) => {
+        lines.push(`| ${shortAt(e.at)} | ${e.mode} | ${e.model ?? '—'} | ${e.passed}/${e.cases} | ${pct(e.passRate)} | ${deltaCell(e.passRate, i > 0 ? rows[i - 1].passRate : undefined)} | ${e.avgScore.toFixed(2)} | ${(e.durationMs / 1000).toFixed(1)}s |`);
+    });
+    // 分套件环比：定位是哪一组掉了
+    const suiteDeltas = [];
+    if (prev) {
+        for (const [suite, cur] of Object.entries(latest.bySuite)) {
+            const before = prev.bySuite[suite];
+            if (!before)
+                continue;
+            const d = (cur.passRate - before.passRate) * 100;
+            if (Math.abs(d) >= 0.05) {
+                suiteDeltas.push(`  - ${suite}: ${pct(before.passRate)} → ${pct(cur.passRate)}（${d > 0 ? '+' : ''}${d.toFixed(1)}%）`);
+            }
+        }
+    }
+    if (suiteDeltas.length > 0) {
+        lines.push('');
+        lines.push(`分套件环比:`);
+        lines.push(...suiteDeltas);
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=history.js.map

package/dist/src/core/history.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"history.js","sourceRoot":"","sources":["../../../src/core/history.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoB7C,gBAAgB;AAChB,MAAM,UAAU,oBAAoB,CAAC,MAAkB;IACrD,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,EAAE,EAAE,MAAM,CAAC,SAAS;QACpB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;QAC5B,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;QAC9B,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ;QAChC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ;QAChC,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU;QACpC,OAAO,EAAE,MAAM,CAAC,WAAW,CACzB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YAC7C,CAAC;YACD,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE;SAC3D,CAAC,CACH;KACF,CAAC;AACJ,CAAC;AAED,2BAA2B;AAC3B,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAkB,EAClB,WAAmB;IAEnB,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAClC,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,UAAU,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,6BAA6B;AAC7B,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,WAAmB,EACnB,KAAK,GAAG,EAAE;IAEV,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAClC,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,IAAI,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAiB,CAAC,CAAC;QAC9C,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AACrD,CAAC;AAED,kEAAkE;AAElE,MAAM,WAAW,GAAG,UAAU,CAAC;AAE/B,6BAA6B;AAC7B,MAAM,UAAU,SAAS,CAAC,MAAgB,EAAE,KAAK,GAAG,EAAE;IACpD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;IACjC,OAAO,GAAG;SACP,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3D,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AACd,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,SAAS,CAAC,GAAW,EAAE,IAAwB;IACtD,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,CAAC;IACnC,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC;IAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI;QAAE,OAAO,OAAO,CAAC;IACvC,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACjC,OAAO,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAC1C,CAAC;AAED,SAAS,OAAO,CAAC,GAAW;IAC1B,yCAAyC;IACzC,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;AACpD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAuB,EAAE,KAAK,GAAG,EAAE;IACpE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,UAAU,CAAC;IAE5C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEjE,KAAK,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IACrI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;IACjE,KAAK,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IAChD,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACpB,KAAK,CAAC,IAAI,CACR,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAC5O,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,iBAAiB;IACjB,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,IAAI,EAAE,CAAC;QACT,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACnC,IAAI,CAAC,MAAM;gBAAE,SAAS;YACtB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC;YACjD,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBACxB,WAAW,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACxH,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/src/core/import.d.ts

@@ -0,0 +1,30 @@
+/**
+ * packages/eval/src/core/import.ts - trace 回流用例入库（M4）
+ *
+ * 把 observability-server `POST /api/v1/export-eval` 导出的 EvalCase JSON
+ * 写入用例目录（入库 = 纳入日常 eval 回归）。入库前走 validateEvalCase
+ * （fail fast），支持单用例 / 数组 / { cases: [...] } 三种文件形态，
+ * 以及 suite / origin / id 前缀的人工修正（bad case 固化纪律）。
+ */
+import type { CaseOrigin } from './types';
+export interface ImportOptions {
+    /** 导出文件路径（单 case / 数组 / { cases: [...] }） */
+    file: string;
+    /** 用例目录（缺省 defaultCasesDir） */
+    outDir?: string;
+    /** 覆盖 suite */
+    suite?: string;
+    /** 覆盖 origin（'trace' | 'bugfix'） */
+    origin?: CaseOrigin;
+    /** case id 加前缀（如 'bugfix'），避免与既有 id 冲突 */
+    prefix?: string;
+    /** 只校验不写盘 */
+    dryRun?: boolean;
+}
+export interface ImportResult {
+    /** 写入的文件路径（相对 outDir） */
+    written: string[];
+    /** 校验/写入错误（带 case 上下文） */
+    errors: string[];
+}
+export declare function importCases(opts: ImportOptions): Promise<ImportResult>;

package/dist/src/core/import.js

@@ -0,0 +1,111 @@
+/**
+ * packages/eval/src/core/import.ts - trace 回流用例入库（M4）
+ *
+ * 把 observability-server `POST /api/v1/export-eval` 导出的 EvalCase JSON
+ * 写入用例目录（入库 = 纳入日常 eval 回归）。入库前走 validateEvalCase
+ * （fail fast），支持单用例 / 数组 / { cases: [...] } 三种文件形态，
+ * 以及 suite / origin / id 前缀的人工修正（bad case 固化纪律）。
+ */
+import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { validateEvalCase } from './validate.js';
+import { defaultCasesDir } from './loader.js';
+function normalizeId(id) {
+    // 文件名 = id 的最后一段（'trace-export/t-1' → 't-1.json'），非法字符收敛
+    const last = id.split('/').pop() ?? id;
+    return last.replace(/[^a-zA-Z0-9_-]/g, '_');
+}
+export async function importCases(opts) {
+    const outDir = resolve(opts.outDir ?? defaultCasesDir());
+    const raw = await readFile(opts.file, 'utf-8');
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (e) {
+        return { written: [], errors: [`JSON 解析失败: ${e.message}`] };
+    }
+    let list;
+    if (Array.isArray(parsed))
+        list = parsed;
+    else if (parsed && typeof parsed === 'object' && Array.isArray(parsed.cases)) {
+        list = parsed.cases;
+    }
+    else
+        list = [parsed];
+    if (list.length === 0)
+        return { written: [], errors: ['文件中没有用例'] };
+    const written = [];
+    const errors = [];
+    /** 已占用文件名：suite → Set<fileName> */
+    const taken = new Map();
+    const takeFor = (suiteName) => {
+        let set = taken.get(suiteName);
+        if (!set) {
+            set = new Set();
+            taken.set(suiteName, set);
+        }
+        return set;
+    };
+    if (!opts.dryRun) {
+        // 同目录已存在的文件也占位，避免覆盖人工已修正的用例
+        try {
+            const suiteDirGuess = opts.suite ?? guessSuite(list);
+            if (suiteDirGuess) {
+                const existing = await readdir(resolve(outDir, suiteDirGuess)).catch(() => []);
+                for (const f of existing)
+                    takeFor(suiteDirGuess).add(f);
+            }
+        }
+        catch {
+            // outDir 不存在时忽略
+        }
+    }
+    for (const [i, item] of list.entries()) {
+        const c = item;
+        const label = `#${i} ${c.id ?? '<missing id>'}`;
+        if (!c || typeof c !== 'object') {
+            errors.push(`${label}: 不是对象`);
+            continue;
+        }
+        if (opts.suite)
+            c.suite = opts.suite;
+        if (opts.origin)
+            c.origin = opts.origin;
+        if (opts.prefix)
+            c.id = `${opts.prefix}/${String(c.id ?? `case-${i}`)}`;
+        const validation = validateEvalCase(c);
+        if (validation.length > 0) {
+            errors.push(`${label}: ${validation.join('; ')}`);
+            continue;
+        }
+        const suite = String(c.suite);
+        const takenSet = takeFor(suite);
+        let fileName = `${normalizeId(String(c.id))}.json`;
+        let n = 1;
+        while (takenSet.has(fileName)) {
+            fileName = `${normalizeId(String(c.id))}-${n}.json`;
+            n += 1;
+        }
+        takenSet.add(fileName);
+        if (opts.dryRun) {
+            written.push(`${suite}/${fileName}`);
+            continue;
+        }
+        const target = resolve(outDir, suite, fileName);
+        try {
+            await mkdir(resolve(outDir, suite), { recursive: true });
+            await writeFile(target, `${JSON.stringify(c, null, 2)}\n`, 'utf-8');
+            written.push(`${suite}/${fileName}`);
+        }
+        catch (e) {
+            errors.push(`${label}: 写入失败 - ${e.message}`);
+        }
+    }
+    return { written, errors };
+}
+function guessSuite(list) {
+    const first = list[0];
+    return typeof first?.suite === 'string' && first.suite ? first.suite : undefined;
+}
+//# sourceMappingURL=import.js.map

package/dist/src/core/import.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"import.js","sourceRoot":"","sources":["../../../src/core/import.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAwB3C,SAAS,WAAW,CAAC,EAAU;IAC7B,yDAAyD;IACzD,MAAM,IAAI,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;IACvC,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAmB;IACnD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC,CAAC;IACzD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,cAAe,CAAW,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;IACzE,CAAC;IAED,IAAI,IAAe,CAAC;IACpB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,IAAI,GAAG,MAAM,CAAC;SACpC,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAE,MAAgC,CAAC,KAAK,CAAC,EAAE,CAAC;QACxG,IAAI,GAAI,MAA+B,CAAC,KAAK,CAAC;IAChD,CAAC;;QAAM,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAEvB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;IAEnE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,mCAAmC;IACnC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC7C,MAAM,OAAO,GAAG,CAAC,SAAiB,EAAe,EAAE;QACjD,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;YACxB,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IACF,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,4BAA4B;QAC5B,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAc,CAAC,CAAC;gBAC3F,KAAK,MAAM,CAAC,IAAI,QAAQ;oBAAE,OAAO,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,gBAAgB;QAClB,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,IAA+B,CAAC;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,IAAK,CAAC,CAAC,EAAa,IAAI,cAAc,EAAE,CAAC;QAC5D,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,KAAK;YAAE,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM;YAAE,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACxC,IAAI,IAAI,CAAC,MAAM;YAAE,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;QAExE,MAAM,UAAU,GAAG,gBAAgB,CAAC,CAAU,CAAC,CAAC;QAChD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClD,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,QAAQ,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC;QACnD,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OAAO,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,QAAQ,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;YACpD,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,QAAQ,EAAE,CAAC,CAAC;YACrC,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACpE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,QAAQ,EAAE,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,YAAa,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,UAAU,CAAC,IAAe;IACjC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAoC,CAAC;IACzD,OAAO,OAAO,KAAK,EAAE,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACnF,CAAC"}

package/dist/src/core/judge-llm.d.ts

@@ -0,0 +1,13 @@
+/**
+ * packages/eval/src/core/judge-llm.ts - judge 模型装配（M5）
+ *
+ * 把 StreamFn + 框架 Model 装配成 llm-judge 需要的 complete(prompt)。
+ * 与 runner 的 live 装配对称：judge 模型也走 createLiveLlm / 注入 StreamFn，
+ * 单次非流式调用（内部仍消费事件流，聚合 text delta）。
+ */
+import type { Model, StreamFn } from '@aipack-ai/agent';
+import type { JudgeDeps } from './scorer';
+/** 单条消息 → complete(prompt) 的适配（temperature 0，judge 要确定性） */
+export declare function createCompleteFromStreamFn(streamFn: StreamFn, model: Model | undefined): (prompt: string) => Promise<string>;
+/** JudgeDeps 快捷装配 */
+export declare function createJudgeDeps(streamFn: StreamFn, model: Model | undefined, modelLabel?: string): JudgeDeps;

package/dist/src/core/judge-llm.js

@@ -0,0 +1,47 @@
+/**
+ * packages/eval/src/core/judge-llm.ts - judge 模型装配（M5）
+ *
+ * 把 StreamFn + 框架 Model 装配成 llm-judge 需要的 complete(prompt)。
+ * 与 runner 的 live 装配对称：judge 模型也走 createLiveLlm / 注入 StreamFn，
+ * 单次非流式调用（内部仍消费事件流，聚合 text delta）。
+ */
+/** 单条消息 → complete(prompt) 的适配（temperature 0，judge 要确定性） */
+export function createCompleteFromStreamFn(streamFn, model) {
+    return async (prompt) => {
+        const context = {
+            systemPrompt: '',
+            messages: [{ role: 'user', content: prompt, timestamp: Date.now() }],
+        };
+        let text = '';
+        const result = streamFn(model, context);
+        for await (const event of result) {
+            if (event.type === 'text_delta')
+                text += event.delta;
+            else if (event.type === 'done') {
+                // 兜底：done 消息里的 text（个别 provider 不发 text_delta）
+                if (!text) {
+                    const content = event.message.content;
+                    if (typeof content === 'string')
+                        text = content;
+                    else
+                        for (const block of content)
+                            if (block.type === 'text')
+                                text += block.text;
+                }
+                break;
+            }
+            else if (event.type === 'error') {
+                throw new Error('judge 流式调用错误');
+            }
+        }
+        return text;
+    };
+}
+/** JudgeDeps 快捷装配 */
+export function createJudgeDeps(streamFn, model, modelLabel) {
+    return {
+        complete: createCompleteFromStreamFn(streamFn, model),
+        ...(modelLabel ? { modelLabel } : {}),
+    };
+}
+//# sourceMappingURL=judge-llm.js.map

package/dist/src/core/judge-llm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-llm.js","sourceRoot":"","sources":["../../../src/core/judge-llm.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,4DAA4D;AAC5D,MAAM,UAAU,0BAA0B,CACxC,QAAkB,EAClB,KAAwB;IAExB,OAAO,KAAK,EAAE,MAAc,EAAmB,EAAE;QAC/C,MAAM,OAAO,GAAG;YACd,YAAY,EAAE,EAAE;YAChB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SAC9E,CAAC;QACF,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAc,EAAE,OAAO,CAAC,CAAC;QACjD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY;gBAAE,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC;iBAChD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC/B,+CAA+C;gBAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;oBACtC,IAAI,OAAO,OAAO,KAAK,QAAQ;wBAAE,IAAI,GAAG,OAAO,CAAC;;wBAC3C,KAAK,MAAM,KAAK,IAAI,OAAO;4BAAE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;gCAAE,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC;gBAClF,CAAC;gBACD,MAAM;YACR,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;AACJ,CAAC;AAED,qBAAqB;AACrB,MAAM,UAAU,eAAe,CAC7B,QAAkB,EAClB,KAAwB,EACxB,UAAmB;IAEnB,OAAO;QACL,QAAQ,EAAE,0BAA0B,CAAC,QAAQ,EAAE,KAAK,CAAC;QACrD,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtC,CAAC;AACJ,CAAC"}