@aipack-ai/eval 1.1.6

package/dist/src/core/report.d.ts

@@ -0,0 +1,26 @@
+/**
+ * packages/eval/src/core/report.ts - 报告渲染与 baseline 门禁
+ *
+ * - JSON + Markdown 双格式输出到报告目录（缺省 ./eval-results）
+ * - baseline：对比整体与分套件通过率，回归超阈值判失败
+ */
+import type { BaselineComparison, BaselineFile, EvalReport, RunConfig } from './types';
+export declare function renderMarkdown(report: EvalReport): string;
+export declare function writeReport(report: EvalReport, reportDir: string): Promise<{
+    jsonPath: string;
+    mdPath: string;
+}>;
+export declare function reportToBaseline(report: EvalReport): BaselineFile;
+export declare function writeBaseline(report: EvalReport, baselinePath: string): Promise<string>;
+/**
+ * 对比当前报告与 baseline。
+ * ok = 整体通过率不低于 baseline - threshold，且无套件回归超阈值。
+ */
+export declare function compareBaseline(report: EvalReport, baseline: BaselineFile, threshold?: number): BaselineComparison;
+export declare function readBaseline(baselinePath: string): Promise<BaselineFile | undefined>;
+export declare function finalizeReport(report: EvalReport, config: RunConfig): Promise<{
+    jsonPath?: string;
+    mdPath?: string;
+    baseline?: BaselineFile;
+    comparison?: BaselineComparison;
+}>;

package/dist/src/core/report.js

@@ -0,0 +1,149 @@
+/**
+ * packages/eval/src/core/report.ts - 报告渲染与 baseline 门禁
+ *
+ * - JSON + Markdown 双格式输出到报告目录（缺省 ./eval-results）
+ * - baseline：对比整体与分套件通过率，回归超阈值判失败
+ */
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+// ─── Markdown 渲染 ────────────────────────────────────────────────
+function pct(x) {
+    return `${(x * 100).toFixed(1)}%`;
+}
+function renderSummaryTable(byKey, keyHeader) {
+    const rows = Object.entries(byKey)
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([k, s]) => `| ${k} | ${s.total} | ${s.passed} | ${pct(s.passRate)} | ${s.avgScore.toFixed(2)} |`);
+    return [
+        `| ${keyHeader} | 用例数 | 通过 | 通过率 | 平均分 |`,
+        `|---|---|---|---|---|`,
+        ...rows,
+    ].join('\n');
+}
+export function renderMarkdown(report) {
+    const lines = [];
+    lines.push(`# aipack Eval 报告`);
+    lines.push('');
+    lines.push(`- Run: \`${report.runId}\`（mode: ${report.mode}）`);
+    lines.push(`- 总览: **${report.totals.passed}/${report.totals.cases} 通过（${pct(report.totals.passRate)}）**，平均分 ${report.totals.avgScore.toFixed(2)}，耗时 ${(report.durationMs / 1000).toFixed(2)}s`);
+    lines.push('');
+    lines.push(`## 按套件`);
+    lines.push('');
+    lines.push(renderSummaryTable(report.bySuite, '套件'));
+    lines.push('');
+    lines.push(`## 按来源（分布漂移诊断）`);
+    lines.push('');
+    lines.push(renderSummaryTable(report.byOrigin, 'origin'));
+    lines.push('');
+    const failures = report.results.filter((r) => !r.passed);
+    if (failures.length === 0) {
+        lines.push(`## 失败用例`);
+        lines.push('');
+        lines.push(`无 ✅`);
+    }
+    else {
+        lines.push(`## 失败用例（${failures.length}）`);
+        lines.push('');
+        for (const f of failures) {
+            lines.push(`### ${f.caseId}`);
+            lines.push('');
+            if (f.error)
+                lines.push(`- 运行错误: ${f.error}`);
+            for (const s of f.scores) {
+                if (!s.passed)
+                    lines.push(`- [${s.scorer}] ${s.reason}`);
+            }
+            lines.push('');
+        }
+    }
+    return lines.join('\n');
+}
+// ─── 报告写盘 ─────────────────────────────────────────────────────
+export async function writeReport(report, reportDir) {
+    const dir = resolve(reportDir);
+    await mkdir(dir, { recursive: true });
+    const jsonPath = join(dir, `${report.runId}.json`);
+    const mdPath = join(dir, `${report.runId}.md`);
+    await writeFile(jsonPath, JSON.stringify(report, null, 2), 'utf-8');
+    await writeFile(mdPath, renderMarkdown(report), 'utf-8');
+    return { jsonPath, mdPath };
+}
+// ─── Baseline ─────────────────────────────────────────────────────
+export function reportToBaseline(report) {
+    return {
+        runId: report.runId,
+        updatedAt: new Date().toISOString(),
+        totals: {
+            cases: report.totals.cases,
+            passed: report.totals.passed,
+            passRate: report.totals.passRate,
+        },
+        bySuite: Object.fromEntries(Object.entries(report.bySuite).map(([k, s]) => [
+            k,
+            { total: s.total, passed: s.passed, passRate: s.passRate },
+        ])),
+    };
+}
+export async function writeBaseline(report, baselinePath) {
+    const path = resolve(baselinePath);
+    await mkdir(dirname(path), { recursive: true });
+    await writeFile(path, JSON.stringify(reportToBaseline(report), null, 2), 'utf-8');
+    return path;
+}
+/**
+ * 对比当前报告与 baseline。
+ * ok = 整体通过率不低于 baseline - threshold，且无套件回归超阈值。
+ */
+export function compareBaseline(report, baseline, threshold = 0.02) {
+    const overallDelta = report.totals.passRate - baseline.totals.passRate;
+    const regressions = [];
+    for (const [suite, b] of Object.entries(baseline.bySuite)) {
+        const cur = report.bySuite[suite];
+        if (!cur)
+            continue; // baseline 中存在但本次未跑的套件不判回归
+        const delta = cur.passRate - b.passRate;
+        if (delta < -threshold) {
+            regressions.push({
+                suite,
+                baseline: b.passRate,
+                current: cur.passRate,
+                delta,
+            });
+        }
+    }
+    return {
+        ok: overallDelta >= -threshold && regressions.length === 0,
+        overallDelta,
+        regressions,
+    };
+}
+export async function readBaseline(baselinePath) {
+    const { readFile } = await import('node:fs/promises');
+    try {
+        return JSON.parse(await readFile(resolve(baselinePath), 'utf-8'));
+    }
+    catch {
+        return undefined;
+    }
+}
+// ─── 便捷：跑完即出报告 ───────────────────────────────────────────
+export async function finalizeReport(report, config) {
+    const out = {};
+    if (config.reportDir) {
+        const { jsonPath, mdPath } = await writeReport(report, config.reportDir);
+        out.jsonPath = jsonPath;
+        out.mdPath = mdPath;
+    }
+    if (config.updateBaseline && config.baselinePath) {
+        await writeBaseline(report, config.baselinePath);
+    }
+    else if (config.baselinePath) {
+        const baseline = await readBaseline(config.baselinePath);
+        if (baseline) {
+            out.baseline = baseline;
+            out.comparison = compareBaseline(report, baseline, config.regressionThreshold ?? 0.02);
+        }
+    }
+    return out;
+}
+//# sourceMappingURL=report.js.map

package/dist/src/core/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/core/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASnD,mEAAmE;AAEnE,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAmC,EACnC,SAAiB;IAEjB,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;SAC/B,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SACtC,GAAG,CACF,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CACT,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACxF,CAAC;IACJ,OAAO;QACL,KAAK,SAAS,2BAA2B;QACzC,uBAAuB;QACvB,GAAG,IAAI;KACR,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAkB;IAC/C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,KAAK,YAAY,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;IAC/D,KAAK,CAAC,IAAI,CACR,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACpL,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC1D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,IAAI,CAAC,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC9C,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,CAAC,CAAC,CAAC,MAAM;oBAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3D,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,iEAAiE;AAEjE,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAkB,EAClB,SAAiB;IAEjB,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/B,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,OAAO,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC;IAC/C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACpE,MAAM,SAAS,CAAC,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;IACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC;AAED,qEAAqE;AAErE,MAAM,UAAU,gBAAgB,CAAC,MAAkB;IACjD,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE;YACN,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;YAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC5B,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ;SACjC;QACD,OAAO,EAAE,MAAM,CAAC,WAAW,CACzB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YAC7C,CAAC;YACD,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE;SAC3D,CAAC,CACH;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAkB,EAClB,YAAoB;IAEpB,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACnC,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CACb,IAAI,EACJ,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,EACjD,OAAO,CACR,CAAC;IACF,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAkB,EAClB,QAAsB,EACtB,SAAS,GAAG,IAAI;IAEhB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;IACvE,MAAM,WAAW,GAAsC,EAAE,CAAC;IAE1D,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1D,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,GAAG;YAAE,SAAS,CAAC,2BAA2B;QAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;QACxC,IAAI,KAAK,GAAG,CAAC,SAAS,EAAE,CAAC;YACvB,WAAW,CAAC,IAAI,CAAC;gBACf,KAAK;gBACL,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,OAAO,EAAE,GAAG,CAAC,QAAQ;gBACrB,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO;QACL,EAAE,EAAE,YAAY,IAAI,CAAC,SAAS,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAC1D,YAAY;QACZ,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,YAAoB;IAEpB,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;IACtD,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,CAAiB,CAAC;IACpF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,4DAA4D;AAE5D,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAkB,EAClB,MAAiB;IAEjB,MAAM,GAAG,GAKL,EAAE,CAAC;IAEP,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QACzE,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACxB,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;IACtB,CAAC;IAED,IAAI,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACjD,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IACnD,CAAC;SAAM,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACzD,IAAI,QAAQ,EAAE,CAAC;YACb,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACxB,GAAG,CAAC,UAAU,GAAG,eAAe,CAC9B,MAAM,EACN,QAAQ,EACR,MAAM,CAAC,mBAAmB,IAAI,IAAI,CACnC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/src/core/runner.d.ts

@@ -0,0 +1,18 @@
+/**
+ * packages/eval/src/core/runner.ts - Eval Runner
+ *
+ * mock 模式：每个用例构造独立 Runtime（createMockStreamFn 注入
+ * streamFn + createMockTools 提供工具），run() 后经 getMessages()
+ * 重建完整轨迹，再交给规则评分器。
+ *
+ * 工程保障：
+ *   - 用例间完全隔离（独立 runtime / 独立 fs 副本 / 独立 sessionKey）
+ *   - 墙钟超时（Promise.race）
+ *   - 步数上限（映射 RuntimeOptions.maxTurns）
+ *   - 预算熔断（usage.total 超 case.metadata.maxTokens 判失败）
+ *   - repeats 消随机性（live 模式用；mock 模式缺省 1）
+ *   - 并发受控（简单队列实现）
+ */
+import type { EvalCase, EvalReport, RunConfig } from './types';
+/** 执行一次评测 */
+export declare function runEval(cases: EvalCase[], config: RunConfig): Promise<EvalReport>;

package/dist/src/core/runner.js

@@ -0,0 +1,213 @@
+/**
+ * packages/eval/src/core/runner.ts - Eval Runner
+ *
+ * mock 模式：每个用例构造独立 Runtime（createMockStreamFn 注入
+ * streamFn + createMockTools 提供工具），run() 后经 getMessages()
+ * 重建完整轨迹，再交给规则评分器。
+ *
+ * 工程保障：
+ *   - 用例间完全隔离（独立 runtime / 独立 fs 副本 / 独立 sessionKey）
+ *   - 墙钟超时（Promise.race）
+ *   - 步数上限（映射 RuntimeOptions.maxTurns）
+ *   - 预算熔断（usage.total 超 case.metadata.maxTokens 判失败）
+ *   - repeats 消随机性（live 模式用；mock 模式缺省 1）
+ *   - 并发受控（简单队列实现）
+ */
+import { createRuntime, createRequest, } from '@aipack-ai/agent';
+import { createMockStreamFn } from './mock-stream.js';
+import { createMockTools } from './mock-tools.js';
+import { countTurns, extractTrajectory } from './trajectory.js';
+import { resolveScorers, scoreTrace } from './scorer/index.js';
+const DEFAULT_TIMEOUT_MS = 30_000;
+const DEFAULT_MAX_STEPS = 50;
+const DEFAULT_CONCURRENCY = 8;
+/** sanitize sessionKey（用例 id 可能含 '/'） */
+function sessionKeyFor(caseId, repeat) {
+    return `eval:${caseId.replace(/[^a-zA-Z0-9_-]/g, '_')}#r${repeat}`;
+}
+/** 单用例单次运行：返回轨迹（失败时返回 error 字符串） */
+async function runCaseOnce(c, config, repeat) {
+    if (!c.input.mock) {
+        return { error: 'mock 模式要求 input.mock 脚本（live 模式 M3 接入）' };
+    }
+    const timeoutMs = c.metadata?.timeoutMs ?? config.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const maxSteps = c.metadata?.maxSteps ?? config.maxSteps ?? DEFAULT_MAX_STEPS;
+    const tools = createMockTools(c.input.fs ?? {}, c.input.tools);
+    const runtime = createRuntime({
+        systemPrompt: c.input.systemPrompt ?? '',
+        streamFn: createMockStreamFn(c.input.mock),
+        tools,
+        maxTurns: maxSteps,
+        traceIdGenerator: () => sessionKeyFor(c.id, repeat),
+    });
+    const sessionKey = sessionKeyFor(c.id, repeat);
+    const startedAt = Date.now();
+    let run;
+    try {
+        run = runtime
+            .run(createRequest(c.input.message, { sessionKey }))
+            .then((result) => {
+            const messages = runtime.getMessages(sessionKey);
+            const trajectory = extractTrajectory(messages);
+            const turns = countTurns(messages);
+            const durationMs = Date.now() - startedAt;
+            const usageTotal = typeof result.usage?.total === 'number' ? result.usage.total : 0;
+            return {
+                ok: true,
+                result,
+                messages,
+                trajectory,
+                turns,
+                durationMs,
+                usageTotal,
+            };
+        })
+            .catch((e) => ({
+            ok: false,
+            error: `框架异常: ${e.message}`,
+        }));
+        // 墙钟超时：到期时把 pending 的 run 替换为失败结果
+        const timeoutOutcome = new Promise((resolve) => {
+            const t = setTimeout(() => resolve({ ok: false, error: `超时（${timeoutMs}ms）` }), timeoutMs);
+            t.unref?.();
+        });
+        const outcome = await Promise.race([run, timeoutOutcome]);
+        if (!outcome.ok)
+            return { error: outcome.error };
+        const trace = {
+            caseId: c.id,
+            suite: c.suite,
+            origin: c.origin,
+            result: outcome.result,
+            trajectory: outcome.trajectory,
+            messages: outcome.messages,
+            durationMs: outcome.durationMs,
+            usageTotal: outcome.usageTotal,
+            turns: outcome.turns,
+        };
+        // 预算熔断：token 超限直接判失败
+        const maxTokens = c.metadata?.maxTokens;
+        if (maxTokens !== undefined && trace.usageTotal > maxTokens) {
+            return { error: `预算熔断：usage.total=${trace.usageTotal} > maxTokens=${maxTokens}` };
+        }
+        return { trace };
+    }
+    finally {
+        await runtime.close().catch(() => undefined);
+    }
+}
+/** 单用例（含 repeats） */
+async function runCase(c, config) {
+    const repeats = config.mode === 'mock' ? (config.repeats ?? 1) : (config.repeats ?? 1);
+    const scorers = resolveScorers(c.expected, c.scorers);
+    const startedAt = Date.now();
+    let passed = true;
+    let score = 0;
+    let scoreCount = 0;
+    const scoreResults = [];
+    let runError;
+    let steps = 0;
+    let usageTotal = 0;
+    let durationMs = 0;
+    for (let r = 0; r < repeats; r++) {
+        const { trace, error } = await runCaseOnce(c, config, r);
+        durationMs += Date.now() - startedAt;
+        if (error || !trace) {
+            passed = false;
+            runError = error;
+            scoreResults.push({
+                scorer: 'success',
+                score: 0,
+                passed: false,
+                reason: `运行失败: ${error ?? '未知错误'}`,
+            });
+            break;
+        }
+        steps = trace.trajectory.length;
+        usageTotal += trace.usageTotal;
+        const caseScore = scoreTrace(trace, scorers);
+        passed = passed && caseScore.passed;
+        score += caseScore.score;
+        scoreCount += 1;
+        if (!caseScore.passed) {
+            scoreResults.push(...caseScore.scores.filter((s) => !s.passed).map((s) => ({
+                ...s,
+                reason: s.reason,
+            })));
+        }
+    }
+    return {
+        caseId: c.id,
+        suite: c.suite,
+        origin: c.origin,
+        passed,
+        score: scoreCount === 0 ? 0 : score / scoreCount,
+        scores: scoreResults,
+        error: runError,
+        durationMs,
+        steps,
+        usageTotal,
+    };
+}
+/** 简单并发队列（避免引入依赖） */
+async function runAllConcurrent(items, concurrency, fn) {
+    const results = new Array(items.length);
+    let next = 0;
+    const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+        while (next < items.length) {
+            const i = next++;
+            results[i] = await fn(items[i]);
+        }
+    });
+    await Promise.all(workers);
+    return results;
+}
+function summarize(results) {
+    const groups = new Map();
+    for (const r of results) {
+        const list = groups.get(r.suite) ?? [];
+        list.push(r);
+        groups.set(r.suite, list);
+    }
+    const out = {};
+    for (const [key, list] of groups) {
+        const total = list.length;
+        const passed = list.filter((r) => r.passed).length;
+        out[key] = {
+            total,
+            passed,
+            passRate: total === 0 ? 0 : passed / total,
+            avgScore: total === 0 ? 0 : list.reduce((a, r) => a + r.score, 0) / total,
+        };
+    }
+    return out;
+}
+/** 执行一次评测 */
+export async function runEval(cases, config) {
+    const startedAt = Date.now();
+    const filtered = config.suites && config.suites.length > 0
+        ? cases.filter((c) => config.suites.includes(c.suite))
+        : cases;
+    const concurrency = config.concurrency ?? DEFAULT_CONCURRENCY;
+    const results = await runAllConcurrent(filtered, concurrency, (c) => runCase(c, config));
+    const casesPassed = results.filter((r) => r.passed).length;
+    return {
+        runId: `run_${new Date().toISOString().replace(/[:.]/g, '-')}`,
+        startedAt: new Date(startedAt).toISOString(),
+        durationMs: Date.now() - startedAt,
+        mode: config.mode,
+        totals: {
+            cases: results.length,
+            passed: casesPassed,
+            passRate: results.length === 0 ? 0 : casesPassed / results.length,
+            avgScore: results.length === 0
+                ? 0
+                : results.reduce((a, r) => a + r.score, 0) / results.length,
+            usageTotal: results.reduce((a, r) => a + r.usageTotal, 0),
+        },
+        bySuite: summarize(results),
+        byOrigin: summarize(results.map((r) => ({ ...r, suite: r.origin }))),
+        results,
+    };
+}
+//# sourceMappingURL=runner.js.map

package/dist/src/core/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/core/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EACL,aAAa,EACb,aAAa,GACd,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAUtD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAE9B,yCAAyC;AACzC,SAAS,aAAa,CAAC,MAAc,EAAE,MAAc;IACnD,OAAO,QAAQ,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,KAAK,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,oCAAoC;AACpC,KAAK,UAAU,WAAW,CACxB,CAAW,EACX,MAAiB,EACjB,MAAc;IAEd,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,KAAK,EAAE,wCAAwC,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAClF,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,IAAI,MAAM,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IAE9E,MAAM,KAAK,GAAW,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACvE,MAAM,OAAO,GAAG,aAAa,CAAC;QAC5B,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE;QACxC,QAAQ,EAAE,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QAC1C,KAAK;QACL,QAAQ,EAAE,QAAQ;QAClB,gBAAgB,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC;KACpD,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAa7B,IAAI,GAAwB,CAAC;IAE7B,IAAI,CAAC;QACH,GAAG,GAAG,OAAO;aACV,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;aACnD,IAAI,CAAC,CAAC,MAAM,EAAc,EAAE;YAC3B,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACnC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAC1C,MAAM,UAAU,GACd,OAAO,MAAM,CAAC,KAAK,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnE,OAAO;gBACL,EAAE,EAAE,IAAI;gBACR,MAAM;gBACN,QAAQ;gBACR,UAAU;gBACV,KAAK;gBACL,UAAU;gBACV,UAAU;aACX,CAAC;QACJ,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,CAAU,EAAc,EAAE,CAAC,CAAC;YAClC,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,SAAU,CAAW,CAAC,OAAO,EAAE;SACvC,CAAC,CAAC,CAAC;QAEN,kCAAkC;QAClC,MAAM,cAAc,GAAG,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,EAAE;YACzD,MAAM,CAAC,GAAG,UAAU,CAClB,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,KAAK,EAAE,CAAC,EACzD,SAAS,CACV,CAAC;YACD,CAA4B,CAAC,KAAK,EAAE,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,OAAO,CAAC,EAAE;YAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;QAEjD,MAAM,KAAK,GAAa;YACtB,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC;QAEF,qBAAqB;QACrB,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC;QACxC,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;YAC5D,OAAO,EAAE,KAAK,EAAE,oBAAoB,KAAK,CAAC,UAAU,gBAAgB,SAAS,EAAE,EAAE,CAAC;QACpF,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,CAAC;IACnB,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,qBAAqB;AACrB,KAAK,UAAU,OAAO,CACpB,CAAW,EACX,MAAiB;IAEjB,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;IACvF,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,YAAY,GAAyB,EAAE,CAAC;IAC9C,IAAI,QAA4B,CAAC;IACjC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QACjC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACzD,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACrC,IAAI,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,GAAG,KAAK,CAAC;YACf,QAAQ,GAAG,KAAK,CAAC;YACjB,YAAY,CAAC,IAAI,CAAC;gBAChB,MAAM,EAAE,SAAS;gBACjB,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,SAAS,KAAK,IAAI,MAAM,EAAE;aACnC,CAAC,CAAC;YACH,MAAM;QACR,CAAC;QACD,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;QAChC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;QAC/B,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7C,MAAM,GAAG,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC;QACpC,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC;QACzB,UAAU,IAAI,CAAC,CAAC;QAChB,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YACtB,YAAY,CAAC,IAAI,CACf,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACvD,GAAG,CAAC;gBACJ,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB,CAAC,CAAC,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE,CAAC,CAAC,EAAE;QACZ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,MAAM;QACN,KAAK,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,UAAU;QAChD,MAAM,EAAE,YAAY;QACpB,KAAK,EAAE,QAAQ;QACf,UAAU;QACV,KAAK;QACL,UAAU;KACX,CAAC;AACJ,CAAC;AAED,qBAAqB;AACrB,KAAK,UAAU,gBAAgB,CAC7B,KAAU,EACV,WAAmB,EACnB,EAA2B;IAE3B,MAAM,OAAO,GAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACrF,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,OAAqB;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,MAAM,GAAG,GAAiC,EAAE,CAAC;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACnD,GAAG,CAAC,GAAG,CAAC,GAAG;YACT,KAAK;YACL,MAAM;YACN,QAAQ,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK;YAC1C,QAAQ,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,KAAK;SAC1E,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,aAAa;AACb,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAiB,EACjB,MAAiB;IAEjB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QACvC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACvD,CAAC,CAAC,KAAK,CAAC;IAEZ,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,mBAAmB,CAAC;IAC9D,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAClE,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CACnB,CAAC;IAEF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,OAAO;QACL,KAAK,EAAE,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE;QAC9D,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;QAC5C,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE;YACN,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM,EAAE,WAAW;YACnB,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM;YACjE,QAAQ,EACN,OAAO,CAAC,MAAM,KAAK,CAAC;gBAClB,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;YAC/D,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC;QAC3B,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACpE,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/src/core/scorer/index.d.ts

@@ -0,0 +1,19 @@
+/**
+ * packages/eval/src/core/scorer/index.ts - 评分入口与 expected 归一化
+ *
+ * - expected（声明式）→ ScorerConfig[]（与显式 scorers 合并）
+ * - scoreCase：依次执行规则评分器，加权合成用例总分
+ */
+import type { ExpectedResult, RunTrace, ScoreResult, ScorerConfig } from '../types';
+export { RULE_SCORER_TYPES, runRuleScorer, partialMatch } from './rule';
+/** 声明式 expected → 评分器配置 */
+export declare function expectedToScorers(expected: ExpectedResult): ScorerConfig[];
+/** 合并用例的 expected + scorers */
+export declare function resolveScorers(expected?: ExpectedResult, explicit?: ScorerConfig[]): ScorerConfig[];
+export interface CaseScore {
+    passed: boolean;
+    score: number;
+    scores: ScoreResult[];
+}
+/** 对一次运行轨迹执行全部评分器 */
+export declare function scoreTrace(trace: RunTrace, scorers: ScorerConfig[]): CaseScore;

package/dist/src/core/scorer/index.js

@@ -0,0 +1,56 @@
+/**
+ * packages/eval/src/core/scorer/index.ts - 评分入口与 expected 归一化
+ *
+ * - expected（声明式）→ ScorerConfig[]（与显式 scorers 合并）
+ * - scoreCase：依次执行规则评分器，加权合成用例总分
+ */
+import { runRuleScorer } from './rule.js';
+export { RULE_SCORER_TYPES, runRuleScorer, partialMatch } from './rule.js';
+/** 声明式 expected → 评分器配置 */
+export function expectedToScorers(expected) {
+    switch (expected.type) {
+        case 'exact':
+        case 'contains':
+            return [{ type: expected.type, params: { value: expected.value } }];
+        case 'regex':
+            return [
+                { type: 'regex', params: { value: expected.value, flags: expected.flags ?? '' } },
+            ];
+        case 'tool-call':
+            return [
+                {
+                    type: 'tool-call',
+                    params: { calls: expected.calls, order: expected.order ?? 'subset' },
+                },
+            ];
+        case 'tools-used':
+            return [{ type: 'tools-used', params: { tools: expected.tools } }];
+        case 'json-field':
+            return 'value' in expected
+                ? [{ type: 'json-field', params: { path: expected.path, value: expected.value } }]
+                : [{ type: 'json-field', params: { path: expected.path } }];
+        case 'success':
+            return [{ type: 'success', params: { value: expected.value ?? true } }];
+        case 'stop-reason':
+            return [{ type: 'stop-reason', params: { value: expected.value } }];
+    }
+}
+/** 合并用例的 expected + scorers */
+export function resolveScorers(expected, explicit) {
+    return [...(expected ? expectedToScorers(expected) : []), ...(explicit ?? [])];
+}
+/** 对一次运行轨迹执行全部评分器 */
+export function scoreTrace(trace, scorers) {
+    const scores = scorers.map((s) => runRuleScorer(trace, s.type, s.params));
+    const totalWeight = scorers.reduce((acc, s) => acc + (s.weight ?? 1), 0);
+    const weighted = totalWeight === 0
+        ? 0
+        : scores.reduce((acc, r, i) => acc + r.score * (scorers[i].weight ?? 1), 0) /
+            totalWeight;
+    return {
+        passed: scores.length > 0 && scores.every((r) => r.passed),
+        score: scorers.length === 0 ? 0 : weighted,
+        scores,
+    };
+}
+//# sourceMappingURL=index.js.map

package/dist/src/core/scorer/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/core/scorer/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,OAAO,EAAqB,aAAa,EAAE,MAAM,QAAQ,CAAC;AAE1D,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAExE,2BAA2B;AAC3B,MAAM,UAAU,iBAAiB,CAAC,QAAwB;IACxD,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,OAAO,CAAC;QACb,KAAK,UAAU;YACb,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtE,KAAK,OAAO;YACV,OAAO;gBACL,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE;aAClF,CAAC;QACJ,KAAK,WAAW;YACd,OAAO;gBACL;oBACE,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,QAAQ,EAAE;iBACrE;aACF,CAAC;QACJ,KAAK,YAAY;YACf,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrE,KAAK,YAAY;YACf,OAAO,OAAO,IAAI,QAAQ;gBACxB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;gBAClF,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAChE,KAAK,SAAS;YACZ,OAAO,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1E,KAAK,aAAa;YAChB,OAAO,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,+BAA+B;AAC/B,MAAM,UAAU,cAAc,CAC5B,QAAyB,EACzB,QAAyB;IAEzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AACjF,CAAC;AAQD,qBAAqB;AACrB,MAAM,UAAU,UAAU,CAAC,KAAe,EAAE,OAAuB;IACjE,MAAM,MAAM,GAAkB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9C,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CACvC,CAAC;IACF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,QAAQ,GACZ,WAAW,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YACzE,WAAW,CAAC;IAClB,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1D,KAAK,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ;QAC1C,MAAM;KACP,CAAC;AACJ,CAAC"}

package/dist/src/core/scorer/rule.d.ts

@@ -0,0 +1,17 @@
+/**
+ * packages/eval/src/core/scorer/rule.ts - 规则评分器
+ *
+ * M1 范围：8 个确定性评分器（零 LLM 成本）：
+ *   exact / contains / regex   —— 最终文本断言
+ *   json-field                 —— 结构化输出断言（dot 路径取值）
+ *   tool-call                  —— 工具调用轨迹断言（顺序 + 参数部分匹配）
+ *   tools-used                 —— 去重工具集合断言（对齐 Result.toolsUsed）
+ *   success / stop-reason      —— 运行状态断言
+ */
+import type { RunTrace, ScoreResult } from '../types';
+export declare function partialMatch(expected: unknown, actual: unknown): boolean;
+type RuleParams = Record<string, unknown>;
+export declare const RULE_SCORER_TYPES: string[];
+/** 执行单个规则评分器；未知类型 / 参数错误返回 scorer 自身失败而非抛出 */
+export declare function runRuleScorer(trace: RunTrace, type: string, params: RuleParams): ScoreResult;
+export {};