oh-my-knowledge 0.0.1 → 0.18.0

package/dist/src/grading/layered-scores.js

@@ -0,0 +1,62 @@
+const FACTUAL_ASSERTION_TYPES = new Set([
+    'contains',
+    'not_contains',
+    'regex',
+    'json_valid',
+    'json_schema',
+    'equals',
+    'not_equals',
+    'contains_all',
+    'contains_any',
+    'semantic_similarity',
+    'tool_output_contains',
+    'tool_input_contains',
+]);
+const BEHAVIORAL_ASSERTION_TYPES = new Set([
+    'starts_with',
+    'ends_with',
+    'min_length',
+    'max_length',
+    'word_count_min',
+    'word_count_max',
+    'cost_max',
+    'latency_max',
+    'turns_min',
+    'turns_max',
+    'tools_called',
+    'tools_not_called',
+    'tools_count_min',
+    'tools_count_max',
+    'custom',
+]);
+function ratioToScore(ratio) {
+    return Number((1 + ratio * 4).toFixed(2));
+}
+function scoreFromDetails(details) {
+    if (details.length === 0)
+        return null;
+    const totalWeight = details.reduce((s, d) => s + d.weight, 0);
+    if (totalWeight === 0)
+        return null;
+    const passedWeight = details.filter((d) => d.passed).reduce((s, d) => s + d.weight, 0);
+    return ratioToScore(passedWeight / totalWeight);
+}
+export function computeLayeredScores(results) {
+    const layered = {};
+    if (results.assertions?.details) {
+        const factDetails = results.assertions.details.filter((d) => FACTUAL_ASSERTION_TYPES.has(d.type));
+        const behaviorDetails = results.assertions.details.filter((d) => BEHAVIORAL_ASSERTION_TYPES.has(d.type));
+        layered.factScore = scoreFromDetails(factDetails) ?? undefined;
+        layered.behaviorScore = scoreFromDetails(behaviorDetails) ?? undefined;
+    }
+    // 评委打 0 分是"完全不合格"的合法低分,不是缺失——保留进 composite 计算,不做 > 0 gate。
+    // 只排除 undefined(真正缺数据:没配 judge 或打分失败)。fact/behavior 通过 ratioToScore
+    // 产出的分数理论上 ≥ 1,null 判断即可。
+    if (typeof results.llmScore === 'number') {
+        layered.judgeScore = results.llmScore;
+    }
+    const scores = [layered.factScore, layered.behaviorScore, layered.judgeScore].filter((s) => s != null);
+    const compositeScore = scores.length > 0 ? Number((scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(2)) : 0;
+    return { compositeScore, layeredScores: layered };
+}
+//# sourceMappingURL=layered-scores.js.map

package/dist/src/grading/layered-scores.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"layered-scores.js","sourceRoot":"","sources":["../../../src/grading/layered-scores.ts"],"names":[],"mappings":"AAEA,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC;IACtC,UAAU;IACV,cAAc;IACd,OAAO;IACP,YAAY;IACZ,aAAa;IACb,QAAQ;IACR,YAAY;IACZ,cAAc;IACd,cAAc;IACd,qBAAqB;IACrB,sBAAsB;IACtB,qBAAqB;CACtB,CAAC,CAAC;AAEH,MAAM,0BAA0B,GAAG,IAAI,GAAG,CAAC;IACzC,aAAa;IACb,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,gBAAgB;IAChB,gBAAgB;IAChB,UAAU;IACV,aAAa;IACb,WAAW;IACX,WAAW;IACX,cAAc;IACd,kBAAkB;IAClB,iBAAiB;IACjB,iBAAiB;IACjB,QAAQ;CACT,CAAC,CAAC;AAEH,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,gBAAgB,CAAC,OAA0B;IAClD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9D,IAAI,WAAW,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvF,OAAO,YAAY,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC;AAClD,CAAC;AAOD,MAAM,UAAU,oBAAoB,CAAC,OAAuB;IAC1D,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,IAAI,OAAO,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC;QAChC,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClG,MAAM,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,0BAA0B,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACzG,OAAO,CAAC,SAAS,GAAG,gBAAgB,CAAC,WAAW,CAAC,IAAI,SAAS,CAAC;QAC/D,OAAO,CAAC,aAAa,GAAG,gBAAgB,CAAC,eAAe,CAAC,IAAI,SAAS,CAAC;IACzE,CAAC;IAED,2DAA2D;IAC3D,oEAAoE;IACpE,0BAA0B;IAC1B,IAAI,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACzC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC;IACxC,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IACpH,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtH,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;AACpD,CAAC"}

package/dist/src/inputs/eval-config.d.ts

@@ -0,0 +1,13 @@
+import type { EvalConfig, EvalConfigVariant, VariantSpec } from '../types.js';
+/**
+ * Load and validate an eval.yaml (or .json) config file.
+ * All relative paths in the config are resolved against the config file's directory.
+ */
+export declare function loadEvalConfig(configPath: string): EvalConfig;
+/**
+ * Convert EvalConfig.variants into VariantSpec[] (the CLI-internal representation).
+ * A variant's `cwd` (if present) is merged into the expression as `artifact@cwd`
+ * so the downstream variant resolver can treat CLI and config uniformly.
+ */
+export declare function configVariantsToSpecs(variants: EvalConfigVariant[]): VariantSpec[];
+//# sourceMappingURL=eval-config.d.ts.map

package/dist/src/inputs/eval-config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-config.d.ts","sourceRoot":"","sources":["../../../src/inputs/eval-config.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACZ,MAAM,aAAa,CAAC;AAIrB;;;GAGG;AACH,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,UAAU,CAU7D;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,WAAW,EAAE,CAMlF"}

package/dist/src/inputs/eval-config.js

@@ -0,0 +1,136 @@
+import { readFileSync, existsSync } from 'node:fs';
+import { resolve, dirname, isAbsolute, join } from 'node:path';
+import { parseYaml } from './load-samples.js';
+const VALID_ROLES = ['control', 'treatment'];
+/**
+ * Load and validate an eval.yaml (or .json) config file.
+ * All relative paths in the config are resolved against the config file's directory.
+ */
+export function loadEvalConfig(configPath) {
+    const absPath = resolve(configPath);
+    if (!existsSync(absPath)) {
+        throw new Error(`--config 指定的文件不存在: ${absPath}`);
+    }
+    const raw = readFileSync(absPath, 'utf-8');
+    const isJson = absPath.endsWith('.json');
+    const parsed = isJson ? JSON.parse(raw) : parseYaml(raw);
+    const config = validateEvalConfig(parsed, configPath);
+    return resolveConfigPaths(config, dirname(absPath));
+}
+/**
+ * Convert EvalConfig.variants into VariantSpec[] (the CLI-internal representation).
+ * A variant's `cwd` (if present) is merged into the expression as `artifact@cwd`
+ * so the downstream variant resolver can treat CLI and config uniformly.
+ */
+export function configVariantsToSpecs(variants) {
+    return variants.map((v) => ({
+        name: v.name,
+        role: v.role,
+        expr: v.cwd ? `${v.artifact}@${v.cwd}` : v.artifact,
+    }));
+}
+function validateEvalConfig(parsed, configPath) {
+    if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+        throw new Error(`${configPath}: 顶层必须是对象`);
+    }
+    const obj = parsed;
+    if (typeof obj.samples !== 'string' || !obj.samples) {
+        throw new Error(`${configPath}: samples 字段必填，需为字符串`);
+    }
+    if (!Array.isArray(obj.variants) || obj.variants.length === 0) {
+        throw new Error(`${configPath}: variants 字段必填，需为非空数组`);
+    }
+    const variants = [];
+    const seen = new Set();
+    let hasAnyRole = false;
+    for (const [i, raw] of obj.variants.entries()) {
+        if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
+            throw new Error(`${configPath}: variants[${i}] 必须是对象`);
+        }
+        const v = raw;
+        if (typeof v.name !== 'string' || !v.name) {
+            throw new Error(`${configPath}: variants[${i}].name 必填，需为字符串`);
+        }
+        if (typeof v.role !== 'string' || !VALID_ROLES.includes(v.role)) {
+            throw new Error(`${configPath}: variants[${i}].role 必须是 'control' 或 'treatment'（当前：${JSON.stringify(v.role)}）`);
+        }
+        if (typeof v.artifact !== 'string' || !v.artifact) {
+            throw new Error(`${configPath}: variants[${i}].artifact 必填，需为字符串`);
+        }
+        if (v.cwd !== undefined && typeof v.cwd !== 'string') {
+            throw new Error(`${configPath}: variants[${i}].cwd 必须是字符串`);
+        }
+        if (seen.has(v.name)) {
+            throw new Error(`${configPath}: variants[${i}].name "${v.name}" 重复`);
+        }
+        seen.add(v.name);
+        hasAnyRole = true;
+        variants.push({
+            name: v.name,
+            role: v.role,
+            artifact: v.artifact,
+            cwd: v.cwd,
+        });
+    }
+    if (!hasAnyRole) {
+        throw new Error(`${configPath}: variants 里至少要有一个 control 或 treatment`);
+    }
+    const assertStringOpt = (key) => {
+        if (obj[key] !== undefined && typeof obj[key] !== 'string') {
+            throw new Error(`${configPath}: ${key} 必须是字符串`);
+        }
+    };
+    const assertNumberOpt = (key) => {
+        if (obj[key] !== undefined && typeof obj[key] !== 'number') {
+            throw new Error(`${configPath}: ${key} 必须是数字`);
+        }
+    };
+    const assertBoolOpt = (key) => {
+        if (obj[key] !== undefined && typeof obj[key] !== 'boolean') {
+            throw new Error(`${configPath}: ${key} 必须是布尔值`);
+        }
+    };
+    assertStringOpt('executor');
+    assertStringOpt('model');
+    if (obj.judgeModel !== undefined && obj.judgeModel !== null && typeof obj.judgeModel !== 'string') {
+        throw new Error(`${configPath}: judgeModel 必须是字符串或 null`);
+    }
+    if (obj.judgeExecutor !== undefined && obj.judgeExecutor !== null && typeof obj.judgeExecutor !== 'string') {
+        throw new Error(`${configPath}: judgeExecutor 必须是字符串或 null`);
+    }
+    assertNumberOpt('concurrency');
+    assertNumberOpt('timeoutMs');
+    assertBoolOpt('noCache');
+    assertBoolOpt('blind');
+    assertStringOpt('mcpConfig');
+    return {
+        samples: obj.samples,
+        executor: obj.executor,
+        model: obj.model,
+        judgeModel: obj.judgeModel,
+        judgeExecutor: obj.judgeExecutor,
+        concurrency: obj.concurrency,
+        timeoutMs: obj.timeoutMs,
+        noCache: obj.noCache,
+        blind: obj.blind,
+        mcpConfig: obj.mcpConfig,
+        variants,
+    };
+}
+function resolveConfigPaths(config, configDir) {
+    const resolveRel = (p) => (isAbsolute(p) ? p : join(configDir, p));
+    // Artifact expressions that are not file paths (baseline / git: / plain names) stay as-is.
+    const looksLikePath = (expr) => expr.startsWith('./') || expr.startsWith('../') || expr.startsWith('/') || /\.(md|yaml|yml|json)$/i.test(expr);
+    const isNonPathExpr = (expr) => expr === 'baseline' || expr.startsWith('git:');
+    return {
+        ...config,
+        samples: resolveRel(config.samples),
+        mcpConfig: config.mcpConfig ? resolveRel(config.mcpConfig) : undefined,
+        variants: config.variants.map((v) => ({
+            ...v,
+            artifact: isNonPathExpr(v.artifact) ? v.artifact : (looksLikePath(v.artifact) ? resolveRel(v.artifact) : v.artifact),
+            cwd: v.cwd ? resolveRel(v.cwd) : undefined,
+        })),
+    };
+}
+//# sourceMappingURL=eval-config.js.map

package/dist/src/inputs/eval-config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-config.js","sourceRoot":"","sources":["../../../src/inputs/eval-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAQ9C,MAAM,WAAW,GAA8B,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;AAExE;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,UAAkB;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,sBAAsB,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,MAAM,GAAY,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACtD,OAAO,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AACtD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAA6B;IACjE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ;KACpD,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAe,EAAE,UAAkB;IAC7D,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,WAAW,CAAC,CAAC;IAC5C,CAAC;IACD,MAAM,GAAG,GAAG,MAAiC,CAAC;IAE9C,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,sBAAsB,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,wBAAwB,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,QAAQ,GAAwB,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,IAAK,GAAG,CAAC,QAAsB,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7D,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,SAAS,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,CAAC,GAAG,GAA8B,CAAC;QACzC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,iBAAiB,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAsB,CAAC,EAAE,CAAC;YAClF,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,cAAc,CAAC,yCAAyC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAC/F,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,qBAAqB,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,CAAC,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,cAAc,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,WAAW,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC;QACvE,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjB,UAAU,GAAG,IAAI,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAsB;YAC9B,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,GAAG,EAAE,CAAC,CAAC,GAAyB;SACjC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,wCAAwC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,eAAe,GAAG,CAAC,GAAW,EAAQ,EAAE;QAC5C,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,KAAK,GAAG,SAAS,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC;IACF,MAAM,eAAe,GAAG,CAAC,GAAW,EAAQ,EAAE;QAC5C,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,KAAK,GAAG,QAAQ,CAAC,CAAC;QACjD,CAAC;IACH,CAAC,CAAC;IACF,MAAM,aAAa,GAAG,CAAC,GAAW,EAAQ,EAAE;QAC1C,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,KAAK,GAAG,SAAS,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC;IAEF,eAAe,CAAC,UAAU,CAAC,CAAC;IAC5B,eAAe,CAAC,OAAO,CAAC,CAAC;IACzB,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,CAAC,UAAU,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QAClG,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,2BAA2B,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC3G,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,8BAA8B,CAAC,CAAC;IAC/D,CAAC;IACD,eAAe,CAAC,aAAa,CAAC,CAAC;IAC/B,eAAe,CAAC,WAAW,CAAC,CAAC;IAC7B,aAAa,CAAC,SAAS,CAAC,CAAC;IACzB,aAAa,CAAC,OAAO,CAAC,CAAC;IACvB,eAAe,CAAC,WAAW,CAAC,CAAC;IAE7B,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,OAAiB;QAC9B,QAAQ,EAAE,GAAG,CAAC,QAA8B;QAC5C,KAAK,EAAE,GAAG,CAAC,KAA2B;QACtC,UAAU,EAAE,GAAG,CAAC,UAAuC;QACvD,aAAa,EAAE,GAAG,CAAC,aAA0C;QAC7D,WAAW,EAAE,GAAG,CAAC,WAAiC;QAClD,SAAS,EAAE,GAAG,CAAC,SAA+B;QAC9C,OAAO,EAAE,GAAG,CAAC,OAA8B;QAC3C,KAAK,EAAE,GAAG,CAAC,KAA4B;QACvC,SAAS,EAAE,GAAG,CAAC,SAA+B;QAC9C,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAkB,EAAE,SAAiB;IAC/D,MAAM,UAAU,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IACnF,2FAA2F;IAC3F,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjH,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAEjD,OAAO;QACL,GAAG,MAAM;QACT,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC;QACnC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;QACtE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACpC,GAAG,CAAC;YACJ,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;YACpH,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS;SAC3C,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}

package/dist/src/inputs/load-samples.d.ts

@@ -0,0 +1,16 @@
+import type { Sample } from '../types.js';
+import type { DependencyRequirements } from '../eval-core/dependency-checker.js';
+export declare function parseYaml(text: string): unknown;
+export interface LoadSamplesResult {
+    samples: Sample[];
+    requires?: DependencyRequirements;
+}
+/**
+ * Load samples from a JSON or YAML file.
+ *
+ * Supports two formats:
+ * - Array: `[ { sample_id, prompt, ... } ]` (legacy)
+ * - Object wrapper: `{ requires?: { tools, files, env }, samples: [...] }`
+ */
+export declare function loadSamples(samplesPath: string): LoadSamplesResult;
+//# sourceMappingURL=load-samples.d.ts.map

package/dist/src/inputs/load-samples.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-samples.d.ts","sourceRoot":"","sources":["../../../src/inputs/load-samples.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AAQjF,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAQ/C;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,sBAAsB,CAAC;CACnC;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,iBAAiB,CA8BlE"}

package/dist/src/inputs/load-samples.js

@@ -0,0 +1,51 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import yaml from 'js-yaml';
+export function parseYaml(text) {
+    try {
+        return yaml.load(text);
+    }
+    catch (err) {
+        const yamlError = (typeof err === 'object' && err !== null ? err : {});
+        const line = yamlError.mark ? ` at line ${yamlError.mark.line + 1}` : '';
+        throw new Error(`YAML parse error${line}: ${yamlError.reason || yamlError.message || 'unknown error'}`);
+    }
+}
+/**
+ * Load samples from a JSON or YAML file.
+ *
+ * Supports two formats:
+ * - Array: `[ { sample_id, prompt, ... } ]` (legacy)
+ * - Object wrapper: `{ requires?: { tools, files, env }, samples: [...] }`
+ */
+export function loadSamples(samplesPath) {
+    const rawContent = readFileSync(resolve(samplesPath), 'utf-8');
+    const isYaml = samplesPath.endsWith('.yaml') || samplesPath.endsWith('.yml');
+    const parsed = isYaml ? parseYaml(rawContent) : JSON.parse(rawContent);
+    let samples;
+    let requires;
+    if (Array.isArray(parsed)) {
+        // Legacy array format
+        samples = parsed;
+    }
+    else if (typeof parsed === 'object' && parsed !== null && 'samples' in parsed) {
+        // Object wrapper format
+        const wrapper = parsed;
+        samples = wrapper.samples;
+        requires = wrapper.requires;
+    }
+    else {
+        throw new Error(`无效的样本文件格式: ${samplesPath}（期望数组或包含 samples 字段的对象）`);
+    }
+    if (!Array.isArray(samples) || samples.length === 0) {
+        throw new Error(`无效的样本文件: ${samplesPath}`);
+    }
+    for (const [i, sample] of samples.entries()) {
+        if (!sample.sample_id)
+            throw new Error(`samples[${i}] 缺少必填字段: sample_id`);
+        if (!sample.prompt)
+            throw new Error(`samples[${i}] (${sample.sample_id}) 缺少必填字段: prompt`);
+    }
+    return { samples, requires };
+}
+//# sourceMappingURL=load-samples.js.map

package/dist/src/inputs/load-samples.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-samples.js","sourceRoot":"","sources":["../../../src/inputs/load-samples.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,IAAI,MAAM,SAAS,CAAC;AAU3B,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,SAAS,GAAG,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAkB,CAAC;QACxF,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,SAAS,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,KAAK,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC,OAAO,IAAI,eAAe,EAAE,CAAC,CAAC;IAC1G,CAAC;AACH,CAAC;AAOD;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,WAAmB;IAC7C,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAY,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAEhF,IAAI,OAAiB,CAAC;IACtB,IAAI,QAA4C,CAAC;IAEjD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,sBAAsB;QACtB,OAAO,GAAG,MAAkB,CAAC;IAC/B,CAAC;SAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;QAChF,wBAAwB;QACxB,MAAM,OAAO,GAAG,MAAkE,CAAC;QACnF,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC1B,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAC9B,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,cAAc,WAAW,yBAAyB,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,YAAY,WAAW,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;QAC1E,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,MAAM,CAAC,SAAS,kBAAkB,CAAC,CAAC;IAC5F,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AAC/B,CAAC"}

package/dist/src/inputs/mcp-resolver.d.ts

@@ -0,0 +1,50 @@
+/**
+ * MCP-based URL resolver for eval-samples.
+ *
+ * Reads a MCP config file (.mcp.json), launches configured MCP servers,
+ * and uses them to fetch URL content that cannot be reached via plain HTTP
+ * (e.g. private documentation behind SSO).
+ *
+ * Config format (.mcp.json):
+ * {
+ *   "mcpServers": {
+ *     "docs": {
+ *       "command": "npx",
+ *       "args": ["@example/docs-mcp-server"],
+ *       "env": { "DOCS_API_TOKEN": "xxx" },
+ *       "urlPatterns": ["docs.example.com"],
+ *       "fetchTool": { "name": "get_doc", "urlParam": "url" }
+ *     }
+ *   }
+ * }
+ *
+ * Advanced: urlTransform + contentExtract for complex MCP tools:
+ * {
+ *   "fetchTool": {
+ *     "name": "fetch_doc",
+ *     "urlTransform": {
+ *       "regex": "docs\\.example\\.com/([^/]+/[^/]+)/([^/?#]+)",
+ *       "params": { "namespace": "$1", "slug": "$2" }
+ *     },
+ *     "contentExtract": "data.body"
+ *   }
+ * }
+ */
+import type { Sample, McpServers } from '../types.js';
+/**
+ * Load MCP config from a JSON file.
+ * Returns only servers that declare `urlPatterns` + `fetchTool`.
+ * Returns null if file not found or no eligible servers.
+ */
+export declare function loadMcpConfig(configPath?: string): McpServers | null;
+/**
+ * Resolve URLs in samples via MCP servers.
+ * Mutates samples in-place — same contract as url-fetcher's resolveUrls().
+ */
+export declare function resolveMcpUrls(samples: Sample[], mcpServers: McpServers | null): Promise<Set<string>>;
+/**
+ * Close all active MCP client connections.
+ * Should be called when the evaluation run finishes.
+ */
+export declare function stopAllServers(): Promise<void>;
+//# sourceMappingURL=mcp-resolver.d.ts.map

package/dist/src/inputs/mcp-resolver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp-resolver.d.ts","sourceRoot":"","sources":["../../../src/inputs/mcp-resolver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAOH,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAA8B,MAAM,aAAa,CAAC;AAgDlF;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAwBpE;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,UAAU,GAAG,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CA0F3G;AAED;;;GAGG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAWpD"}