oh-my-knowledge 0.23.0 → 0.25.0

package/dist/src/doctor/rules.js

@@ -0,0 +1,216 @@
+/**
+ * omk doctor 内置规则注册表。
+ *
+ * doctor 是**纯静态/低成本检查**, 不碰 LLM 连通性 — executor / judge 连通性由
+ * evaluation preflight 负责。边界清晰: doctor 静态(零 LLM 调用), eval 动态。
+ *
+ * 每条 rule 回答一个独立的「skill 能不能被有意义评测」子问题:
+ *   - skill_readable: 文件能读、内容非空且有最小长度
+ *   - skill_metadata: front-matter(若有) YAML 合法、directory-skill 有 SKILL.md
+ *   - dependencies: 引用的 tool / file / env / preflight 完整(复用 preflightDependencies)
+ *   - samples_contract: 仅在传 samples 时跑,warn 级,校验 samples 非空 + 含 prompt
+ *
+ * fatal-fail 时 rule 引擎不中断后续 rule 执行(让用户一次看到全貌),
+ * 但 DoctorReport.failed 会置 true。
+ *
+ * 扩展接口预留: registerRule() 允许业务方注入自定义 rule(v0.22 不暴露 CLI flag,
+ * 仅作为 library API 占位)。
+ */
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import yaml from 'js-yaml';
+import { tCli } from '../cli/i18n.js';
+import { preflightDependencies } from '../eval-core/dependency-checker.js';
+/** 用 js-yaml 真正解析 front-matter。能抓 unterminated string、非法缩进、
+ *  duplicate key 等手写 pattern check 漏掉的 case。 */
+function checkFrontmatter(content) {
+    const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+    if (!match)
+        return { ok: true };
+    try {
+        yaml.load(match[1]);
+        return { ok: true };
+    }
+    catch (err) {
+        const e = (typeof err === 'object' && err !== null ? err : {});
+        const lineSuffix = e.mark ? ` at line ${e.mark.line + 1}` : '';
+        return { ok: false, error: `${e.reason || e.message || 'YAML parse error'}${lineSuffix}` };
+    }
+}
+function summarizeDependencyIssues(missing, lang) {
+    const counts = { tool: 0, file: 0, env: 0 };
+    for (const m of missing)
+        counts[m.category] += 1;
+    const parts = [];
+    const labels = lang === 'zh'
+        ? { tool: '工具', file: '文件', env: '环境变量' }
+        : { tool: 'tool', file: 'file', env: 'env' };
+    if (counts.tool > 0)
+        parts.push(`${counts.tool} ${labels.tool}`);
+    if (counts.file > 0)
+        parts.push(`${counts.file} ${labels.file}`);
+    if (counts.env > 0)
+        parts.push(`${counts.env} ${labels.env}`);
+    return parts.join(', ') || (lang === 'zh' ? '未知' : 'unknown');
+}
+// ---------------------------------------------------------------------------
+// Rules
+// ---------------------------------------------------------------------------
+const SKILL_MIN_LENGTH = 10;
+export const skillReadableRule = {
+    id: 'skill_readable',
+    severity: 'fatal',
+    labelKey: 'cli.doctor.rule.skill_readable',
+    async check(ctx) {
+        const content = ctx.artifact.content;
+        if (content === null || content === undefined) {
+            return {
+                status: 'fail',
+                message: tCli('cli.doctor.skill_readable.fail.missing', ctx.lang),
+                hint: tCli('cli.doctor.skill_readable.hint.missing', ctx.lang),
+                detail: { length: 0 },
+            };
+        }
+        const trimmed = content.trim();
+        if (trimmed.length === 0) {
+            return {
+                status: 'fail',
+                message: tCli('cli.doctor.skill_readable.fail.empty', ctx.lang),
+                hint: tCli('cli.doctor.skill_readable.hint.missing', ctx.lang),
+                detail: { length: 0 },
+            };
+        }
+        if (trimmed.length < SKILL_MIN_LENGTH) {
+            return {
+                status: 'fail',
+                message: tCli('cli.doctor.skill_readable.fail.too_short', ctx.lang, { length: trimmed.length }),
+                hint: tCli('cli.doctor.skill_readable.hint.too_short', ctx.lang),
+                detail: { length: trimmed.length, minimum: SKILL_MIN_LENGTH },
+            };
+        }
+        return {
+            status: 'pass',
+            message: tCli('cli.doctor.skill_readable.pass', ctx.lang, { length: trimmed.length }),
+            detail: { length: trimmed.length },
+        };
+    },
+};
+export const skillMetadataRule = {
+    id: 'skill_metadata',
+    severity: 'fatal',
+    labelKey: 'cli.doctor.rule.skill_metadata',
+    async check(ctx) {
+        // directory-skill 必须有 SKILL.md
+        if (ctx.artifact.skillRoot) {
+            const skillMdPath = join(ctx.artifact.skillRoot, 'SKILL.md');
+            if (!existsSync(skillMdPath)) {
+                return {
+                    status: 'fail',
+                    message: tCli('cli.doctor.skill_metadata.fail.missing_skillmd', ctx.lang),
+                    hint: tCli('cli.doctor.skill_metadata.hint.missing_skillmd', ctx.lang),
+                    detail: { skillRoot: ctx.artifact.skillRoot, expectedPath: skillMdPath },
+                };
+            }
+        }
+        // front-matter 若存在则要合法(走 js-yaml 真解析)。不存在 = pure markdown skill,合法。
+        const content = ctx.artifact.content ?? '';
+        const fmCheck = checkFrontmatter(content);
+        if (!fmCheck.ok) {
+            return {
+                status: 'fail',
+                message: tCli('cli.doctor.skill_metadata.fail.frontmatter_invalid', ctx.lang, { error: fmCheck.error ?? '' }),
+                hint: tCli('cli.doctor.skill_metadata.hint.frontmatter', ctx.lang),
+                detail: { error: fmCheck.error },
+            };
+        }
+        return {
+            status: 'pass',
+            message: tCli('cli.doctor.skill_metadata.pass', ctx.lang),
+        };
+    },
+};
+export const dependenciesPresentRule = {
+    id: 'dependencies_present',
+    severity: 'fatal',
+    labelKey: 'cli.doctor.rule.dependencies',
+    async check(ctx) {
+        const content = ctx.artifact.content ?? '';
+        const result = await preflightDependencies([content], ctx.samples ?? [], ctx.dependencyCwd ?? ctx.cwd, // 与 evaluation preflight 同规则; 见 DoctorContext.dependencyCwd
+        ctx.requires, // samples wrapper 显式 requires
+        [ctx.artifact]);
+        if (!result.ok) {
+            const summary = summarizeDependencyIssues(result.missing, ctx.lang);
+            return {
+                status: 'fail',
+                message: tCli('cli.doctor.dependencies.fail', ctx.lang, { summary }),
+                hint: tCli('cli.doctor.dependencies.hint', ctx.lang),
+                detail: { missing: result.missing },
+            };
+        }
+        return {
+            status: 'pass',
+            message: tCli('cli.doctor.dependencies.pass', ctx.lang),
+        };
+    },
+};
+export const samplesContractAlignedRule = {
+    id: 'samples_contract_aligned',
+    severity: 'warn',
+    labelKey: 'cli.doctor.rule.samples_contract',
+    async check(ctx) {
+        if (!ctx.samples) {
+            return {
+                status: 'skipped',
+                message: tCli('cli.doctor.samples_contract.skipped', ctx.lang),
+            };
+        }
+        if (ctx.samples.length === 0) {
+            return {
+                status: 'warn',
+                message: tCli('cli.doctor.samples_contract.warn.empty', ctx.lang),
+                hint: tCli('cli.doctor.samples_contract.hint', ctx.lang),
+                detail: { count: 0 },
+            };
+        }
+        const missing = ctx.samples.filter((s) => !s.prompt || s.prompt.trim().length === 0);
+        if (missing.length > 0) {
+            return {
+                status: 'warn',
+                message: tCli('cli.doctor.samples_contract.warn.missing_prompt', ctx.lang, { count: missing.length }),
+                hint: tCli('cli.doctor.samples_contract.hint', ctx.lang),
+                detail: { missingCount: missing.length, totalCount: ctx.samples.length },
+            };
+        }
+        return {
+            status: 'pass',
+            message: tCli('cli.doctor.samples_contract.pass', ctx.lang, { count: ctx.samples.length }),
+            detail: { count: ctx.samples.length },
+        };
+    },
+};
+// ---------------------------------------------------------------------------
+// Registry
+// ---------------------------------------------------------------------------
+/** v0.22 内置规则注册表。执行顺序即此处定义顺序。
+ *  doctor 不包含 executor 连通性检查 — 该职责属于 evaluation preflight。 */
+export const BUILTIN_RULES = [
+    skillReadableRule,
+    skillMetadataRule,
+    dependenciesPresentRule,
+    samplesContractAlignedRule,
+];
+/** 扩展 hook(v0.22 不通过 CLI flag 暴露,仅 library API 占位)。 */
+const customRules = [];
+export function registerRule(rule) {
+    if (BUILTIN_RULES.some((r) => r.id === rule.id) || customRules.some((r) => r.id === rule.id)) {
+        throw new Error(`doctor rule id collision: ${rule.id}`);
+    }
+    customRules.push(rule);
+}
+export function getRegisteredRules() {
+    return [...BUILTIN_RULES, ...customRules];
+}
+export function __resetCustomRulesForTest() {
+    customRules.length = 0;
+}
+//# sourceMappingURL=rules.js.map

package/dist/src/doctor/rules.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rules.js","sourceRoot":"","sources":["../../../src/doctor/rules.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,IAAI,MAAM,SAAS,CAAC;AAC3B,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAkB3E;gDACgD;AAChD,SAAS,gBAAgB,CAAC,OAAe;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC3D,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACpB,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;IACtB,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAkB,CAAC;QAChF,MAAM,UAAU,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,OAAO,IAAI,kBAAkB,GAAG,UAAU,EAAE,EAAE,CAAC;IAC7F,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAAC,OAA0B,EAAE,IAAiB;IAC9E,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,OAAO;QAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACjD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,KAAK,IAAI;QAC1B,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE;QACzC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;IAC/C,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACjE,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACjE,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAC9D,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AAChE,CAAC;AAED,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,MAAM,CAAC,MAAM,iBAAiB,GAAe;IAC3C,EAAE,EAAE,gBAAgB;IACpB,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,gCAAgC;IAC1C,KAAK,CAAC,KAAK,CAAC,GAAkB;QAC5B,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QACrC,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC9C,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,wCAAwC,EAAE,GAAG,CAAC,IAAI,CAAC;gBACjE,IAAI,EAAE,IAAI,CAAC,wCAAwC,EAAE,GAAG,CAAC,IAAI,CAAC;gBAC9D,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;aACtB,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,sCAAsC,EAAE,GAAG,CAAC,IAAI,CAAC;gBAC/D,IAAI,EAAE,IAAI,CAAC,wCAAwC,EAAE,GAAG,CAAC,IAAI,CAAC;gBAC9D,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;aACtB,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;YACtC,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,0CAA0C,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC/F,IAAI,EAAE,IAAI,CAAC,0CAA0C,EAAE,GAAG,CAAC,IAAI,CAAC;gBAChE,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE;aAC9D,CAAC;QACJ,CAAC;QACD,OAAO;YACL,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,gCAAgC,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YACrF,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;SACnC,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAe;IAC3C,EAAE,EAAE,gBAAgB;IACpB,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,gCAAgC;IAC1C,KAAK,CAAC,KAAK,CAAC,GAAkB;QAC5B,+BAA+B;QAC/B,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YAC7D,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7B,OAAO;oBACL,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,IAAI,CAAC,gDAAgD,EAAE,GAAG,CAAC,IAAI,CAAC;oBACzE,IAAI,EAAE,IAAI,CAAC,gDAAgD,EAAE,GAAG,CAAC,IAAI,CAAC;oBACtE,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE;iBACzE,CAAC;YACJ,CAAC;QACH,CAAC;QACD,oEAAoE;QACpE,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAChB,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,oDAAoD,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;gBAC7G,IAAI,EAAE,IAAI,CAAC,4CAA4C,EAAE,GAAG,CAAC,IAAI,CAAC;gBAClE,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;aACjC,CAAC;QACJ,CAAC;QACD,OAAO;YACL,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,gCAAgC,EAAE,GAAG,CAAC,IAAI,CAAC;SAC1D,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAe;IACjD,EAAE,EAAE,sBAAsB;IAC1B,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,8BAA8B;IACxC,KAAK,CAAC,KAAK,CAAC,GAAkB;QAC5B,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,MAAM,qBAAqB,CACxC,CAAC,OAAO,CAAC,EACT,GAAG,CAAC,OAAO,IAAI,EAAE,EACjB,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,GAAG,EAAE,4DAA4D;QAC1F,GAAG,CAAC,QAAQ,EAAmB,8BAA8B;QAC7D,CAAC,GAAG,CAAC,QAAQ,CAAC,CACf,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,yBAAyB,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YACpE,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,8BAA8B,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC;gBACpE,IAAI,EAAE,IAAI,CAAC,8BAA8B,EAAE,GAAG,CAAC,IAAI,CAAC;gBACpD,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE;aACpC,CAAC;QACJ,CAAC;QACD,OAAO;YACL,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,8BAA8B,EAAE,GAAG,CAAC,IAAI,CAAC;SACxD,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,0BAA0B,GAAe;IACpD,EAAE,EAAE,0BAA0B;IAC9B,QAAQ,EAAE,MAAM;IAChB,QAAQ,EAAE,kCAAkC;IAC5C,KAAK,CAAC,KAAK,CAAC,GAAkB;QAC5B,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;gBACL,MAAM,EAAE,SAAS;gBACjB,OAAO,EAAE,IAAI,CAAC,qCAAqC,EAAE,GAAG,CAAC,IAAI,CAAC;aAC/D,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,wCAAwC,EAAE,GAAG,CAAC,IAAI,CAAC;gBACjE,IAAI,EAAE,IAAI,CAAC,kCAAkC,EAAE,GAAG,CAAC,IAAI,CAAC;gBACxD,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;aACrB,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QACrF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,iDAAiD,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;gBACrG,IAAI,EAAE,IAAI,CAAC,kCAAkC,EAAE,GAAG,CAAC,IAAI,CAAC;gBACxD,MAAM,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE;aACzE,CAAC;QACJ,CAAC;QACD,OAAO;YACL,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,kCAAkC,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAC1F,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE;SACtC,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,8EAA8E;AAC9E,WAAW;AACX,8EAA8E;AAE9E;8DAC8D;AAC9D,MAAM,CAAC,MAAM,aAAa,GAAiB;IACzC,iBAAiB;IACjB,iBAAiB;IACjB,uBAAuB;IACvB,0BAA0B;CAC3B,CAAC;AAEF,uDAAuD;AACvD,MAAM,WAAW,GAAiB,EAAE,CAAC;AAErC,MAAM,UAAU,YAAY,CAAC,IAAgB;IAC3C,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QAC7F,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,OAAO,CAAC,GAAG,aAAa,EAAE,GAAG,WAAW,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,yBAAyB;IACvC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;AACzB,CAAC"}

package/dist/src/eval-core/bootstrap.d.ts

@@ -62,7 +62,7 @@ export declare function bootstrapMeanCI(scores: number[], alpha?: number, sample
 export declare function bootstrapDiffCI(scoresA: number[], scoresB: number[], alpha?: number, samples?: number, seed?: number): BootstrapDiffCI;
 /**
  * Generic bootstrap CI for an arbitrary sample-level metric. Used by
- * saturation analysis (Phase 4) to get CI on metrics like stddev or
+ * saturation analysis to get CI on metrics like stddev or
  * agreement, not just mean.
  *
  * @param scores   Original sample.

package/dist/src/eval-core/bootstrap.js

@@ -144,7 +144,7 @@ export function bootstrapDiffCI(scoresA, scoresB, alpha = 0.05, samples = 1000,
 }
 /**
  * Generic bootstrap CI for an arbitrary sample-level metric. Used by
- * saturation analysis (Phase 4) to get CI on metrics like stddev or
+ * saturation analysis to get CI on metrics like stddev or
  * agreement, not just mean.
  *
  * @param scores   Original sample.

package/dist/src/eval-core/cache.d.ts

@@ -2,14 +2,16 @@
  * Executor result cache.
  *
  * Caches successful executor results to disk to avoid redundant API calls.
- * Cache key v2 = "v2:" + sha256(model + system + prompt + cwd + allowedSkills).
+ * Cache key v4 = sha256(model + system + prompt + cwd + allowedSkills + executor + runtime).
  * Loaded into memory on init, flushed to disk on save().
  *
- * key prefix bump v1 → v2 invalidates pre-isolation cache entries
- * (otherwise a strict-baseline run could replay a non-isolated cached result
- * and silently keep the contamination).
+ * Prefix bumps intentionally invalidate old entries when construct-validity
+ * dimensions enter the key:
+ * - v2: allowedSkills / strict isolation
+ * - v3: executor name
+ * - v4: executor runtime fingerprint
  */
 import type { ExecutorCache } from '../types/index.js';
 export declare function createCache(cacheDir: string): ExecutorCache;
-export declare function cacheKey(model: string, system: string, prompt: string, cwd?: string | null, allowedSkills?: string[]): string;
+export declare function cacheKey(model: string, system: string, prompt: string, cwd?: string | null, allowedSkills?: string[], executor?: string, runtimeFingerprint?: string): string;
 //# sourceMappingURL=cache.d.ts.map

package/dist/src/eval-core/cache.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../../src/eval-core/cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,OAAO,KAAK,EAAc,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAInE,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,CAsC3D;AAED,wBAAgB,QAAQ,CACtB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,EACnB,aAAa,CAAC,EAAE,MAAM,EAAE,GACvB,MAAM,CAWR"}
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../../src/eval-core/cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,OAAO,KAAK,EAAc,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAInE,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,CAsC3D;AAED,wBAAgB,QAAQ,CACtB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,EACnB,aAAa,CAAC,EAAE,MAAM,EAAE,EACxB,QAAQ,CAAC,EAAE,MAAM,EACjB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,MAAM,CAaR"}

package/dist/src/eval-core/cache.js

@@ -2,12 +2,14 @@
  * Executor result cache.
  *
  * Caches successful executor results to disk to avoid redundant API calls.
- * Cache key v2 = "v2:" + sha256(model + system + prompt + cwd + allowedSkills).
+ * Cache key v4 = sha256(model + system + prompt + cwd + allowedSkills + executor + runtime).
  * Loaded into memory on init, flushed to disk on save().
  *
- * key prefix bump v1 → v2 invalidates pre-isolation cache entries
- * (otherwise a strict-baseline run could replay a non-isolated cached result
- * and silently keep the contamination).
+ * Prefix bumps intentionally invalidate old entries when construct-validity
+ * dimensions enter the key:
+ * - v2: allowedSkills / strict isolation
+ * - v3: executor name
+ * - v4: executor runtime fingerprint
  */
 import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
@@ -48,16 +50,18 @@ export function createCache(cacheDir) {
         },
     };
 }
-export function cacheKey(model, system, prompt, cwd, allowedSkills) {
+export function cacheKey(model, system, prompt, cwd, allowedSkills, executor, runtimeFingerprint) {
     // allowedSkills 序列化:undefined → "" / [] → "[]" / [...] → 排序后 JSON。
     // 排序保证 ["a","b"] 和 ["b","a"] 命中同一缓存(语义等价)。
     const isoStr = allowedSkills === undefined
         ? ''
         : JSON.stringify([...allowedSkills].sort());
+    // executor + runtime 进 cache key:同 model 名走不同 executor 或同 executor
+    // 换 binary/SDK 版本时输出可能不同,旧 cache 不可复用。
     const hash = createHash('sha256')
-        .update(`${model || ''}\n${system || ''}\n${prompt || ''}\n${cwd || ''}\n${isoStr}`)
+        .update(`${model || ''}\n${system || ''}\n${prompt || ''}\n${cwd || ''}\n${isoStr}\n${executor || ''}\n${runtimeFingerprint || ''}`)
         .digest('hex')
         .slice(0, 16);
-    return `v2:${hash}`;
+    return `v4:${hash}`;
 }
 //# sourceMappingURL=cache.js.map

package/dist/src/eval-core/cache.js.map

@@ -1 +1 @@
-{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../../src/eval-core/cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAEzC,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAE5C,IAAI,KAAK,GAA+B,EAAE,CAAC;IAC3C,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAED,IAAI,KAAK,GAAG,KAAK,CAAC;IAElB,OAAO;QACL,GAAG,CAAC,GAAW;YACb,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QAC5B,CAAC;QAED,GAAG,CAAC,GAAW,EAAE,KAAiB;YAChC,MAAM,SAAS,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;YAC/B,OAAO,SAAS,CAAC,KAAK,CAAC;YACvB,OAAO,SAAS,CAAC,SAAS,CAAC;YAC3B,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;YACvB,KAAK,GAAG,IAAI,CAAC;QACf,CAAC;QAED,IAAI;YACF,IAAI,CAAC,KAAK;gBAAE,OAAO;YACnB,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACxD,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;QAED,IAAI;YACF,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACnC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,KAAa,EACb,MAAc,EACd,MAAc,EACd,GAAmB,EACnB,aAAwB;IAExB,mEAAmE;IACnE,2CAA2C;IAC3C,MAAM,MAAM,GAAG,aAAa,KAAK,SAAS;QACxC,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;SAC9B,MAAM,CAAC,GAAG,KAAK,IAAI,EAAE,KAAK,MAAM,IAAI,EAAE,KAAK,MAAM,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,MAAM,EAAE,CAAC;SACnF,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChB,OAAO,MAAM,IAAI,EAAE,CAAC;AACtB,CAAC"}
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../../src/eval-core/cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAEzC,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAE5C,IAAI,KAAK,GAA+B,EAAE,CAAC;IAC3C,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAED,IAAI,KAAK,GAAG,KAAK,CAAC;IAElB,OAAO;QACL,GAAG,CAAC,GAAW;YACb,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QAC5B,CAAC;QAED,GAAG,CAAC,GAAW,EAAE,KAAiB;YAChC,MAAM,SAAS,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;YAC/B,OAAO,SAAS,CAAC,KAAK,CAAC;YACvB,OAAO,SAAS,CAAC,SAAS,CAAC;YAC3B,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;YACvB,KAAK,GAAG,IAAI,CAAC;QACf,CAAC;QAED,IAAI;YACF,IAAI,CAAC,KAAK;gBAAE,OAAO;YACnB,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACxD,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;QAED,IAAI;YACF,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACnC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,KAAa,EACb,MAAc,EACd,MAAc,EACd,GAAmB,EACnB,aAAwB,EACxB,QAAiB,EACjB,kBAA2B;IAE3B,mEAAmE;IACnE,2CAA2C;IAC3C,MAAM,MAAM,GAAG,aAAa,KAAK,SAAS;QACxC,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,mEAAmE;IACnE,uCAAuC;IACvC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;SAC9B,MAAM,CAAC,GAAG,KAAK,IAAI,EAAE,KAAK,MAAM,IAAI,EAAE,KAAK,MAAM,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,MAAM,KAAK,QAAQ,IAAI,EAAE,KAAK,kBAAkB,IAAI,EAAE,EAAE,CAAC;SACnI,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChB,OAAO,MAAM,IAAI,EAAE,CAAC;AACtB,CAAC"}

package/dist/src/eval-core/comparability.d.ts

@@ -0,0 +1,11 @@
+import type { Lang, Report } from '../types/index.js';
+export interface ComparabilityWarning {
+    code: string;
+    severity: 'warning';
+    zh: string;
+    en: string;
+}
+export declare function reportComparabilityWarnings(report: Report): ComparabilityWarning[];
+export declare function crossReportComparabilityWarnings(before: Report, after: Report): ComparabilityWarning[];
+export declare function formatComparabilityWarnings(warnings: ComparabilityWarning[], lang?: Lang): string;
+//# sourceMappingURL=comparability.d.ts.map

package/dist/src/eval-core/comparability.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparability.d.ts","sourceRoot":"","sources":["../../../src/eval-core/comparability.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,EAAkE,MAAM,mBAAmB,CAAC;AAEtH,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,SAAS,CAAC;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;CACZ;AA0JD,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,MAAM,GAAG,oBAAoB,EAAE,CAoDlF;AAED,wBAAgB,gCAAgC,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,oBAAoB,EAAE,CAwLtG;AAED,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,oBAAoB,EAAE,EAAE,IAAI,GAAE,IAAW,GAAG,MAAM,CAMvG"}

package/dist/src/eval-core/comparability.js

@@ -0,0 +1,296 @@
+function stableStringify(value) {
+    if (value === null || typeof value !== 'object')
+        return JSON.stringify(value);
+    if (Array.isArray(value))
+        return '[' + value.map(stableStringify).join(',') + ']';
+    const entries = Object.keys(value).sort();
+    return '{' + entries.map((k) => JSON.stringify(k) + ':' + stableStringify(value[k])).join(',') + '}';
+}
+function runtimeLabel(runtime) {
+    if (!runtime)
+        return 'missing';
+    const bits = [`${runtime.executor}:${runtime.model}`, `fp=${runtime.fingerprint}`];
+    if (runtime.binary?.version)
+        bits.push(`binary=${runtime.binary.version}`);
+    if (runtime.sdk?.version)
+        bits.push(`sdk=${runtime.sdk.version}`);
+    return bits.join(' ');
+}
+function push(warnings, code, zh, en) {
+    warnings.push({ code, severity: 'warning', zh, en });
+}
+function sorted(values) {
+    return [...(values ?? [])].sort();
+}
+function hasJudge(meta) {
+    return Boolean(meta.judgeModels && meta.judgeModels.length > 0 && !meta.noJudge);
+}
+function judgeKeys(meta) {
+    return (meta.judgeModels ?? []).map((j) => `${j.executor}:${j.model}`);
+}
+function effectiveJudgeRepeat(meta) {
+    return meta.judgeRepeat ?? 1;
+}
+function packageReasons(pkg, label) {
+    if (!pkg)
+        return [`${label} missing`];
+    const reasons = [];
+    if (pkg.error)
+        reasons.push(`${label} probe failed`);
+    if (!pkg.version)
+        reasons.push(`${label} version missing`);
+    return reasons;
+}
+function runtimeUnverifiableReasons(runtime) {
+    const reasons = [];
+    if (runtime.kind === 'api')
+        return reasons;
+    if (runtime.kind === 'agent-sdk') {
+        reasons.push(...packageReasons(runtime.sdk, 'sdk'));
+    }
+    const binary = runtime.binary;
+    if (!binary) {
+        reasons.push('binary missing');
+        return reasons;
+    }
+    if (binary.error)
+        reasons.push('binary probe failed');
+    if (binary.source === 'path') {
+        if (!binary.path)
+            reasons.push('binary path missing');
+        if (!binary.version)
+            reasons.push('binary version missing');
+    }
+    else if (binary.source === 'bundled') {
+        if (!binary.version && !binary.package?.version)
+            reasons.push('bundled binary version missing');
+        if (binary.package)
+            reasons.push(...packageReasons(binary.package, 'bundled package').filter((r) => r !== 'bundled package missing'));
+    }
+    else if (binary.source === 'unknown') {
+        reasons.push('binary source unknown');
+    }
+    if (runtime.kind === 'unknown')
+        reasons.push('runtime kind unknown');
+    return [...new Set(reasons)];
+}
+function pushRuntimeUnverifiable(warnings, code, zhSubject, enSubject, runtime) {
+    if (!runtime)
+        return;
+    const reasons = runtimeUnverifiableReasons(runtime);
+    if (reasons.length === 0)
+        return;
+    push(warnings, code, `${zhSubject} runtime 指纹不可完全审计: ${runtimeLabel(runtime)} (${reasons.join('; ')})。`, `${enSubject} runtime fingerprint is not fully verifiable: ${runtimeLabel(runtime)} (${reasons.join('; ')}).`);
+}
+function runtimeMapKeys(meta) {
+    return sorted(meta.executorRuntimes ? Object.keys(meta.executorRuntimes) : undefined);
+}
+function reportExecutorRuntimeWarnings(report, warnings) {
+    const runtimes = report.meta.executorRuntimes;
+    if (runtimes && Object.keys(runtimes).length > 0) {
+        const missing = report.meta.variants.filter((variant) => !runtimes[variant]);
+        if (missing.length > 0) {
+            push(warnings, 'executor_runtime_missing', `报告缺少部分 variant 的 executor runtime 指纹: ${missing.join(', ')}。`, `Report is missing executor runtime fingerprints for variants: ${missing.join(', ')}.`);
+        }
+        for (const [variant, runtime] of Object.entries(runtimes)) {
+            pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', `variant ${variant} executor`, `Variant ${variant} executor`, runtime);
+        }
+        return;
+    }
+    if (!report.meta.executorRuntime) {
+        push(warnings, 'executor_runtime_missing', '报告缺少 executor runtime 指纹；无法审计 binary / SDK 版本，跨报告严格比较需谨慎。', 'Report is missing executor runtime fingerprint; binary / SDK versions cannot be audited for strict cross-report comparison.');
+        return;
+    }
+    if (report.meta.variants.length > 1) {
+        push(warnings, 'executor_runtimes_missing', '报告缺少 per-variant executor runtime 指纹；多 variant run 无法审计每个分组实际使用的 binary / SDK 版本。', 'Report is missing per-variant executor runtime fingerprints; multi-variant runs cannot audit each group’s actual binary / SDK version.');
+    }
+    pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', 'executor', 'Executor', report.meta.executorRuntime);
+}
+function countSampleHashMismatches(a, b) {
+    const ah = a.meta.sampleHashes;
+    const bh = b.meta.sampleHashes;
+    if (!ah || !bh)
+        return null;
+    let mismatched = 0;
+    let missing = 0;
+    let common = 0;
+    const ids = new Set([...Object.keys(ah), ...Object.keys(bh)]);
+    for (const id of ids) {
+        if (ah[id] == null || bh[id] == null) {
+            missing++;
+            continue;
+        }
+        common++;
+        if (ah[id] !== bh[id])
+            mismatched++;
+    }
+    return { mismatched, missing, common };
+}
+export function reportComparabilityWarnings(report) {
+    const warnings = [];
+    reportExecutorRuntimeWarnings(report, warnings);
+    if (hasJudge(report.meta) && !report.meta.judgePromptHash) {
+        push(warnings, 'judge_prompt_hash_missing', '报告缺少评委提示词指纹；无法确认 LLM 评分语义是否一致。', 'Report is missing judge prompt fingerprint; LLM scoring semantics cannot be verified.');
+    }
+    // Judge runtime audit: every entry in meta.judgeModels carries its own runtime fingerprint.
+    // Skip when noJudge=true (judge didn't run, runtime intentionally absent).
+    if (hasJudge(report.meta)) {
+        const entries = report.meta.judgeModels ?? [];
+        const missing = entries.filter((e) => !e.runtime).map((e) => `${e.executor}:${e.model}`);
+        if (missing.length > 0) {
+            push(warnings, 'judge_runtime_missing', entries.length === 1
+                ? '报告缺少评委 runtime 指纹；无法审计评委 executor 的 binary / SDK 版本。'
+                : `多评委 ensemble 缺少 runtime 指纹: ${missing.join(', ')}。`, entries.length === 1
+                ? 'Report is missing judge runtime fingerprint; judge executor binary / SDK versions cannot be audited.'
+                : `Multi-judge ensemble is missing runtime fingerprints: ${missing.join(', ')}.`);
+        }
+        for (const e of entries) {
+            if (!e.runtime)
+                continue;
+            const label = entries.length === 1 ? '评委' : `评委 ${e.executor}:${e.model}`;
+            const enLabel = entries.length === 1 ? 'Judge' : `Judge ${e.executor}:${e.model}`;
+            pushRuntimeUnverifiable(warnings, 'judge_runtime_unverifiable', label, enLabel, e.runtime);
+        }
+    }
+    if (!report.meta.sampleHashes) {
+        push(warnings, 'sample_hashes_missing', '报告缺少用例指纹；无法确认不同 run 是否测的是同一组用例。', 'Report is missing sample fingerprints; runs cannot be verified to use the same test cases.');
+    }
+    if (!report.meta.skillIsolation) {
+        push(warnings, 'skill_isolation_missing', '报告缺少 skill isolation 快照；baseline 是否被 skill 污染不可审计。', 'Report is missing skill-isolation snapshot; baseline contamination cannot be audited.');
+    }
+    return warnings;
+}
+export function crossReportComparabilityWarnings(before, after) {
+    const warnings = [];
+    const b = before.meta;
+    const a = after.meta;
+    if (b.model !== a.model) {
+        push(warnings, 'model_mismatch', `任务执行模型不同: ${b.model} → ${a.model}。`, `Task execution model changed: ${b.model} → ${a.model}.`);
+    }
+    if (b.executor !== a.executor) {
+        push(warnings, 'executor_mismatch', `executor 不同: ${b.executor} → ${a.executor}。`, `Executor changed: ${b.executor} → ${a.executor}.`);
+    }
+    const bExecutorRuntimeKeys = runtimeMapKeys(b);
+    const aExecutorRuntimeKeys = runtimeMapKeys(a);
+    const hasExecutorRuntimeMap = bExecutorRuntimeKeys.length > 0 || aExecutorRuntimeKeys.length > 0;
+    if (hasExecutorRuntimeMap) {
+        const keys = sorted([...new Set([...b.variants, ...a.variants, ...bExecutorRuntimeKeys, ...aExecutorRuntimeKeys])]);
+        const missing = keys.filter((key) => !b.executorRuntimes?.[key] || !a.executorRuntimes?.[key]);
+        if (missing.length > 0) {
+            push(warnings, 'executor_runtime_missing', `至少一份报告缺少 per-variant executor runtime 指纹: ${missing.join(', ')}。`, `At least one report is missing per-variant executor runtime fingerprints: ${missing.join(', ')}.`);
+        }
+        for (const key of keys) {
+            const beforeRuntime = b.executorRuntimes?.[key];
+            const afterRuntime = a.executorRuntimes?.[key];
+            if (beforeRuntime?.fingerprint && afterRuntime?.fingerprint && beforeRuntime.fingerprint !== afterRuntime.fingerprint) {
+                push(warnings, 'executor_runtime_mismatch', `variant ${key} executor runtime 指纹不同: ${runtimeLabel(beforeRuntime)} → ${runtimeLabel(afterRuntime)}。`, `Variant ${key} executor runtime fingerprint changed: ${runtimeLabel(beforeRuntime)} → ${runtimeLabel(afterRuntime)}.`);
+            }
+            pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', `before variant ${key} executor`, `Before variant ${key} executor`, beforeRuntime);
+            pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', `after variant ${key} executor`, `After variant ${key} executor`, afterRuntime);
+        }
+    }
+    else if (b.executorRuntime?.fingerprint && a.executorRuntime?.fingerprint) {
+        if (b.executorRuntime.fingerprint !== a.executorRuntime.fingerprint) {
+            push(warnings, 'executor_runtime_mismatch', `executor runtime 指纹不同: ${runtimeLabel(b.executorRuntime)} → ${runtimeLabel(a.executorRuntime)}。`, `Executor runtime fingerprint changed: ${runtimeLabel(b.executorRuntime)} → ${runtimeLabel(a.executorRuntime)}.`);
+        }
+    }
+    else {
+        push(warnings, 'executor_runtime_missing', '至少一份报告缺少 executor runtime 指纹；无法确认 binary / SDK 版本一致。', 'At least one report is missing executor runtime fingerprint; binary / SDK version parity cannot be verified.');
+    }
+    if (!hasExecutorRuntimeMap) {
+        pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', 'before executor', 'Before executor', b.executorRuntime);
+        pushRuntimeUnverifiable(warnings, 'executor_runtime_unverifiable', 'after executor', 'After executor', a.executorRuntime);
+    }
+    // Judge comparability — only meaningful when at least one report actually ran a judge.
+    // - 都 noJudge: skip 整段(judge 没跑,任何比较都误导;两份 noJudge 报告 judge 视角下完全可比)
+    // - 一边 noJudge 一边 judged: 给 judge_presence_mismatch 一条明确警告,跳过 runtime/model 比较
+    //   (这两份报告本身就不在同一个测量框架,详细 runtime / model diff 是噪声)
+    // - 两边都 judged: 走 model / repeat / runtime full audit
+    const bJudged = hasJudge(b);
+    const aJudged = hasJudge(a);
+    if (!bJudged && !aJudged) {
+        // 都没跑评委,judge audit 完全跳过;evaluationFramework / skillIsolation 等非 judge 项继续往下走。
+    }
+    else if (bJudged !== aJudged) {
+        push(warnings, 'judge_presence_mismatch', `评委开关不同: ${bJudged ? 'judged' : 'no-judge'} → ${aJudged ? 'judged' : 'no-judge'}。两次跑测的测量框架不一致(LLM 维度缺失),不可在 judge 维度做严格对比。`, `Judge presence differs: ${bJudged ? 'judged' : 'no-judge'} → ${aJudged ? 'judged' : 'no-judge'}. The runs are in different measurement frames (LLM dimension absent on one side); judge-axis comparison is not meaningful.`);
+    }
+    else {
+        // 两边都 judged — single judge 是 ensemble 的 1-entry degenerate case,同一段代码处理。
+        const bKeys = sorted(judgeKeys(b));
+        const aKeys = sorted(judgeKeys(a));
+        if (stableStringify(bKeys) !== stableStringify(aKeys)) {
+            const isSingle = bKeys.length <= 1 && aKeys.length <= 1;
+            push(warnings, isSingle ? 'judge_model_mismatch' : 'judge_models_mismatch', isSingle
+                ? `评委模型不同: ${bKeys[0] ?? 'none'} → ${aKeys[0] ?? 'none'}。`
+                : `多评委 ensemble 配置不同: ${bKeys.join(', ') || 'none'} → ${aKeys.join(', ') || 'none'}。`, isSingle
+                ? `Judge model changed: ${bKeys[0] ?? 'none'} → ${aKeys[0] ?? 'none'}.`
+                : `Multi-judge ensemble changed: ${bKeys.join(', ') || 'none'} → ${aKeys.join(', ') || 'none'}.`);
+        }
+        if (effectiveJudgeRepeat(b) !== effectiveJudgeRepeat(a)) {
+            push(warnings, 'judge_repeat_mismatch', `每条用例评委评价次数不同: ${effectiveJudgeRepeat(b)} → ${effectiveJudgeRepeat(a)}。`, `Judge repeat count changed: ${effectiveJudgeRepeat(b)} → ${effectiveJudgeRepeat(a)}.`);
+        }
+        // Per-judge runtime fingerprint comparison. Iterate the union of judges across both reports.
+        const judgeUnion = sorted([...new Set([...bKeys, ...aKeys])]);
+        if (judgeUnion.length > 0) {
+            const bRuntimeByKey = new Map((b.judgeModels ?? []).map((e) => [`${e.executor}:${e.model}`, e.runtime]));
+            const aRuntimeByKey = new Map((a.judgeModels ?? []).map((e) => [`${e.executor}:${e.model}`, e.runtime]));
+            const missing = judgeUnion.filter((key) => !bRuntimeByKey.get(key) || !aRuntimeByKey.get(key));
+            if (missing.length > 0) {
+                const isSingle = judgeUnion.length === 1;
+                push(warnings, 'judge_runtime_missing', isSingle
+                    ? '至少一份报告缺少评委 runtime 指纹；无法确认评委 binary / SDK 版本一致。'
+                    : `至少一份报告缺少多评委 runtime 指纹: ${missing.join(', ')}。`, isSingle
+                    ? 'At least one report is missing judge runtime fingerprint; judge binary / SDK version parity cannot be verified.'
+                    : `At least one report is missing multi-judge runtime fingerprints: ${missing.join(', ')}.`);
+            }
+            for (const key of judgeUnion) {
+                const beforeRuntime = bRuntimeByKey.get(key);
+                const afterRuntime = aRuntimeByKey.get(key);
+                if (beforeRuntime?.fingerprint && afterRuntime?.fingerprint && beforeRuntime.fingerprint !== afterRuntime.fingerprint) {
+                    push(warnings, 'judge_runtime_mismatch', judgeUnion.length === 1
+                        ? `评委 runtime 指纹不同: ${runtimeLabel(beforeRuntime)} → ${runtimeLabel(afterRuntime)}。`
+                        : `评委 ${key} runtime 指纹不同: ${runtimeLabel(beforeRuntime)} → ${runtimeLabel(afterRuntime)}。`, judgeUnion.length === 1
+                        ? `Judge runtime fingerprint changed: ${runtimeLabel(beforeRuntime)} → ${runtimeLabel(afterRuntime)}.`
+                        : `Judge ${key} runtime fingerprint changed: ${runtimeLabel(beforeRuntime)} → ${runtimeLabel(afterRuntime)}.`);
+                }
+                const labelPrefix = judgeUnion.length === 1 ? '评委' : `评委 ${key}`;
+                const enLabelPrefix = judgeUnion.length === 1 ? 'judge' : `judge ${key}`;
+                pushRuntimeUnverifiable(warnings, 'judge_runtime_unverifiable', `before ${labelPrefix}`, `Before ${enLabelPrefix}`, beforeRuntime);
+                pushRuntimeUnverifiable(warnings, 'judge_runtime_unverifiable', `after ${labelPrefix}`, `After ${enLabelPrefix}`, afterRuntime);
+            }
+        }
+    }
+    // Prompt hash 只在两边都 judged 时比较;一边 noJudge 一边 judged 由上面 judge_presence_mismatch 已覆盖,
+    // 两边 noJudge 时双方都没 hash 也不报错。
+    if (bJudged && aJudged) {
+        if (!b.judgePromptHash || !a.judgePromptHash) {
+            push(warnings, 'judge_prompt_hash_missing', `至少一份报告缺少评委提示词指纹: ${b.judgePromptHash ?? 'missing'} → ${a.judgePromptHash ?? 'missing'}。`, `At least one report is missing judge prompt hash: ${b.judgePromptHash ?? 'missing'} → ${a.judgePromptHash ?? 'missing'}.`);
+        }
+        else if (b.judgePromptHash !== a.judgePromptHash) {
+            push(warnings, 'judge_prompt_hash_mismatch', `评委提示词指纹不同: ${b.judgePromptHash ?? 'missing'} → ${a.judgePromptHash ?? 'missing'}。`, `Judge prompt hash changed: ${b.judgePromptHash ?? 'missing'} → ${a.judgePromptHash ?? 'missing'}.`);
+        }
+    }
+    if ((b.evaluationFramework || a.evaluationFramework) && b.evaluationFramework !== a.evaluationFramework) {
+        push(warnings, 'evaluation_framework_mismatch', `统计框架不同: ${b.evaluationFramework ?? 'legacy'} → ${a.evaluationFramework ?? 'legacy'}。`, `Evaluation framework changed: ${b.evaluationFramework ?? 'legacy'} → ${a.evaluationFramework ?? 'legacy'}.`);
+    }
+    if (stableStringify(b.skillIsolation ?? null) !== stableStringify(a.skillIsolation ?? null)) {
+        push(warnings, 'skill_isolation_mismatch', 'skill isolation 快照不同；baseline 污染边界不一致，严格比较需谨慎。', 'Skill-isolation snapshots differ; baseline contamination boundaries are not equivalent.');
+    }
+    const sampleHashDiff = countSampleHashMismatches(before, after);
+    if (!sampleHashDiff) {
+        push(warnings, 'sample_hashes_missing', '至少一份报告缺少用例指纹；无法确认两次 run 使用同一组用例。', 'At least one report is missing sample fingerprints; test-case parity cannot be verified.');
+    }
+    else if (sampleHashDiff.mismatched > 0 || sampleHashDiff.missing > 0) {
+        push(warnings, 'sample_hashes_mismatch', `用例指纹不一致: ${sampleHashDiff.mismatched} 条内容变化，${sampleHashDiff.missing} 条只出现在其中一份报告。`, `Sample fingerprints differ: ${sampleHashDiff.mismatched} content mismatches, ${sampleHashDiff.missing} samples only appear in one report.`);
+    }
+    return warnings;
+}
+export function formatComparabilityWarnings(warnings, lang = 'zh') {
+    if (warnings.length === 0)
+        return '';
+    const header = lang === 'zh'
+        ? '可比性提示: 以下差异会影响 strict comparison / construct validity'
+        : 'Comparability warnings: the following differences affect strict comparison / construct validity';
+    return [header, ...warnings.map((warning) => `  - ${lang === 'zh' ? warning.zh : warning.en}`)].join('\n');
+}
+//# sourceMappingURL=comparability.js.map