oh-my-knowledge 0.38.0 → 0.40.0

package/dist/renderer/summary.js

@@ -2,7 +2,8 @@ import { e, fmtNum, fmtCost, fmtDuration, COLORS, t } from './layout.js';
 import { icon as svgIcon } from './icons.js';
 import { generateAnalysisSummary } from '../analysis/report-diagnostics.js';
 import { pValueCategory } from '../eval-core/statistics.js';
-import { computeVerdict, ENSEMBLE_STRONG_PEARSON, ENSEMBLE_DISSENT_PEARSON } from '../eval-core/verdict.js';
+import { ciLevelLabel } from '../eval-core/bootstrap.js';
+import { computeVerdict, medianStabilityCV, STABILITY_UNSTABLE_CV, ENSEMBLE_STRONG_PEARSON, ENSEMBLE_DISSENT_PEARSON } from '../eval-core/verdict.js';
 /**
  * Verdict pill — sticky banner at the top of the HTML report giving the same
  * one-line conclusion as the `omk eval` CLI. Both surfaces share the
@@ -95,19 +96,10 @@ const SHIP_LABEL = {
     SOLO: { zh: '缺对照', en: 'ADD CONTROL' },
 };
 function computeMedianCVPercent(report) {
-    const variance = report.variance;
-    if (!variance || (variance.runs ?? 0) < 2)
-        return null;
-    const cvs = [];
-    for (const v of Object.values(variance.perVariant ?? {})) {
-        if (typeof v.stddev === 'number' && typeof v.mean === 'number' && v.mean > 0) {
-            cvs.push((v.stddev / v.mean) * 100);
-        }
-    }
-    if (cvs.length === 0)
-        return null;
-    cvs.sort((a, b) => a - b);
-    return cvs[Math.floor(cvs.length / 2)];
+    // 单一来源:与 verdict 稳定性门控读同一个真·中位 CV(偶数 variant 取中间两项平均,非上中位),
+    // 杜绝 hero CV chip 与 verdict 在偶数 variant 上对不齐。返回百分号刻度。
+    const stab = medianStabilityCV(report);
+    return stab ? stab.cv * 100 : null;
 }
 export function renderVerdictPill(report, lang) {
     let result;
@@ -134,11 +126,12 @@ export function renderVerdictPill(report, lang) {
     if (ci) {
         const sign = ci.estimate >= 0 ? '+' : '';
         const cvSuffix = cvPct != null
-            ? (lang === 'zh' ? `;多轮稳定性 CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? '稳' : cvPct < 15 ? '中' : '不稳'})` : `; CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? 'stable' : cvPct < 15 ? 'moderate' : 'unstable'})`)
+            ? (lang === 'zh' ? `;多轮稳定性 CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? '稳' : cvPct <= STABILITY_UNSTABLE_CV * 100 ? '中' : '不稳'})` : `; CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? 'stable' : cvPct <= STABILITY_UNSTABLE_CV * 100 ? 'moderate' : 'unstable'})`)
             : '';
+        const pctLabel = ciLevelLabel(report.meta?.pairComparisons?.[0]?.alpha);
         const ciTipBase = lang === 'zh'
-            ? `实验组与对照组综合分均值差(Δ)。bootstrap 95% 可信区间 [${ci.low}, ${ci.high}]，${ci.significant ? '不含 0 = 差异显著' : '跨过 0 = 差异不显著'}${cvSuffix}`
-            : `Treatment minus control mean composite score (Δ). Bootstrap 95% CI [${ci.low}, ${ci.high}], ${ci.significant ? 'excludes 0 ⇒ significant' : 'spans 0 ⇒ not significant'}${cvSuffix}`;
+            ? `实验组与对照组综合分均值差(Δ)。bootstrap ${pctLabel} 可信区间 [${ci.low}, ${ci.high}]，${ci.significant ? '不含 0 = 差异显著' : '跨过 0 = 差异不显著'}${cvSuffix}`
+            : `Treatment minus control mean composite score (Δ). Bootstrap ${pctLabel} CI [${ci.low}, ${ci.high}], ${ci.significant ? 'excludes 0 ⇒ significant' : 'spans 0 ⇒ not significant'}${cvSuffix}`;
         metrics.push({
             label: lang === 'zh' ? '分差' : 'Δ',
             value: `${sign}${ci.estimate}`,
@@ -167,6 +160,12 @@ export function renderPairwiseDiff(pairs, lang) {
     const validPairs = pairs.filter((p) => p.diffBootstrapCI);
     if (validPairs.length === 0)
         return '';
+    // 同一报告内所有比较共享 family α(α/K),取首对的 alpha 作 family 级标签;K=1 无 alpha → 名义 95%、不加注解
+    // (与历史渲染逐字节一致,保护既有快照)。仅 K≥2(alpha 存在)才追加 Bonferroni 说明 + 非 95% 的 CI 标签。
+    const pctLabel = ciLevelLabel(validPairs[0].alpha);
+    const corrected = validPairs[0].alpha != null;
+    const bonferroniNoteZh = corrected ? `多比较已按 Bonferroni 收 α(故 CI 宽于 95%)。` : '';
+    const bonferroniNoteEn = corrected ? ` Bonferroni-corrected α across multiple comparisons (CI wider than 95%).` : '';
     const rows = validPairs.map((p) => {
         const ci = p.diffBootstrapCI;
         const sigClass = ci.significant ? 'green' : 'text-muted';
@@ -182,13 +181,13 @@ export function renderPairwiseDiff(pairs, lang) {
     }).join('');
     return `
     <h2 style="margin-top:24px">${lang === 'zh' ? '配对对比 (Bootstrap CI)' : 'Pairwise comparison (bootstrap CI)'}</h2>
-    <p style="font-size:13px;color:var(--text-secondary);margin:4px 0 12px">${lang === 'zh' ? 'control vs treatment 的均值差 95% CI。CI 不含 0 = 显著差异。bootstrap 不假设分布,适合 LLM 序数评分。' : '95% CI on (treatment - control) mean diff. 0 outside CI = significant. Bootstrap is distribution-free, fits ordinal LLM scores.'}</p>
+    <p style="font-size:13px;color:var(--text-secondary);margin:4px 0 12px">${lang === 'zh' ? `control vs treatment 的均值差 ${pctLabel} CI。CI 不含 0 = 显著差异。bootstrap 不假设分布,适合 LLM 序数评分。${bonferroniNoteZh}` : `${pctLabel} CI on (treatment - control) mean diff. 0 outside CI = significant. Bootstrap is distribution-free, fits ordinal LLM scores.${bonferroniNoteEn}`}</p>
     <div class="table-wrap">
     <table class="summary-table">
       <thead><tr>
         <th>${lang === 'zh' ? '对照' : 'Pair'}</th>
         <th title="${t('bootstrapDiffLabel', lang)}">${t('bootstrapDiffLabel', lang)}</th>
-        <th>95% CI</th>
+        <th>${pctLabel} CI</th>
         <th>${lang === 'zh' ? '显著性' : 'Significance'}</th>
         <th>${lang === 'zh' ? '重采样数' : 'samples'}</th>
       </tr></thead>
@@ -525,7 +524,7 @@ export function renderSummaryCards(variants, summary, lang, variance) {
                 label = lang === 'zh' ? '稳定' : 'Stable';
                 stabColor = 'var(--green)';
             }
-            else if (cvPct < 15) {
+            else if (cvPct <= STABILITY_UNSTABLE_CV * 100) {
                 label = lang === 'zh' ? '较稳' : 'Moderate';
                 stabColor = 'var(--yellow)';
             }
@@ -873,9 +872,11 @@ function computeJudgeAgreementBadge(variants, summary, lang) {
     };
 }
 function computeSignificanceBadge(report, lang) {
-    const ci = report.meta?.pairComparisons?.[0]?.diffBootstrapCI;
+    const pair0 = report.meta?.pairComparisons?.[0];
+    const ci = pair0?.diffBootstrapCI;
     if (!ci)
         return null;
+    const pctLabel = ciLevelLabel(pair0?.alpha);
     const status = ci.significant ? 'pass' : 'warn';
     const label = ci.significant
         ? (lang === 'zh' ? '差异显著' : 'significant')
@@ -884,7 +885,7 @@ function computeSignificanceBadge(report, lang) {
         key: 'sig',
         label,
         status,
-        detail: lang === 'zh' ? `bootstrap 95% CI [${ci.low}, ${ci.high}]${ci.significant ? '（不含 0）' : '（跨过 0）'}` : `Bootstrap 95% CI [${ci.low}, ${ci.high}]${ci.significant ? ' (excludes 0)' : ' (spans 0)'}`,
+        detail: lang === 'zh' ? `bootstrap ${pctLabel} CI [${ci.low}, ${ci.high}]${ci.significant ? '（不含 0）' : '（跨过 0）'}` : `Bootstrap ${pctLabel} CI [${ci.low}, ${ci.high}]${ci.significant ? ' (excludes 0)' : ' (spans 0)'}`,
     };
 }
 function computeSaturationBadge(report, variants, lang) {

package/dist/server/report-server.js

@@ -9,7 +9,7 @@ import { renderDoctorDetail } from '../renderer/doctor-detail-renderer.js';
 import { assessHealth } from '../renderer/skill-detail-renderer.js';
 import { renderObservationInboxPage } from '../renderer/observation-inbox-renderer.js';
 import { DEFAULT_LANG, t, layout } from '../renderer/layout.js';
-import { loadAllManagedRecords, resolveManagedDir, managedDir as projectManagedDir, listManagedRows } from '../managed/index.js';
+import { loadAllManagedRecords, resolveManagedDir, managedDir as projectManagedDir, listManagedRows, buildVersionScores } from '../managed/index.js';
 import { renderManagedList, renderManagedHistory } from '../renderer/managed-history-renderer.js';
 import { DEFAULT_JOBS_DIR } from '../eval-core/default-dirs.js';
 import { resolveObserveHealthDir, projectObserveHealthDir, resolveDoctorsDir, projectDoctorsDir, projectReportsDir, globalReportsDir } from '../eval-core/measurement-dirs.js';
@@ -706,8 +706,17 @@ export function createReportServer({ port, host: hostOption, reportsDir, analyse
                     res.end(lang === 'en' ? 'managed record not found' : '受管记录不存在');
                     return;
                 }
+                // 版本回归曲线要每版的 composite/CI —— 按 evidence.reportId 把报告读出来(去重),读不到的点
+                // buildVersionScores 自会跳过。曲线数据在路由侧算好,renderManagedHistory 保持纯函数、可 snapshot。
+                const reportsById = new Map();
+                for (const ev of record.evidence) {
+                    // 按「打分视角」结构化看报告:受管证据指向的是单次 eval 报告(带 artifactHashes / summary);
+                    // 万一是 batch 报告,meta 无 artifactHashes → buildVersionScores 自会跳过,不会误画。
+                    if (!reportsById.has(ev.reportId))
+                        reportsById.set(ev.reportId, (await reportStore.get(ev.reportId)));
+                }
                 res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
-                res.end(renderManagedHistory(record, lang));
+                res.end(renderManagedHistory(record, lang, buildVersionScores(record, reportsById)));
                 return;
             }
             if (path === '/observe-inbox') {

package/dist/types/eval.d.ts

@@ -139,6 +139,7 @@ export interface Artifact {
     contentHash?: string;
     locator?: string;
     ref?: string;
+    resolvedCommit?: string;
     cwd?: string;
     skillRoot?: string;
     execRoot?: string;
@@ -160,6 +161,7 @@ export interface VariantConfig {
     cwd: string | null;
     locator?: string;
     ref?: string;
+    resolvedCommit?: string;
     allowedSkills?: string[];
 }
 /** 远端 git 源的结构化引用 —— url/ref/spec 分字段,永不拼成单串再 split(避开 parseGitInput 的 `:`

package/dist/types/judge.d.ts

@@ -129,6 +129,14 @@ export interface AssertionDetail {
     weight: number;
     passed: boolean;
     message?: string;
+    /**
+     * Layer override for `computeLayeredScores`. Only set for combinator types whose layer is not a
+     * static property of `type` — currently `assert-set`, whose layer is resolved at grading time from
+     * its leaf children: homogeneous (all fact / all behavior) → that layer; mixed (or empty) → field
+     * left unset so the combinator is excluded from layered attribution (no single layer is honest).
+     * Leaf assertions never set this; they classify by `ASSERTION_LAYER[type]`.
+     */
+    layer?: 'fact' | 'behavior';
 }
 export interface AssertionResults {
     passed: number;

package/dist/types/managed.d.ts

@@ -50,6 +50,13 @@ export interface ManagedEvidenceRef {
         judgePromptHash?: string;
         debiasMode?: Array<'length' | 'position'>;
     };
+    /** §7 #234/#236:这一版被测内容所在的 git commit(full SHA)= 评测时该 variant 的 git ref 解析出的 commit
+     *  (resolveArtifacts 物化时 `git rev-parse <ref>^{commit}`),作每版「还原坐标」指针 —— list / Studio 据此给
+     *  `git checkout <sha> -- <path>` 把源带回这一版(字节级还原由 git 做,omk 不存版本字节)。是 variant 自己的
+     *  ref(可能 branch/tag/HEAD/旧 SHA)而非进程 cwd 的 HEAD —— 内容从 object DB 按 ref 取,故与工作树 dirty
+     *  与否无关。只对本地 git variant 记(远端源还原是重装 fetch-pin SHA;file 源无 git 坐标)。缺失 = 无坐标可
+     *  还原,诚实留空,不臆造。 */
+    gitCommit?: string;
 }
 export type ManagedDecisionKind = 'promote' | 'reject' | 'rollback';
 /** 一次人工管理决定。install 时为空,promote/reject/rollback 追加(append-only 事件流)。 */
@@ -72,6 +79,36 @@ export interface ManagedDecision {
         overriddenBlocks?: string[];
     };
 }
+/** observe → 管理支柱的生产健康观测(#235)。append-only,与 evidence / decisions 同为事实事件流。
+ *  observe 报告只带 skill **名**(无 contentHash)→ 按 name + kind 绑记录,不锚 contentHash;且观测的是线上
+ *  **正在跑的那一版**(可能已非记录当前版),故是**版本无关的生产信号**,产出读时 marker(production gap),
+ *  绝不翻 stale 生命周期(§6.1 只有内容漂移翻 stale)。样本建议只提示,不改样本集。 */
+export interface ManagedObservation {
+    /** 限定判别字(裸 kind 留给 ArtifactKind)。 */
+    observationKind: 'production-health';
+    /** observe-health 报告 id —— append 去重主键(observe 无 contentHash,故不用 (reportId, contentHash))。 */
+    reportId: string;
+    /** 被观测**流量窗口的结束时刻**(report.meta.timeRange.to)——不是报告生成的「此刻」(generatedAt 恒约等于
+     *  now,拿它做 latest-wins 会让所有观测看起来一样新)。deriveProductionGap 据此取最新一条观测。注:观测是
+     *  **版本无关的生产信号**(量的是线上部署版),不按源码版归因,见 deriveProductionGap。 */
+    observedAt: string;
+    /** 该 skill 在窗口内的盲区率与严重度加权盲区率(直接取 observe 已算好的 per-skill 值,不重算)。 */
+    gapRate: number;
+    weightedGapRate: number;
+    /** 统计功效:underpowered = 段数太少不可知(§6.1 unknown,既不放行也不拦截,不当确诊盲区)。 */
+    confidence: 'high' | 'low' | 'underpowered';
+    /** per-skill 色带,由 observe CLI 侧用 observability 的 healthBandOf 算好传入(阈值单一来源)。 */
+    healthBand: 'green' | 'yellow' | 'red';
+    /** 该 skill 在窗口内的被观测段数(功效上下文)。 */
+    segmentCount: number;
+    /** 盲区按信号类型计数 —— 驱动「建议补哪类用例」提示(只展示,不生成用例)。 */
+    gapByType: {
+        failed_search: number;
+        explicit_marker: number;
+        hedging: number;
+        repeated_failure: number;
+    };
+}
 export interface ManagedArtifactSource {
     /** 源类型(限定判别字,非裸 kind)。`file`=本地路径;`git`=当前仓库某 ref 或远端(带 url)。 */
     sourceKind: 'file' | 'git';
@@ -104,6 +141,9 @@ export interface ManagedArtifactRecord {
     distribution: ManagedDistributionTarget[];
     evidence: ManagedEvidenceRef[];
     decisions: ManagedDecision[];
+    /** #235 observe 生产健康观测(append-only)。旧记录无 → optional;deriveManagedState 不依赖它,
+     *  production gap 是读时 marker(deriveProductionGap),不翻 stale。 */
+    observations?: ManagedObservation[];
 }
 /** 读时推导的生命周期标签——不持久(见文件头说明)。`promoted` 由当前内容(contentHash 匹配)有一条
  *  promote 决定推出:它是 measurable 之上的「已人工接受当前版本」,故优先级高于 measurable。 */

package/dist/types/report.d.ts

@@ -182,6 +182,16 @@ export interface VariantPairComparison {
         samples: number;
         significant: boolean;
     };
+    /**
+     * Effective per-comparison significance level used for `diffBootstrapCI` after
+     * multiple-comparison (Bonferroni) correction: `DEFAULT_BOOTSTRAP_ALPHA / K`, where K is the
+     * number of treatment-vs-control comparisons in this report. Present **only when K ≥ 2** —
+     * the CI and `significant` then reflect the corrected (wider) interval. Absent ⇒ the nominal
+     * α (single comparison / classic A-B, no correction). Renderers derive the displayed
+     * confidence level ((1 − alpha)·100%) from this so a Bonferroni-widened CI is never mislabeled
+     * "95%".
+     */
+    alpha?: number;
 }
 export interface GitInfo {
     commit: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oh-my-knowledge",
-  "version": "0.38.0",
+  "version": "0.40.0",
   "packageManager": "yarn@4.16.0",
   "description": "Evaluation framework for LLM knowledge inputs — prompts, RAG corpora, skills, agent workflows. Fix the model, vary the artifact. Built-in statistical rigor: bootstrap CI, Krippendorff α, length-debias, saturation curves.",
   "type": "module",
@@ -92,11 +92,11 @@
   "license": "MIT",
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.3.143",
-    "@anthropic-ai/sdk": "^0.102.0",
+    "@anthropic-ai/sdk": "^0.104.1",
     "@inquirer/prompts": "^8.4.3",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@oclif/core": "^4",
-    "@openai/codex-sdk": "0.137.0",
+    "@openai/codex-sdk": "0.139.0",
     "ajv": "^8.18.0",
     "chart.js": "^4.5.1",
     "js-yaml": "^4.1.1",