oh-my-knowledge 0.41.0 → 0.43.0

package/dist/server/report-store.js

@@ -5,6 +5,8 @@
  */
 import { readdir, readFile, writeFile, unlink, access, mkdir, rename, stat } from 'node:fs/promises';
 import { join } from 'node:path';
+import { isReportFileName, reportFilePath, reportFileStem } from '../eval-core/artifact-file-names.js';
+import { migrateLegacyReportFiles } from '../eval-core/report-file-migration.js';
 // Per-id in-memory mutex for safe read-modify-write.
 // Uses a queue to avoid the race window between checking and acquiring the lock.
 const locks = new Map();
@@ -67,7 +69,7 @@ export function createFileStore(dir) {
     function isEvaluationReport(report) {
         return report.kind === 'evaluation';
     }
-    // Studio 每个 / 和 /skills/<name> 请求都调 list(),里面对每个 .json 同步 readFile +
+    // Studio 每个 / 和 /skills/<name> 请求都调 list(),里面对每个 .report.json 同步 readFile +
     // JSON.parse。报告数上来后这是主性能瓶颈。缓存策略:fingerprint = dir mtime + 文件名
     // 排序串 + 每个文件 mtime;任一变化 invalidate。fingerprint 算 cheap(只 stat),命中后
     // 完全跳过 readFile。
@@ -76,7 +78,7 @@ export function createFileStore(dir) {
     async function computeListFingerprint() {
         try {
             const dirStat = await stat(dir);
-            const files = (await readdir(dir)).filter((f) => f.endsWith('.json')).sort();
+            const files = (await readdir(dir)).filter(isReportFileName).sort();
             const parts = await Promise.all(files.map(async (f) => {
                 try {
                     const s = await stat(join(dir, f));
@@ -99,18 +101,19 @@ export function createFileStore(dir) {
         catch {
             return [];
         }
+        migrateLegacyReportFiles(dir, 'report');
         const fp = await computeListFingerprint();
         if (fp != null && fp === cachedFingerprint && cachedRuns)
             return cachedRuns;
         const files = (await readdir(dir))
-            .filter((f) => f.endsWith('.json'))
+            .filter(isReportFileName)
             .sort()
             .reverse();
         const runs = [];
         for (const file of files) {
             try {
                 const data = JSON.parse(await readFile(join(dir, file), 'utf-8'));
-                const report = normalizeReportDocument(data, file.replace(/\.json$/, ''));
+                const report = normalizeReportDocument(data, reportFileStem(file) ?? file);
                 if (report)
                     runs.push(report);
             }
@@ -128,8 +131,9 @@ export function createFileStore(dir) {
         return runs;
     }
     async function get(id) {
+        migrateLegacyReportFiles(dir, 'report');
         try {
-            const data = JSON.parse(await readFile(join(dir, `${id}.json`), 'utf-8'));
+            const data = JSON.parse(await readFile(reportFilePath(dir, id), 'utf-8'));
             return normalizeReportDocument(data, id);
         }
         catch {
@@ -138,9 +142,11 @@ export function createFileStore(dir) {
     }
     async function save(id, report) {
         await ensureDir();
-        const tmpPath = join(dir, `${id}.json.tmp.${Date.now()}.${Math.random().toString(36).slice(2)}`);
+        migrateLegacyReportFiles(dir, 'report');
+        const targetPath = reportFilePath(dir, id);
+        const tmpPath = `${targetPath}.tmp.${Date.now()}.${Math.random().toString(36).slice(2)}`;
         await writeFile(tmpPath, JSON.stringify(report, null, 2));
-        await rename(tmpPath, join(dir, `${id}.json`));
+        await rename(tmpPath, targetPath);
     }
     /**
      * Atomic read-modify-write with in-memory mutex.
@@ -157,8 +163,9 @@ export function createFileStore(dir) {
         });
     }
     async function remove(id) {
+        migrateLegacyReportFiles(dir, 'report');
         try {
-            await unlink(join(dir, `${id}.json`));
+            await unlink(reportFilePath(dir, id));
             return true;
         }
         catch (err) {
@@ -169,8 +176,9 @@ export function createFileStore(dir) {
         }
     }
     async function exists(id) {
+        migrateLegacyReportFiles(dir, 'report');
         try {
-            await access(join(dir, `${id}.json`));
+            await access(reportFilePath(dir, id));
             return true;
         }
         catch {

package/dist/server/skill-index.js

@@ -9,14 +9,14 @@
  *     因为同一份 EvaluationReport 同时驱动两个 view tab。
  *   - observe 报告(analysesDir, SkillHealthReport):data.bySkill[name] 每个键作为
  *     一个 skill 的 observe snapshot,取最新 generatedAt。
- *   - doctor:暂无独立持久化路径(omk doctor --html 写指定路径,默认不存盘),
- *     该字段保留 null,渲染层显示"未独立运行 omk doctor"。后续 commit 加默认
- *     ~/.oh-my-knowledge/doctors/ 持久化路径再回填。
+ *   - doctor:读取 `.omk/doctors/*.report.json`,按 skill 名聚合体检历史。
  *
  * 综合 band:eval / observe 任一红 → 红,任一黄 → 黄,全绿 → 绿,皆未跑 → gray。
  */
 import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
 import { join, isAbsolute, basename, dirname } from 'node:path';
+import { isReportFileName, reportFileStem } from '../eval-core/artifact-file-names.js';
+import { migrateLegacyReportFiles } from '../eval-core/report-file-migration.js';
 import { confidenceOf } from '../observability/skill-health-analyzer.js';
 import { computeVerdict } from '../eval-core/verdict.js';
 import { artifactIndexDir, listLiveDoctorCards, cardToDoctorSnapshot, listLiveObserveCards, cardTargetSentinel } from '../eval-core/artifact-index.js';
@@ -26,13 +26,13 @@ import { buildStudioDiagnosisSummary, mergeDiagnosisBundles } from '../diagnosis
 let _indexCache = null;
 /**
  * Sync 版 dir-content fingerprint helper,仿 `src/server/report-store.ts:80-92`
- * 的 async `computeListFingerprint` — 把目录本身的 mtime 跟目录下每个 `.json`
+ * 的 async `computeListFingerprint` — 把目录本身的 mtime 跟目录下每个 `.report.json`
  * 文件的 "filename:mtimeMs:size" 三元组排序拼接成 stable 字符串作为 dir-level
  * 的 content-aware fingerprint。
  *
- * - 目录下任何 .json 文件**新增 / 删除 / 重命名** → 目录 mtime 跟着变,且
+ * - 目录下任何 .report.json 文件**新增 / 删除 / 重命名** → 目录 mtime 跟着变,且
  *   sorted-filenames 列表变,字符串变 → cache invalidate。
- * - 任何**已有同名 .json 文件被外部进程原地覆写内容** → 该文件自己的 mtimeMs
+ * - 任何**已有同名 .report.json 文件被外部进程原地覆写内容** → 该文件自己的 mtimeMs
  *   跟通常情况下 size 都变(byte 长度跟内容相关),字符串里那一 entry 的后缀
  *   变,整体字符串变 → cache invalidate。这是 pre-fix 的 fingerprint(只看
  *   dir mtime+文件数)漏掉的信号(reviewer 2026-05-11 P2-a)。
@@ -48,7 +48,7 @@ function safeDirJsonContentFingerprint(dir) {
     let jsonFiles;
     try {
         dirMtimeMs = statSync(dir).mtimeMs;
-        jsonFiles = readdirSync(dir).filter((f) => f.endsWith('.json')).sort();
+        jsonFiles = readdirSync(dir).filter(isReportFileName).sort();
     }
     catch {
         return `missing:${dir}`;
@@ -71,6 +71,9 @@ function buildIndexFingerprint(reports, analysesDir, doctorsDir, observationsDir
     // safeDirJsonContentFingerprint 返回的 "{dir-mtime}|{file1}:{m}:{s},..."
     // content-aware 字符串。
     const reportIds = reports.map((r) => `${r.id}:${r.meta?.timestamp ?? ''}:${r.kind === 'evaluation' ? r.meta.evolve?.skillName ?? '' : ''}`).join(',');
+    migrateLegacyReportFiles(doctorsDir, 'doctor');
+    migrateLegacyReportFiles(analysesDir, 'observe-health');
+    migrateLegacyReportFiles(observationsDir, 'observe-inbox');
     const doctorsFp = safeDirJsonContentFingerprint(doctorsDir);
     const analysesFp = safeDirJsonContentFingerprint(analysesDir);
     const observationsFp = safeDirJsonContentFingerprint(observationsDir);
@@ -224,14 +227,15 @@ function latestEvalSnapshot(list) {
         return null;
     return list[list.length - 1];
 }
-/** 扫 doctorsDir/*.json,按 skill 名分桶,**返回该 skill 的所有历史 snapshot**(asc 时序)。
+/** 扫 doctorsDir/*.report.json,按 skill 名分桶,**返回该 skill 的所有历史 snapshot**(asc 时序)。
  *  renderer 用最后一项做"当前",前面项画 sparkline。 */
 function scanDoctorReports(dir) {
     const out = {};
+    migrateLegacyReportFiles(dir, 'doctor');
     if (!existsSync(dir))
         return out;
     for (const file of readdirSync(dir)) {
-        if (!file.endsWith('.json'))
+        if (!isReportFileName(file))
             continue;
         try {
             const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
@@ -288,16 +292,17 @@ export function buildSkillIndex(reports, analysesDir, doctorsDir, observationsDi
         list.sort((a, b) => evalSnapshotSortKey(a).localeCompare(evalSnapshotSortKey(b)));
     // ── observe 聚合(历史 list)──────────────────────────────
     const observeBy = {};
+    migrateLegacyReportFiles(analysesDir, 'observe-health');
     if (existsSync(analysesDir)) {
         for (const file of readdirSync(analysesDir)) {
-            if (!file.endsWith('.json'))
+            const id = reportFileStem(file);
+            if (!id)
                 continue;
             try {
                 const data = JSON.parse(readFileSync(join(analysesDir, file), 'utf-8'));
                 if (!data?.bySkill || !data.meta)
                     continue;
                 const generatedAt = data.meta.generatedAt;
-                const id = file.replace(/\.json$/, '');
                 for (const [skill, h] of Object.entries(data.bySkill)) {
                     const snap = {
                         analysisId: id, generatedAt,

package/dist/types/artifact-graph.d.ts

@@ -0,0 +1,93 @@
+export interface ArtifactGraphDocument {
+    documentKind: 'artifact-graph';
+    schemaVersion: 1;
+    graphId: string;
+    generatedAt: string;
+    source: ArtifactGraphSource;
+    scope: ArtifactGraphScope;
+    nodes: ArtifactGraphNode[];
+    edges: ArtifactGraphEdge[];
+    summaries?: ArtifactGraphSummary[];
+}
+export type ArtifactGraphSourceKind = 'doctor' | 'eval' | 'observe';
+export interface ArtifactGraphSource {
+    sourceKind: ArtifactGraphSourceKind;
+    sourceId: string;
+    sourcePath?: string;
+    cliVersion?: string;
+}
+export interface ArtifactGraphScope {
+    cwd: string;
+    artifactKind?: 'skill' | 'prompt' | 'agent' | 'workflow';
+    skillName?: string;
+    artifactHash?: string;
+    sourceLocator?: string;
+    sampleSetHash?: string;
+}
+export type ArtifactGraphLayer = 'definition' | 'measurement' | 'production';
+export type ArtifactGraphNodeRole = 'entity' | 'observation' | 'aggregate';
+export type ArtifactGraphNodeKind = 'skill' | 'skill_file' | 'frontmatter' | 'reference' | 'script' | 'tool' | 'env' | 'preflight' | 'hard_rule' | 'workflow' | 'workflow_node' | 'sample' | 'assertion' | 'variant' | 'doctor_rule_result' | 'eval_result' | 'judge_dimension' | 'diagnostic' | 'trace_session' | 'skill_invocation' | 'tool_call' | 'gap_signal';
+export type ArtifactGraphStatus = 'ok' | 'warning' | 'failed' | 'skipped' | 'unknown' | 'not_measured';
+export interface ArtifactGraphBinding {
+    bindingStrength: 'content-hash' | 'source-locator' | 'runtime-trace' | 'name-only' | 'aggregate';
+    keys: Record<string, string>;
+}
+export interface ArtifactGraphAttrs {
+    display?: Record<string, unknown>;
+    producer?: Record<string, unknown>;
+    experimental?: Record<string, unknown>;
+}
+export interface ArtifactGraphNode {
+    id: string;
+    stableKey: string;
+    nodeKind: ArtifactGraphNodeKind;
+    nodeRole: ArtifactGraphNodeRole;
+    layer: ArtifactGraphLayer;
+    label: string;
+    status?: ArtifactGraphStatus;
+    confidence?: number;
+    binding?: ArtifactGraphBinding;
+    metrics?: Record<string, number>;
+    attrs?: ArtifactGraphAttrs;
+    evidenceRefs?: ArtifactGraphEvidenceRef[];
+}
+export type ArtifactGraphEdgeKind = 'contains' | 'declares' | 'requires' | 'references' | 'defines_workflow' | 'next_step' | 'covers' | 'evaluates' | 'passes' | 'fails' | 'diagnoses' | 'invokes' | 'calls_tool' | 'observes' | 'signals_gap' | 'derived_from';
+export interface ArtifactGraphEdge {
+    id: string;
+    fromNodeId: string;
+    toNodeId: string;
+    edgeKind: ArtifactGraphEdgeKind;
+    layer: ArtifactGraphLayer;
+    label?: string;
+    status?: ArtifactGraphStatus;
+    confidence?: number;
+    weight?: number;
+    binding?: ArtifactGraphBinding;
+    metrics?: Record<string, number>;
+    attrs?: ArtifactGraphAttrs;
+    evidenceRefs?: ArtifactGraphEvidenceRef[];
+}
+export type ArtifactGraphEvidenceSourceKind = 'skill-file' | 'doctor-report' | 'eval-report' | 'observe-report' | 'trace' | 'sample' | 'managed-record';
+export type ArtifactGraphEvidenceSelectorKind = 'json-pointer' | 'line-range' | 'sample-id' | 'rule-id' | 'trace-event-id' | 'node-id';
+export interface ArtifactGraphEvidenceSelector {
+    selectorKind: ArtifactGraphEvidenceSelectorKind;
+    value: string;
+}
+export interface ArtifactGraphEvidenceRef {
+    sourceKind: ArtifactGraphEvidenceSourceKind;
+    sourceId?: string;
+    path?: string;
+    selector?: ArtifactGraphEvidenceSelector;
+    contentHash?: string;
+    label?: string;
+    snippet?: string;
+    redaction?: 'none' | 'truncated' | 'redacted';
+}
+export interface ArtifactGraphSummary {
+    summaryKind: 'structure' | 'coverage' | 'risk' | 'gap' | 'workflow' | 'collection';
+    title: string;
+    severity: 'info' | 'low' | 'medium' | 'high';
+    nodeIds?: string[];
+    edgeIds?: string[];
+    evidenceRefs?: ArtifactGraphEvidenceRef[];
+}

package/dist/types/artifact-graph.js

@@ -0,0 +1 @@
+export {};

package/dist/types/eval.d.ts

@@ -220,6 +220,9 @@ export interface EvalConfig {
     budget?: EvalBudget;
     /** --repeat N. Multi-run variance analysis. */
     repeat?: number;
+    /** --holdout-ratio R (0 < R < 1). Hold out a deterministic sample slice and
+     *  report train vs holdout composite as a generalization / overfitting signal. */
+    holdoutRatio?: number;
     /** --judge-repeat N. Each (sample × dimension) judged N times for self-consistency stddev. */
     judgeRepeat?: number;
     /** --bootstrap. Distribution-free CI per variant + pairwise diff. */
@@ -257,6 +260,10 @@ export interface EvaluationRequest {
     dryRun: boolean;
     /** --repeat N; 1 表示单次跑,> 1 走 runMultiple 做 variance 分析 */
     repeat?: number;
+    /** --holdout-ratio R; 0 / 缺省表示不切分(默认)。> 0 时 report-finalize 在结果上
+     *  post-hoc 切出 train / holdout 子集算综合分(`report.analysis.holdout`),供 verdict
+     *  的过拟合门控读取。see src/eval-core/holdout.ts */
+    holdoutRatio?: number;
     /** --batch; default absent/false. True means skill-batch mode. */
     batch?: boolean;
     /** --judge-repeat N; 每条 sample × dimension 用 LLM judge 跑 N 次, 输出 stddev. 默认 1 (单次). */

package/dist/types/index.d.ts

@@ -6,6 +6,7 @@ export * from './report.js';
 export * from './storage.js';
 export * from './doctor.js';
 export * from './diagnosis.js';
+export * from './artifact-graph.js';
 export * from './dependencies.js';
 export * from './skill-index.js';
 export * from './observability.js';

package/dist/types/index.js

@@ -6,6 +6,7 @@ export * from './report.js';
 export * from './storage.js';
 export * from './doctor.js';
 export * from './diagnosis.js';
+export * from './artifact-graph.js';
 export * from './dependencies.js';
 export * from './skill-index.js';
 export * from './observability.js';

package/dist/types/report.d.ts

@@ -480,6 +480,34 @@ export interface AnalysisResult {
      *  (capability / difficulty / construct / provenance); persisted on report
      *  for studio to surface coverage gaps. See docs/specs/sample-design-spec.md. */
     sampleQuality?: SampleQualityAggregate;
+    /** Opt-in train/holdout generalization breakdown (`omk eval --holdout-ratio`).
+     *  Absent on default runs; present only when a holdout ratio was requested. */
+    holdout?: HoldoutBreakdown;
+}
+/** Train vs holdout composite breakdown for `omk eval --holdout-ratio`.
+ *  Computed post-hoc from `report.results` by `computeHoldoutBreakdown`
+ *  (`src/eval-core/holdout.ts`), sharing the same testSetHash watermark as
+ *  gapReports (gap-spec §7.1). A large train − holdout composite gap is the
+ *  sample-set-overfitting signal the verdict's overfitting gate reads. */
+export interface HoldoutBreakdown {
+    /** Held-out fraction requested via --holdout-ratio. */
+    ratio: number;
+    /** true when either side fell below the minimum subset size → scored full-set,
+     *  no usable split. `perVariant` is empty and the verdict gate stays inert. */
+    disabled?: boolean;
+    /** Per-variant train vs holdout composite (1-5 scale). `*Count` is the authored
+     *  split size; `*Scorable` is how many of those actually produced a composite (> 0)
+     *  — they diverge under partial errors, and the overfitting gate trusts `*Scorable`. */
+    perVariant: Record<string, {
+        trainScore: number;
+        holdoutScore: number;
+        trainCount: number;
+        holdoutCount: number;
+        trainScorable: number;
+        holdoutScorable: number;
+    }>;
+    testSetPath?: string | null;
+    testSetHash?: string | null;
 }
 /** Aggregated sample design coverage stats. Built by
  *  `buildSampleQualityAggregate(samples)` from `Sample.capability` /
@@ -502,6 +530,27 @@ export interface SampleQualityAggregate {
     sampleCountWithDifficulty: number;
     sampleCountWithConstruct: number;
     sampleCountWithProvenance: number;
+    /** Relative-balance / skew of the sample set (derived from the distributions
+     *  above). Flags over-representation — "70% of samples are easy" — without an
+     *  external denominator. Diagnostic only; never feeds grading / judge / verdict. */
+    representativeness?: Representativeness;
+}
+/** Distribution skew over what the sample set declares. Pure relative balance —
+ *  there is no authored "expected" capability list to measure absolute coverage
+ *  against (capabilities are free-form strings), so this reports concentration
+ *  (dominant bucket share, 0-1) and the dominant label per dimension. */
+export interface Representativeness {
+    /** Distinct capabilities declared across the set. */
+    capabilityCount: number;
+    /** Dominant capability's share of all capability tags (0-1); 0 when none declared. */
+    capabilityConcentration: number;
+    dominantCapability?: string;
+    /** Dominant difficulty bucket's share of samples that declared a difficulty (0-1). */
+    difficultyConcentration: number;
+    dominantDifficulty?: 'easy' | 'medium' | 'hard';
+    /** Dominant construct's share of samples that declared a construct (0-1). */
+    constructConcentration: number;
+    dominantConstruct?: string;
 }
 export interface HedgingVerdict {
     isUncertainty: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oh-my-knowledge",
-  "version": "0.41.0",
+  "version": "0.43.0",
   "packageManager": "yarn@4.16.0",
   "description": "Evaluation framework for LLM knowledge inputs — prompts, RAG corpora, skills, agent workflows. Fix the model, vary the artifact. Built-in statistical rigor: bootstrap CI, Krippendorff α, length-debias, saturation curves.",
   "type": "module",