oh-my-knowledge 0.37.0 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh.md +1 -1
- package/dist/assets/agent-skills/omk/SKILL.md +1 -1
- package/dist/assets/agent-skills/omk/references/commands.md +21 -12
- package/dist/authoring/evolver.d.ts +4 -1
- package/dist/authoring/evolver.js +13 -8
- package/dist/cli/commands/doctor.d.ts +1 -0
- package/dist/cli/commands/doctor.js +26 -6
- package/dist/cli/commands/eval/gold/compare.js +7 -4
- package/dist/cli/commands/eval/index.d.ts +1 -0
- package/dist/cli/commands/eval/index.js +14 -3
- package/dist/cli/commands/evolve.d.ts +1 -0
- package/dist/cli/commands/evolve.js +45 -0
- package/dist/cli/commands/init.js +18 -6
- package/dist/cli/commands/list.d.ts +0 -2
- package/dist/cli/commands/list.js +2 -7
- package/dist/cli/commands/observe/inbox.d.ts +1 -0
- package/dist/cli/commands/observe/inbox.js +14 -4
- package/dist/cli/commands/observe/index.d.ts +11 -0
- package/dist/cli/commands/observe/index.js +35 -12
- package/dist/cli/commands/observe/ingest.d.ts +1 -0
- package/dist/cli/commands/observe/ingest.js +13 -4
- package/dist/cli/commands/observe/show.d.ts +1 -0
- package/dist/cli/commands/observe/show.js +12 -2
- package/dist/cli/commands/promote.js +1 -2
- package/dist/cli/commands/sample.js +15 -9
- package/dist/cli/commands/studio.d.ts +2 -0
- package/dist/cli/commands/studio.js +62 -12
- package/dist/cli/lib/cmd-flags.d.ts +8 -0
- package/dist/cli/lib/i18n-dict/common.js +1 -1
- package/dist/cli/lib/i18n-dict/evolve.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/evolve.js +8 -0
- package/dist/cli/lib/i18n-dict/help.js +27 -45
- package/dist/cli/lib/i18n-dict/init.js +2 -2
- package/dist/cli/lib/parse-run-config.d.ts +0 -1
- package/dist/cli/lib/parse-run-config.js +6 -4
- package/dist/cli/lib/record-evolve-outcome.d.ts +31 -0
- package/dist/cli/lib/record-evolve-outcome.js +110 -0
- package/dist/cli/lib/resolve-skill-input.d.ts +6 -0
- package/dist/cli/lib/resolve-skill-input.js +4 -2
- package/dist/diagnosis/types.js +1 -1
- package/dist/doctor/index.js +4 -1
- package/dist/eval-core/artifact-index.d.ts +106 -0
- package/dist/eval-core/artifact-index.js +277 -0
- package/dist/eval-core/default-dirs.d.ts +34 -0
- package/dist/eval-core/default-dirs.js +36 -0
- package/dist/eval-core/evaluation-execution.js +3 -3
- package/dist/eval-core/evaluation-reporting.d.ts +7 -0
- package/dist/eval-core/evaluation-reporting.js +20 -5
- package/dist/eval-core/execution-strategy.js +3 -3
- package/dist/eval-core/measurement-dirs.d.ts +18 -0
- package/dist/eval-core/measurement-dirs.js +76 -0
- package/dist/eval-workflows/batch-evaluation-workflow.js +2 -1
- package/dist/eval-workflows/evaluation-pipeline/run-state.js +2 -1
- package/dist/eval-workflows/run-evaluation.js +5 -2
- package/dist/executors/codex-cli.js +2 -2
- package/dist/executors/script.js +43 -4
- package/dist/inputs/materialize-copy.d.ts +1 -1
- package/dist/inputs/materialize-copy.js +3 -3
- package/dist/inputs/skill-loader.d.ts +1 -1
- package/dist/inputs/skill-loader.js +1 -1
- package/dist/managed/index.d.ts +2 -0
- package/dist/managed/index.js +2 -0
- package/dist/managed/list-query.d.ts +7 -0
- package/dist/managed/list-query.js +11 -0
- package/dist/managed/source-probe.d.ts +15 -0
- package/dist/managed/source-probe.js +129 -0
- package/dist/managed/store.d.ts +10 -0
- package/dist/managed/store.js +25 -2
- package/dist/observability/inbox.js +6 -3
- package/dist/observability/skill-health-analyzer.d.ts +3 -0
- package/dist/observability/skill-health-analyzer.js +2 -0
- package/dist/renderer/html-renderer.js +4 -2
- package/dist/renderer/managed-history-renderer.d.ts +5 -0
- package/dist/renderer/managed-history-renderer.js +255 -0
- package/dist/renderer/observation-inbox-renderer.js +19 -19
- package/dist/renderer/skill-detail-renderer.js +17 -8
- package/dist/renderer/skill-list-renderer.js +1 -0
- package/dist/server/indexed-report-store.d.ts +8 -0
- package/dist/server/indexed-report-store.js +130 -0
- package/dist/server/job-store.d.ts +0 -1
- package/dist/server/job-store.js +0 -2
- package/dist/server/report-server.d.ts +17 -3
- package/dist/server/report-server.js +231 -85
- package/dist/server/report-store.d.ts +14 -8
- package/dist/server/report-store.js +46 -0
- package/dist/server/skill-index.d.ts +7 -1
- package/dist/server/skill-index.js +48 -12
- package/dist/types/executor.d.ts +4 -2
- package/dist/types/report.d.ts +15 -0
- package/dist/types/skill-index.d.ts +3 -0
- package/package.json +1 -1
|
@@ -1,13 +1,10 @@
|
|
|
1
1
|
export const helpDict = {
|
|
2
2
|
'cli.help.observe': {
|
|
3
3
|
zh: `
|
|
4
|
-
omk observe——分析真实 session trace,生成 skill
|
|
4
|
+
omk observe——分析真实 session trace,生成 skill 健康度报告
|
|
5
5
|
|
|
6
6
|
用法:
|
|
7
7
|
omk observe <sessions-dir> [options]
|
|
8
|
-
omk observe ingest <sessions-dir> [options]
|
|
9
|
-
omk observe inbox [options]
|
|
10
|
-
omk observe show <inbox_id> [options]
|
|
11
8
|
|
|
12
9
|
选项:
|
|
13
10
|
--kb <path> 知识库根路径(默认:从 trace cwd 推断)
|
|
@@ -15,27 +12,16 @@ omk observe——分析真实 session trace,生成 skill 健康度日报
|
|
|
15
12
|
--from <iso> 窗口起点,优先级高于 --last
|
|
16
13
|
--to <iso> 窗口终点,优先级高于 --last
|
|
17
14
|
--skills <n1,n2,...> 只分析指定 skill
|
|
18
|
-
--output-dir <path>
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
inbox --input-dir <path> 读取 observe inbox 数据
|
|
23
|
-
inbox --limit <n> 展示 top N(默认:20)
|
|
24
|
-
inbox --skill <name> 只看指定 skill
|
|
25
|
-
inbox --explore <n> 从最近 50 条 medium/low 问题里抽样查看长尾
|
|
26
|
-
inbox --include-noise --explore 时显式包含 noise 桶
|
|
27
|
-
inbox --by-skill 按 skill 输出资产看板
|
|
28
|
-
inbox --json 输出 JSON
|
|
29
|
-
show <inbox_id> --input-dir <path> 查看单条 observation 的前后上下文
|
|
15
|
+
--output-dir <path> 输出目录(默认:项目级 .omk/observe-health)
|
|
16
|
+
--global 写全局 ~/.oh-my-knowledge/observe-health,而非项目 .omk/
|
|
17
|
+
|
|
18
|
+
观测收件箱(observe inbox)是另一条线,见 omk observe inbox --help。
|
|
30
19
|
`,
|
|
31
20
|
en: `
|
|
32
21
|
omk observe — analyze production session traces and produce skill health reports
|
|
33
22
|
|
|
34
23
|
Usage:
|
|
35
24
|
omk observe <sessions-dir> [options]
|
|
36
|
-
omk observe ingest <sessions-dir> [options]
|
|
37
|
-
omk observe inbox [options]
|
|
38
|
-
omk observe show <inbox_id> [options]
|
|
39
25
|
|
|
40
26
|
Options:
|
|
41
27
|
--kb <path> Knowledge base root (default: infer from trace cwd)
|
|
@@ -43,18 +29,10 @@ Options:
|
|
|
43
29
|
--from <iso> Window start, overrides --last
|
|
44
30
|
--to <iso> Window end, overrides --last
|
|
45
31
|
--skills <n1,n2,...> Only analyze selected skills
|
|
46
|
-
--output-dir <path> Output directory (default:
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
inbox --input-dir <path> Read observe inbox data
|
|
51
|
-
inbox --limit <n> Show top N (default: 20)
|
|
52
|
-
inbox --skill <name> Only show one skill
|
|
53
|
-
inbox --explore <n> Sample long-tail issues from the latest 50 medium/low items
|
|
54
|
-
inbox --include-noise Explicitly include the noise bucket with --explore
|
|
55
|
-
inbox --by-skill Print the skill-level asset board
|
|
56
|
-
inbox --json Print JSON
|
|
57
|
-
show <inbox_id> --input-dir <path> Show context around one observation
|
|
32
|
+
--output-dir <path> Output directory (default: project-level .omk/observe-health)
|
|
33
|
+
--global Write to global ~/.oh-my-knowledge/observe-health instead of project .omk/
|
|
34
|
+
|
|
35
|
+
The observe inbox is a separate line; see omk observe inbox --help.
|
|
58
36
|
`,
|
|
59
37
|
},
|
|
60
38
|
'cli.help.observe_ingest': {
|
|
@@ -65,7 +43,7 @@ omk observe ingest — 读取真实 session trace,写入 observe inbox 数据
|
|
|
65
43
|
omk observe ingest <sessions-dir-or-file> [options]
|
|
66
44
|
|
|
67
45
|
选项:
|
|
68
|
-
--output-dir <path> 输出目录(默认:.omk/
|
|
46
|
+
--output-dir <path> 输出目录(默认:.omk/observe-inbox;读取时兜底到 ~/.oh-my-knowledge/observe-inbox)
|
|
69
47
|
|
|
70
48
|
支持:
|
|
71
49
|
Claude Code JSONL
|
|
@@ -78,7 +56,7 @@ Usage:
|
|
|
78
56
|
omk observe ingest <sessions-dir-or-file> [options]
|
|
79
57
|
|
|
80
58
|
Options:
|
|
81
|
-
--output-dir <path> Output directory (default: .omk/
|
|
59
|
+
--output-dir <path> Output directory (default: .omk/observe-inbox; read fallback: ~/.oh-my-knowledge/observe-inbox)
|
|
82
60
|
|
|
83
61
|
Supported:
|
|
84
62
|
Claude Code JSONL
|
|
@@ -93,7 +71,7 @@ omk observe inbox — 查看已写入的 observe inbox 问题列表
|
|
|
93
71
|
omk observe inbox [options]
|
|
94
72
|
|
|
95
73
|
选项:
|
|
96
|
-
--input-dir <path> 读取目录(默认:.omk/
|
|
74
|
+
--input-dir <path> 读取目录(默认:.omk/observe-inbox;兜底到 ~/.oh-my-knowledge/observe-inbox)
|
|
97
75
|
--limit <n> 展示 top N(默认:20)
|
|
98
76
|
--skill <name> 只看指定 skill
|
|
99
77
|
--explore <n> 从最近 50 条 medium/low 问题里抽样查看长尾
|
|
@@ -112,7 +90,7 @@ Usage:
|
|
|
112
90
|
omk observe inbox [options]
|
|
113
91
|
|
|
114
92
|
Options:
|
|
115
|
-
--input-dir <path> Input directory (default: .omk/
|
|
93
|
+
--input-dir <path> Input directory (default: .omk/observe-inbox; fallback: ~/.oh-my-knowledge/observe-inbox)
|
|
116
94
|
--limit <n> Show top N (default: 20)
|
|
117
95
|
--skill <name> Only show one skill
|
|
118
96
|
--explore <n> Sample long-tail issues from the latest 50 medium/low items
|
|
@@ -133,7 +111,7 @@ omk observe show — 查看单条 observation 的原始上下文
|
|
|
133
111
|
omk observe show <inbox_id> [options]
|
|
134
112
|
|
|
135
113
|
选项:
|
|
136
|
-
--input-dir <path> 读取目录(默认:.omk/
|
|
114
|
+
--input-dir <path> 读取目录(默认:.omk/observe-inbox;兜底到 ~/.oh-my-knowledge/observe-inbox)
|
|
137
115
|
`,
|
|
138
116
|
en: `
|
|
139
117
|
omk observe show — inspect the raw context around one observation
|
|
@@ -142,7 +120,7 @@ Usage:
|
|
|
142
120
|
omk observe show <inbox_id> [options]
|
|
143
121
|
|
|
144
122
|
Options:
|
|
145
|
-
--input-dir <path> Input directory (default: .omk/
|
|
123
|
+
--input-dir <path> Input directory (default: .omk/observe-inbox; fallback: ~/.oh-my-knowledge/observe-inbox)
|
|
146
124
|
`,
|
|
147
125
|
},
|
|
148
126
|
'cli.help.evolve': {
|
|
@@ -243,16 +221,18 @@ omk studio——打开本地知识工作台
|
|
|
243
221
|
选项:
|
|
244
222
|
--port <n> 本地服务端口(默认:7799)
|
|
245
223
|
--host <host> 监听地址(默认:127.0.0.1;局域网访问可用 0.0.0.0)
|
|
246
|
-
--reports-dir <path>
|
|
247
|
-
--analyses-dir <path>
|
|
248
|
-
--
|
|
224
|
+
--reports-dir <path> 只看指定报告目录(默认机器级聚合:当前项目 + 全局 + 别项目索引)
|
|
225
|
+
--analyses-dir <path> 观测健康报告目录(默认:项目级 .omk/observe-health,空则全局兜底)
|
|
226
|
+
--doctors-dir <path> 体检报告目录(默认:项目级 .omk/doctors,空则全局兜底)
|
|
227
|
+
--observations-dir <path> observe inbox 数据目录(默认:.omk/observe-inbox)
|
|
228
|
+
--global 只看全局 reports / observe-health / doctors 目录(~/.oh-my-knowledge/*),而非机器级聚合 / 项目优先;managed / observe-inbox 不受影响
|
|
249
229
|
--no-open 只启动服务,不自动打开浏览器
|
|
250
230
|
--dev 开发模式:文件变化时自动重启
|
|
251
231
|
|
|
252
232
|
示例:
|
|
253
233
|
omk studio
|
|
254
234
|
omk studio --port 7798
|
|
255
|
-
omk studio --host 0.0.0.0 --observations-dir .omk/
|
|
235
|
+
omk studio --host 0.0.0.0 --observations-dir .omk/observe-inbox
|
|
256
236
|
omk studio --no-open
|
|
257
237
|
`,
|
|
258
238
|
en: `
|
|
@@ -264,16 +244,18 @@ Usage:
|
|
|
264
244
|
Options:
|
|
265
245
|
--port <n> Local server port (default: 7799)
|
|
266
246
|
--host <host> Listen address (default: 127.0.0.1; use 0.0.0.0 for LAN access)
|
|
267
|
-
--reports-dir <path>
|
|
268
|
-
--analyses-dir <path>
|
|
269
|
-
--
|
|
247
|
+
--reports-dir <path> View only this reports dir (default aggregates machine-wide: current project + global + other projects via index)
|
|
248
|
+
--analyses-dir <path> Observe-health reports dir (default: project .omk/observe-health, falls back to global)
|
|
249
|
+
--doctors-dir <path> Doctor reports dir (default: project .omk/doctors, falls back to global)
|
|
250
|
+
--observations-dir <path> Observe inbox data directory (default: .omk/observe-inbox)
|
|
251
|
+
--global View only global reports / observe-health / doctors dirs (~/.oh-my-knowledge/*) instead of machine-wide / project-first; does not affect managed / observe-inbox
|
|
270
252
|
--no-open Start the server without opening a browser
|
|
271
253
|
--dev Dev mode: restart on file changes
|
|
272
254
|
|
|
273
255
|
Examples:
|
|
274
256
|
omk studio
|
|
275
257
|
omk studio --port 7798
|
|
276
|
-
omk studio --host 0.0.0.0 --observations-dir .omk/
|
|
258
|
+
omk studio --host 0.0.0.0 --observations-dir .omk/observe-inbox
|
|
277
259
|
omk studio --no-open
|
|
278
260
|
`,
|
|
279
261
|
},
|
|
@@ -83,7 +83,6 @@ export interface ParseRunConfigResult {
|
|
|
83
83
|
* (e.g. repeat / judgeRepeat / bootstrap — handled in `commands/eval-runner.ts` for input validation). */
|
|
84
84
|
evalConfig: EvalConfig | null;
|
|
85
85
|
}
|
|
86
|
-
export declare const DEFAULT_REPORTS_DIR: string;
|
|
87
86
|
/**
|
|
88
87
|
* 接 typed flags(来自 oclif Command.parse() 输出)。oclif strict 模式已经在
|
|
89
88
|
* 上游对未知 flag 拦截 exit 2,这里不再 parseArgs。eval-runner 等业务 caller
|
|
@@ -17,15 +17,14 @@
|
|
|
17
17
|
* `cli ?? evalConfig ?? default`,刻意不再细拆,否则只是把一长串 ?? 散到多文件,
|
|
18
18
|
* 反而失去「精度链一目了然」的可读性。
|
|
19
19
|
*/
|
|
20
|
-
import { resolve
|
|
21
|
-
import {
|
|
20
|
+
import { resolve } from 'node:path';
|
|
21
|
+
import { projectReportsDir, globalReportsDir } from '../../eval-core/measurement-dirs.js';
|
|
22
22
|
import { loadEvalConfig } from '../../inputs/eval-config.js';
|
|
23
23
|
import { DEFAULT_MODEL } from '../../executors/shared.js';
|
|
24
24
|
import { parseJudgeModelsArgOrExit } from './parse-run-config/judge-models.js';
|
|
25
25
|
import { discoverSamplesPath } from './parse-run-config/samples-discovery.js';
|
|
26
26
|
import { resolveVariantSpecs } from './parse-run-config/variant-resolution.js';
|
|
27
27
|
export { parseJudgeModelsArg, parseJudgeModelsArgOrExit } from './parse-run-config/judge-models.js';
|
|
28
|
-
export const DEFAULT_REPORTS_DIR = join(homedir(), '.oh-my-knowledge', 'reports');
|
|
29
28
|
/**
|
|
30
29
|
* 接 typed flags(来自 oclif Command.parse() 输出)。oclif strict 模式已经在
|
|
31
30
|
* 上游对未知 flag 拦截 exit 2,这里不再 parseArgs。eval-runner 等业务 caller
|
|
@@ -72,7 +71,10 @@ export function parseRunConfig(values) {
|
|
|
72
71
|
const judgeModels = parsedJudges
|
|
73
72
|
?? evalConfig?.judgeModels
|
|
74
73
|
?? [{ executor: executorName, model: 'haiku' }];
|
|
75
|
-
|
|
74
|
+
// 报告默认落项目 `.omk/reports`(绑用例集,construct validity);--global 写全局;--output-dir 最高优先。
|
|
75
|
+
// 同 observe / doctor 写入侧口径。读取侧(studio / resume / gold-compare / 复用)走 overlay 项目→全局兜底。
|
|
76
|
+
const outputDir = resolve(values['output-dir']
|
|
77
|
+
?? (values.global ? globalReportsDir() : projectReportsDir()));
|
|
76
78
|
const concurrencyRaw = values.concurrency !== undefined
|
|
77
79
|
? Number(values.concurrency)
|
|
78
80
|
: evalConfig?.concurrency ?? 1;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { EvaluationReport } from '../../types/index.js';
|
|
2
|
+
export interface EvolveOutcomeInput {
|
|
3
|
+
/** 合并 evolve 报告 id(evolveSkill 返回的 reportId)。 */
|
|
4
|
+
reportId?: string;
|
|
5
|
+
/** 胜出轮(0 = 无改进,不记)。 */
|
|
6
|
+
bestRound: number;
|
|
7
|
+
/** resolveSkillInput 给的 skillPath(文件-skill 为 .md、目录-skill 为内部 SKILL.md)。 */
|
|
8
|
+
skillPath: string;
|
|
9
|
+
/** resolveSkillInput 给的 skillDir。 */
|
|
10
|
+
skillDir: string;
|
|
11
|
+
/** evolve 目标本身的形态(目录-skill / 文件-skill),由原始入参 statSync 判定。用于按形态精确匹配受管记录。 */
|
|
12
|
+
isDirectorySkill: boolean;
|
|
13
|
+
/** 报告加载器覆盖(测试用);默认走 overlay 报告存储(项目 .omk/reports → 全局兜底)按 id 读。 */
|
|
14
|
+
loadReport?: (id: string) => Promise<EvaluationReport | null>;
|
|
15
|
+
/** 受管目录覆盖(测试用)。 */
|
|
16
|
+
dir?: string;
|
|
17
|
+
}
|
|
18
|
+
export interface EvolveOutcomeResult {
|
|
19
|
+
name: string;
|
|
20
|
+
contentHash: string;
|
|
21
|
+
verdict: string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* evolve 把胜出版本写回 source 后,为受管 skill 记一条带 verdict 的证据,并把记录 re-baseline 到新版本,
|
|
25
|
+
* 让 `omk list` 显 measurable(而非永久 stale)。**不写 promote 决定** —— 升 promoted 仍由人 `omk promote`(A 方案:
|
|
26
|
+
* 统计门 ≠ 人的生产接受)。未纳管 / 无改进 / 源不可达 / 报告缺失 → no-op 返回 null。
|
|
27
|
+
*
|
|
28
|
+
* 证据 contentHash 统一锚到 re-baseline 后的源哈(probeSourceState 与 install / list 同口径),规避「文件-skill 哈
|
|
29
|
+
* vs 目录-skill 整树哈」可能与报告内 artifactHash 不在同空间的边角,保证证据判为「当前」。
|
|
30
|
+
*/
|
|
31
|
+
export declare function recordEvolveOutcome(input: EvolveOutcomeInput): Promise<EvolveOutcomeResult | null>;
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
2
|
+
import { createOverlayReportStore } from '../../server/report-store.js';
|
|
3
|
+
import { projectReportsDir, globalReportsDir } from '../../eval-core/measurement-dirs.js';
|
|
4
|
+
import { computeVerdict } from '../../eval-core/verdict.js';
|
|
5
|
+
import { bootstrapDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../../eval-core/bootstrap.js';
|
|
6
|
+
import { resolveManagedDir, managedDir, loadAllManagedRecords, appendManagedEvidence, rebaselineManagedContentHash, buildEvidenceRef, probeSourceState, } from '../../managed/index.js';
|
|
7
|
+
/**
|
|
8
|
+
* round-bestRound vs round-0 的忠实 verdict:复刻 eval 管线(evaluation-reporting.ts)对两变体抽
|
|
9
|
+
* per-sample composite → bootstrapDiffCI → computeVerdict,与 `omk eval --bootstrap` 同口径、同 α / 重采样数。
|
|
10
|
+
* 不自造门限:评委是否显著、是否 PROGRESS 全交给既有 computeVerdict。
|
|
11
|
+
*/
|
|
12
|
+
function winnerVerdict(report, winnerVariant) {
|
|
13
|
+
const baseline = 'round-0';
|
|
14
|
+
const scoresOf = (v) => report.results
|
|
15
|
+
.map((r) => r.variants[v])
|
|
16
|
+
.filter((e) => !!e && typeof e.compositeScore === 'number' && e.compositeScore > 0)
|
|
17
|
+
.map((e) => e.compositeScore);
|
|
18
|
+
const ctrl = scoresOf(baseline);
|
|
19
|
+
const treat = scoresOf(winnerVariant);
|
|
20
|
+
const pairComparisons = ctrl.length >= 2 && treat.length >= 2
|
|
21
|
+
? [{
|
|
22
|
+
control: baseline,
|
|
23
|
+
treatment: winnerVariant,
|
|
24
|
+
diffBootstrapCI: bootstrapDiffCI(ctrl, treat, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES),
|
|
25
|
+
}]
|
|
26
|
+
: undefined;
|
|
27
|
+
const slice = {
|
|
28
|
+
...report,
|
|
29
|
+
meta: {
|
|
30
|
+
...report.meta,
|
|
31
|
+
variants: [baseline, winnerVariant],
|
|
32
|
+
...(pairComparisons ? { pairComparisons } : {}),
|
|
33
|
+
},
|
|
34
|
+
summary: {
|
|
35
|
+
[baseline]: report.summary[baseline],
|
|
36
|
+
[winnerVariant]: report.summary[winnerVariant],
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
return computeVerdict(slice).level;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* 受管目录里按源路径定位被 evolve 的那条记录。**形态感知**,且对目录-skill 的两种 install 落盘形态都认:
|
|
43
|
+
* - 目录-skill(`isDirectorySkill=true`):install 既可能落「目录形态」(`source.isDirectorySkill=true`、
|
|
44
|
+
* locator=skill 根目录 = skillDir),也可能因用户 `install <dir>/SKILL.md` 落「文件形态」
|
|
45
|
+
* (`isDirectorySkill=false`、locator=该 SKILL.md = skillPath)。两者都对应同一个 skill,故都匹配。
|
|
46
|
+
* - 扁平文件-skill(`isDirectorySkill=false`):只认「文件形态 + locator=该 .md 本身(skillPath)」。
|
|
47
|
+
*
|
|
48
|
+
* 绝不拿父目录 skillDir 去撞目录-skill 记录:对散文件 skill(`skills/foo.md`)跑 evolve 时,skillDir = `skills/`,
|
|
49
|
+
* 若并集匹配会误命中一条 locator 恰为 `skills/` 的目录-skill 记录,把证据 / re-baseline 写到根本没被 evolve 的
|
|
50
|
+
* 记录上。这里每条匹配目标(skillDir / skillPath)都唯一指向被 evolve 的 skill 自身,无此越权。
|
|
51
|
+
*/
|
|
52
|
+
function findRecordBySource(records, skillPath, skillDir, isDirectorySkill) {
|
|
53
|
+
const fileTarget = resolve(skillPath);
|
|
54
|
+
const dirTarget = resolve(skillDir);
|
|
55
|
+
return records.find((r) => {
|
|
56
|
+
if (r.source.sourceKind !== 'file')
|
|
57
|
+
return false;
|
|
58
|
+
const loc = resolve(r.source.locator);
|
|
59
|
+
if (isDirectorySkill) {
|
|
60
|
+
return (r.source.isDirectorySkill && loc === dirTarget) // 目录形态记录:locator = skill 根目录
|
|
61
|
+
|| (!r.source.isDirectorySkill && loc === fileTarget); // 文件形态记录(install <dir>/SKILL.md):locator = 该 SKILL.md
|
|
62
|
+
}
|
|
63
|
+
return !r.source.isDirectorySkill && loc === fileTarget; // 扁平文件-skill:只认该 .md 本身,绝不撞父目录
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
async function loadEvolveReport(id) {
|
|
67
|
+
// overlay get:项目 .omk/reports → 全局兜底,evolve 写到全局的合并报告仍命中,eval 新写项目的也能取到。
|
|
68
|
+
const doc = await createOverlayReportStore(projectReportsDir(), globalReportsDir()).get(id);
|
|
69
|
+
return doc && doc.kind === 'evaluation' ? doc : null;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* evolve 把胜出版本写回 source 后,为受管 skill 记一条带 verdict 的证据,并把记录 re-baseline 到新版本,
|
|
73
|
+
* 让 `omk list` 显 measurable(而非永久 stale)。**不写 promote 决定** —— 升 promoted 仍由人 `omk promote`(A 方案:
|
|
74
|
+
* 统计门 ≠ 人的生产接受)。未纳管 / 无改进 / 源不可达 / 报告缺失 → no-op 返回 null。
|
|
75
|
+
*
|
|
76
|
+
* 证据 contentHash 统一锚到 re-baseline 后的源哈(probeSourceState 与 install / list 同口径),规避「文件-skill 哈
|
|
77
|
+
* vs 目录-skill 整树哈」可能与报告内 artifactHash 不在同空间的边角,保证证据判为「当前」。
|
|
78
|
+
*/
|
|
79
|
+
export async function recordEvolveOutcome(input) {
|
|
80
|
+
if (input.bestRound <= 0 || !input.reportId)
|
|
81
|
+
return null;
|
|
82
|
+
const dir = input.dir ?? resolveManagedDir(managedDir());
|
|
83
|
+
const record = findRecordBySource(loadAllManagedRecords(dir), input.skillPath, input.skillDir, input.isDirectorySkill);
|
|
84
|
+
if (!record)
|
|
85
|
+
return null; // 未纳管 → no-op(管理是 install 显式 opt-in)
|
|
86
|
+
const probe = probeSourceState(record);
|
|
87
|
+
if (!probe.reachable || probe.hash === undefined)
|
|
88
|
+
return null; // 源不可达 → 不动
|
|
89
|
+
const newHash = probe.hash;
|
|
90
|
+
const report = input.loadReport ? await input.loadReport(input.reportId) : await loadEvolveReport(input.reportId);
|
|
91
|
+
if (!report || report.kind !== 'evaluation')
|
|
92
|
+
return null;
|
|
93
|
+
const winnerVariant = `round-${input.bestRound}`;
|
|
94
|
+
// 防御:正常 evolve 产物必含基线 round-0 与胜出轮的 summary(同一次 run 产出 bestRound 与合并报告),此守卫只挡
|
|
95
|
+
// 报告损坏 / 被截断的边角 —— 缺任一则无可信 verdict 可算,直接 no-op,避免 winnerVerdict→computeVerdict 读 undefined。
|
|
96
|
+
if (!report.summary?.['round-0'] || !report.summary?.[winnerVariant])
|
|
97
|
+
return null;
|
|
98
|
+
const ref = buildEvidenceRef(report, winnerVariant, winnerVerdict(report, winnerVariant), new Date().toISOString());
|
|
99
|
+
if (!ref)
|
|
100
|
+
return null;
|
|
101
|
+
const evidence = { ...ref, contentHash: newHash };
|
|
102
|
+
// 用 store 写函数的返回值判定:记录在 find 与写之间被删/改(TOCTOU)→ 两次写 no-op 返回 null → 不回报成功,
|
|
103
|
+
// 避免 CLI 打印「已记证据」而盘上其实没写。name/contentHash 取自 append 后的 merged 记录,不用旧 in-memory。
|
|
104
|
+
if (!rebaselineManagedContentHash(dir, record.id, newHash))
|
|
105
|
+
return null;
|
|
106
|
+
const merged = appendManagedEvidence(dir, record.id, evidence);
|
|
107
|
+
if (!merged)
|
|
108
|
+
return null;
|
|
109
|
+
return { name: merged.name, contentHash: merged.contentHash, verdict: evidence.verdict ?? 'UNKNOWN' };
|
|
110
|
+
}
|
|
@@ -3,5 +3,11 @@ export interface ResolvedSkillInput {
|
|
|
3
3
|
skillPath: string;
|
|
4
4
|
skillDir: string;
|
|
5
5
|
samplesPath: string;
|
|
6
|
+
/**
|
|
7
|
+
* 目标是否为目录-skill。**只看解析后形态、不看入参写法**:传目录 `skills/foo` 或传内部
|
|
8
|
+
* `skills/foo/SKILL.md` 都判 true(两者 skillPath 都落到 `.../SKILL.md`),扁平 `bar.md` 判 false。
|
|
9
|
+
* 受管联动按此对齐 install 落的记录形态(`source.isDirectorySkill`),避免「传 SKILL.md 文件路径就匹配不到目录记录」。
|
|
10
|
+
*/
|
|
11
|
+
isDirectorySkill: boolean;
|
|
6
12
|
}
|
|
7
13
|
export declare function resolveSkillInput(input: string, lang: CliLang): ResolvedSkillInput;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { resolve, join, dirname } from 'node:path';
|
|
1
|
+
import { resolve, join, dirname, basename } from 'node:path';
|
|
2
2
|
import { existsSync, statSync } from 'node:fs';
|
|
3
3
|
import { tCli } from './i18n.js';
|
|
4
4
|
// 统一 skill 入参解析:既接受 SKILL.md 文件(老式 + flat skill),也接受 directory-skill
|
|
@@ -35,5 +35,7 @@ export function resolveSkillInput(input, lang) {
|
|
|
35
35
|
];
|
|
36
36
|
// fallback 到 .omk/ 目录:上游会报 "no sample files found in directory" 引导用户创建
|
|
37
37
|
const samplesPath = candidates.find(existsSync) ?? omkDir;
|
|
38
|
-
|
|
38
|
+
// 形态以解析后的 skillPath 命名为准:目录-skill 的 skillPath 总是 `.../SKILL.md`。
|
|
39
|
+
const isDirectorySkill = basename(skillPath) === 'SKILL.md';
|
|
40
|
+
return { skillPath, skillDir, samplesPath, isDirectorySkill };
|
|
39
41
|
}
|
package/dist/diagnosis/types.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* confirmed:目前 mapper 不产出,如果将来 producer / review-state 写出,会被一并算 inactive
|
|
7
7
|
* —— 跟 confirmed soft standard 的「已被认知、进入处理流程」语义一致。
|
|
8
8
|
*
|
|
9
|
-
* 抽这个 helper 是为了:Insight 投影(影响 skill 健康 / 待优化数)和 /api/
|
|
9
|
+
* 抽这个 helper 是为了:Insight 投影(影响 skill 健康 / 待优化数)和 /api/observe-inbox/diagnostics
|
|
10
10
|
* 的 active 列表共用同一份口径,避免「Insight 把 confirmed 算 active 但 API 不算」的口径分叉。
|
|
11
11
|
*/
|
|
12
12
|
const ACTIVE_DIAGNOSIS_LIFECYCLES = new Set([
|
package/dist/doctor/index.js
CHANGED
|
@@ -167,7 +167,10 @@ let reportIdCounter = 0;
|
|
|
167
167
|
function nextReportId() {
|
|
168
168
|
reportIdCounter += 1;
|
|
169
169
|
const ts = new Date().toISOString().replace(/[-:.]/g, '').slice(0, 15);
|
|
170
|
-
|
|
170
|
+
// 进程内计数器只防同进程同秒撞;跨进程同秒(两个 omk doctor 并发 / 不同项目)仍会撞同 id,
|
|
171
|
+
// 经机器级卡片 dedup 当唯一键用时会静默并掉一份。加 4 位随机根治跨进程撞名(id 是标签非测量数)。
|
|
172
|
+
const rand = Math.random().toString(36).slice(2, 6);
|
|
173
|
+
return `doctor-${ts}-${reportIdCounter}-${rand}`;
|
|
171
174
|
}
|
|
172
175
|
function readCliVersion() {
|
|
173
176
|
// 不依赖 package.json import(避免 type 解析复杂度);用环境变量或退回 'unknown'
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { ReportDocument, ReportIndexCard } from '../types/index.js';
|
|
2
|
+
import type { SkillDoctorSnapshot } from '../types/skill-index.js';
|
|
3
|
+
import type { DoctorSkillStatus } from '../types/doctor.js';
|
|
4
|
+
export type ArtifactDomain = 'report' | 'doctor' | 'observe-health';
|
|
5
|
+
/** 某域的索引目录 `<root>/<domain>/`。 */
|
|
6
|
+
export declare function artifactIndexDir(domain: ArtifactDomain): string;
|
|
7
|
+
/** report 域口径(保持向后兼容的既有导出)。 */
|
|
8
|
+
export declare function shouldIndexReport(outputDir: string): boolean;
|
|
9
|
+
/**
|
|
10
|
+
* 某域所有卡片「真身(card.path)」的存在性 + mtime/size 指纹。
|
|
11
|
+
* 消费方(indexed-report-store / buildSkillIndex)的缓存指纹必须纳入它:卡片 JSON 本身没变、只靠卡片目录
|
|
12
|
+
* mtime 的指纹检测不到「真身被带外删」,长会话会继续命中旧缓存展示悬空卡片。真身删/改 → 此 sentinel 变 →
|
|
13
|
+
* 缓存失效 → live 过滤生效。读卡片只为拿 path,字段宽松。
|
|
14
|
+
*/
|
|
15
|
+
export declare function cardTargetSentinel(domain: ArtifactDomain): string;
|
|
16
|
+
/**
|
|
17
|
+
* persistReport 的索引钩子:报告落盘后 best-effort 追加卡片。永不抛、永不阻断报告落盘。
|
|
18
|
+
* 入参故意收 `{id}` 宽松(同 persistReport 的 PersistableReport):非完整报告(无 canonical kind)防御式跳过。
|
|
19
|
+
*/
|
|
20
|
+
export declare function indexReportWrite(report: {
|
|
21
|
+
id: string;
|
|
22
|
+
}, sourcePath: string, outputDir: string): void;
|
|
23
|
+
/** 读 report 域全部卡片(跳过坏文件 / 缺字段 / 坏 kind)。 */
|
|
24
|
+
export declare function listReportCards(): ReportIndexCard[];
|
|
25
|
+
/** report 卡片(过滤悬空真身):供 studio 机器级 list / findBy 展示。 */
|
|
26
|
+
export declare function listLiveReportCards(): ReportIndexCard[];
|
|
27
|
+
/** 卡片 → ReportDocument(results:[]):供 studio list / buildSkillIndex / trend 消费(它们不读 results)。 */
|
|
28
|
+
export declare function cardToReportDocument(card: ReportIndexCard): ReportDocument;
|
|
29
|
+
/** 删 report 域某 id 的卡片(DELETE 报告时连卡片一起删,使其从机器级 list 消失)。幂等、best-effort。 */
|
|
30
|
+
export declare function removeReportCard(id: string): boolean;
|
|
31
|
+
/** doctor 卡片:per-skill 文件(`{skillName}-{reportId}.json`)一张。id = 文件名 stem(唯一),
|
|
32
|
+
* reportId = 该次 doctor run 的 id(同 run 多 skill 共享)。剥掉逐规则 results 重体,只留 per-skill 计数。
|
|
33
|
+
* 无 `kind` 字段 —— domain 已足够判别(doctor 只有一种报告 kind)。 */
|
|
34
|
+
export interface DoctorIndexCard {
|
|
35
|
+
domain: 'doctor';
|
|
36
|
+
id: string;
|
|
37
|
+
path: string;
|
|
38
|
+
skillName: string;
|
|
39
|
+
reportId: string;
|
|
40
|
+
timestamp: string;
|
|
41
|
+
status: DoctorSkillStatus;
|
|
42
|
+
passCount: number;
|
|
43
|
+
warnCount: number;
|
|
44
|
+
failCount: number;
|
|
45
|
+
}
|
|
46
|
+
/** persistDoctorReport per-skill 循环里的索引钩子。best-effort,永不抛、永不阻断正文落盘。 */
|
|
47
|
+
export declare function indexDoctorWrite(card: Omit<DoctorIndexCard, 'domain'>, outputDir: string): void;
|
|
48
|
+
/** 读 doctor 域全部卡片。 */
|
|
49
|
+
export declare function listDoctorCards(): DoctorIndexCard[];
|
|
50
|
+
/** doctor 卡片(过滤悬空真身):供 buildSkillIndex 机器级合并。 */
|
|
51
|
+
export declare function listLiveDoctorCards(): DoctorIndexCard[];
|
|
52
|
+
/** doctor 卡片 → (skillName, SkillDoctorSnapshot)。results:[](逐规则详情需回源项目看)。 */
|
|
53
|
+
export declare function cardToDoctorSnapshot(card: DoctorIndexCard): {
|
|
54
|
+
skillName: string;
|
|
55
|
+
snap: SkillDoctorSnapshot;
|
|
56
|
+
};
|
|
57
|
+
/** 删 doctor 域某 id(文件 stem)的卡片。doctor 历史按 50/skill prune,删正文时必须连卡片一起删,
|
|
58
|
+
* 否则被 prune 掉的报告会经卡片合并在本项目 studio「复活」。 */
|
|
59
|
+
export declare function removeDoctorCard(id: string): boolean;
|
|
60
|
+
/** observe 报告投影到卡片所需的结构子集(不 import observability/SkillHealthReport,保持 eval-core 解耦)。 */
|
|
61
|
+
interface ObserveCardSkill {
|
|
62
|
+
toolFailureRate: number;
|
|
63
|
+
segmentCount: number;
|
|
64
|
+
gap?: {
|
|
65
|
+
weightedGapRate?: number;
|
|
66
|
+
};
|
|
67
|
+
confidence?: 'high' | 'low' | 'underpowered';
|
|
68
|
+
}
|
|
69
|
+
interface ObserveSource {
|
|
70
|
+
meta: {
|
|
71
|
+
generatedAt: string;
|
|
72
|
+
sessionCount: number;
|
|
73
|
+
segmentCount: number;
|
|
74
|
+
};
|
|
75
|
+
overall: {
|
|
76
|
+
healthBand: 'green' | 'yellow' | 'red';
|
|
77
|
+
confidence?: 'high' | 'low' | 'underpowered';
|
|
78
|
+
};
|
|
79
|
+
bySkill: Record<string, ObserveCardSkill>;
|
|
80
|
+
}
|
|
81
|
+
/** observe 卡片:每份报告一张。id = 文件名 stem。剥掉每个 skill 的 gap.signals 等重体,只留 list / skill-index
|
|
82
|
+
* 口径需要的标量(meta 摘要 + overall + per-skill 标量)。详情(含 signals)走 card.path 读真身。无 `kind` 字段。 */
|
|
83
|
+
export interface ObserveIndexCard {
|
|
84
|
+
domain: 'observe-health';
|
|
85
|
+
id: string;
|
|
86
|
+
path: string;
|
|
87
|
+
meta: {
|
|
88
|
+
generatedAt: string;
|
|
89
|
+
sessionCount: number;
|
|
90
|
+
segmentCount: number;
|
|
91
|
+
};
|
|
92
|
+
overall: {
|
|
93
|
+
healthBand: 'green' | 'yellow' | 'red';
|
|
94
|
+
confidence?: 'high' | 'low' | 'underpowered';
|
|
95
|
+
};
|
|
96
|
+
bySkill: Record<string, ObserveCardSkill>;
|
|
97
|
+
}
|
|
98
|
+
/** persistObserveHealthReport 的索引钩子。best-effort,永不抛、永不阻断正文落盘。 */
|
|
99
|
+
export declare function indexObserveWrite(report: ObserveSource, sourcePath: string, outputDir: string, id: string): void;
|
|
100
|
+
/** 读 observe-health 域全部卡片。 */
|
|
101
|
+
export declare function listObserveCards(): ObserveIndexCard[];
|
|
102
|
+
/** observe 卡片(过滤悬空真身):供 listAnalyses / buildSkillIndex 机器级合并。 */
|
|
103
|
+
export declare function listLiveObserveCards(): ObserveIndexCard[];
|
|
104
|
+
/** 删 observe 域某 id 的卡片。 */
|
|
105
|
+
export declare function removeObserveCard(id: string): boolean;
|
|
106
|
+
export {};
|