oh-my-knowledge 0.42.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh.md +1 -1
- package/dist/artifact-graph/doctor.d.ts +21 -0
- package/dist/artifact-graph/doctor.js +569 -0
- package/dist/artifact-graph/eval.d.ts +16 -0
- package/dist/artifact-graph/eval.js +351 -0
- package/dist/assets/agent-skills/omk/SKILL.md +9 -9
- package/dist/assets/agent-skills/omk/references/commands.md +1 -1
- package/dist/cli/commands/doctor.js +62 -61
- package/dist/cli/commands/eval/index.js +20 -2
- package/dist/cli/commands/init.js +1 -0
- package/dist/cli/commands/observe/index.d.ts +2 -2
- package/dist/cli/commands/observe/index.js +8 -7
- package/dist/cli/commands/sample.js +22 -16
- package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/common.js +8 -0
- package/dist/cli/lib/i18n-dict/help.js +12 -10
- package/dist/cli/lib/parse-run-config/samples-discovery.d.ts +5 -7
- package/dist/cli/lib/parse-run-config/samples-discovery.js +10 -32
- package/dist/cli/lib/parse-run-config.js +4 -4
- package/dist/cli/lib/resolve-skill-input.js +10 -12
- package/dist/doctor/index.js +2 -7
- package/dist/doctor/messages.js +2 -2
- package/dist/eval-core/artifact-file-names.d.ts +15 -0
- package/dist/eval-core/artifact-file-names.js +45 -0
- package/dist/eval-core/evaluation-reporting.d.ts +1 -1
- package/dist/eval-core/evaluation-reporting.js +32 -12
- package/dist/eval-core/measurement-dirs.js +13 -7
- package/dist/eval-core/report-file-migration.d.ts +10 -0
- package/dist/eval-core/report-file-migration.js +90 -0
- package/dist/inputs/sample-locator.d.ts +23 -0
- package/dist/inputs/sample-locator.js +195 -0
- package/dist/inputs/skill-loader.js +7 -17
- package/dist/observability/inbox.js +7 -3
- package/dist/renderer/html-renderer.js +3 -2
- package/dist/renderer/skill-detail-renderer.js +832 -0
- package/dist/renderer/skill-list-renderer.js +2 -7
- package/dist/server/report-server.js +55 -15
- package/dist/server/report-store.js +17 -9
- package/dist/server/skill-index.d.ts +6 -0
- package/dist/server/skill-index.js +399 -15
- package/dist/types/artifact-graph.d.ts +93 -0
- package/dist/types/artifact-graph.js +1 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/skill-index.d.ts +38 -0
- package/package.json +1 -1
|
@@ -2,8 +2,8 @@ import { BaseCommand } from '../../oclif/base-command.js';
|
|
|
2
2
|
import type { SkillHealthReport } from '../../../observability/skill-health-analyzer.js';
|
|
3
3
|
/**
|
|
4
4
|
* observe-health 报告落盘:id / 文件名加 4 位随机段,根治「同秒两次 omk observe 直接覆盖、数据丢失」的 bug。
|
|
5
|
-
*
|
|
6
|
-
*
|
|
5
|
+
* 文件名统一为 `{run}.report.json`:目录表达 observe-health 域,文件表达这是可读报告。
|
|
6
|
+
* 落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
|
|
7
7
|
*/
|
|
8
8
|
export declare function persistObserveHealthReport(report: SkillHealthReport, outDir: string): {
|
|
9
9
|
id: string;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
2
|
-
import { resolve
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
3
|
import { Args, Flags } from '@oclif/core';
|
|
4
4
|
import { LANG_FLAG, bilingual } from '../../oclif/i18n.js';
|
|
5
5
|
import { BaseCommand } from '../../oclif/base-command.js';
|
|
@@ -8,17 +8,18 @@ import { tCli } from '../../lib/i18n.js';
|
|
|
8
8
|
import { parseLastWindow } from '../../lib/shared.js';
|
|
9
9
|
import { projectObserveHealthDir, globalObserveHealthDir } from '../../../eval-core/measurement-dirs.js';
|
|
10
10
|
import { indexObserveWrite } from '../../../eval-core/artifact-index.js';
|
|
11
|
+
import { reportFilePath, runFileSuffix } from '../../../eval-core/artifact-file-names.js';
|
|
12
|
+
import { migrateLegacyReportFiles } from '../../../eval-core/report-file-migration.js';
|
|
11
13
|
/**
|
|
12
14
|
* observe-health 报告落盘:id / 文件名加 4 位随机段,根治「同秒两次 omk observe 直接覆盖、数据丢失」的 bug。
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
+
* 文件名统一为 `{run}.report.json`:目录表达 observe-health 域,文件表达这是可读报告。
|
|
16
|
+
* 落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
|
|
15
17
|
*/
|
|
16
18
|
export function persistObserveHealthReport(report, outDir) {
|
|
17
19
|
mkdirSync(outDir, { recursive: true });
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
const
|
|
21
|
-
const jsonPath = join(outDir, `${id}.json`);
|
|
20
|
+
migrateLegacyReportFiles(outDir, 'observe-health');
|
|
21
|
+
const id = runFileSuffix();
|
|
22
|
+
const jsonPath = reportFilePath(outDir, id);
|
|
22
23
|
writeFileSync(jsonPath, JSON.stringify(report, null, 2));
|
|
23
24
|
indexObserveWrite(report, jsonPath, outDir, id);
|
|
24
25
|
return { id, jsonPath };
|
|
@@ -9,6 +9,7 @@ import { CliExit } from '../lib/cli-exit.js';
|
|
|
9
9
|
import { tCli } from '../lib/i18n.js';
|
|
10
10
|
import { projectReportsDir, globalReportsDir } from '../../eval-core/measurement-dirs.js';
|
|
11
11
|
import { loadSamples, parseYaml, listSampleFilesInDir } from '../../inputs/load-samples.js';
|
|
12
|
+
import { defaultFlatSkillSamplesFile, defaultSkillLocalSamplesFile, findFlatSkillSamplesPath, findSkillSamplesPath, } from '../../inputs/sample-locator.js';
|
|
12
13
|
import { hashSample } from '../../eval-core/evaluation-reporting.js';
|
|
13
14
|
import { hashArtifactSource } from '../../inputs/content-hash.js';
|
|
14
15
|
function isRecord(value) {
|
|
@@ -233,23 +234,23 @@ async function runSampleFix(args, flags, lang) {
|
|
|
233
234
|
console.error(lang === 'zh' ? '请指定 skill 路径,如: omk sample skills/my-skill/SKILL.md --fix' : 'Specify skill path: omk sample skills/my-skill/SKILL.md --fix');
|
|
234
235
|
throw new CliExit(1);
|
|
235
236
|
}
|
|
236
|
-
const
|
|
237
|
-
|
|
238
|
-
|
|
237
|
+
const { resolveSkillInput } = await import('../lib/resolve-skill-input.js');
|
|
238
|
+
let resolvedInput;
|
|
239
|
+
try {
|
|
240
|
+
resolvedInput = resolveSkillInput(skillPath, lang);
|
|
241
|
+
}
|
|
242
|
+
catch (err) {
|
|
243
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
239
244
|
throw new CliExit(1);
|
|
240
245
|
}
|
|
241
|
-
const
|
|
242
|
-
const skillDir = isDir ? dirname(resolvedSkillPath) : dirname(resolvedSkillPath);
|
|
243
|
-
const samplesInput = isDir
|
|
244
|
-
? join(skillDir, '.omk')
|
|
245
|
-
: resolve('eval-samples.json');
|
|
246
|
+
const samplesInput = resolvedInput.samplesPath;
|
|
246
247
|
if (!existsSync(samplesInput)) {
|
|
247
248
|
console.error(lang === 'zh' ? `samples 路径不存在: ${samplesInput},先运行 omk sample 生成` : `Samples path not found: ${samplesInput}, run omk sample first`);
|
|
248
249
|
throw new CliExit(1);
|
|
249
250
|
}
|
|
250
|
-
const defaultTreatmentName =
|
|
251
|
-
? basename(skillDir)
|
|
252
|
-
: basename(
|
|
251
|
+
const defaultTreatmentName = resolvedInput.isDirectorySkill
|
|
252
|
+
? basename(resolvedInput.skillDir)
|
|
253
|
+
: basename(resolvedInput.skillPath, extname(resolvedInput.skillPath));
|
|
253
254
|
const treatmentName = flags.treatment ?? defaultTreatmentName;
|
|
254
255
|
process.stderr.write(lang === 'zh' ? `🔍 正在查找 ${treatmentName} 的最新评测报告...\n` : `🔍 Scanning latest report for ${treatmentName}...\n`);
|
|
255
256
|
// 显式 --reports-dir 固定该目录;默认 overlay(项目 .omk/reports 盖全局),findByVariant 记录优先看项目、
|
|
@@ -277,7 +278,7 @@ async function runSampleFix(args, flags, lang) {
|
|
|
277
278
|
throw new CliExit(1);
|
|
278
279
|
}
|
|
279
280
|
const samples = loadedSamples.samples;
|
|
280
|
-
const skillContent = readFileSync(
|
|
281
|
+
const skillContent = readFileSync(resolvedInput.skillPath, 'utf-8');
|
|
281
282
|
const sampleDesignIds = collectSampleDesignFailureIds(report, treatmentName);
|
|
282
283
|
const sampleDesignCount = sampleDesignIds.size;
|
|
283
284
|
if (sampleDesignCount === 0) {
|
|
@@ -286,7 +287,7 @@ async function runSampleFix(args, flags, lang) {
|
|
|
286
287
|
}
|
|
287
288
|
// 当前内容指纹走整树哈,与 eval 报告口径一致:dir-skill(用户传 .../SKILL.md)哈整棵 skill 目录、
|
|
288
289
|
// 单文件 .md 哈单文件字节。
|
|
289
|
-
const currentContentHash = hashArtifactSource(
|
|
290
|
+
const currentContentHash = hashArtifactSource(resolvedInput.isDirectorySkill ? resolvedInput.skillDir : resolvedInput.skillPath, resolvedInput.isDirectorySkill);
|
|
290
291
|
try {
|
|
291
292
|
assertFixReportMatchesCurrentInputs({
|
|
292
293
|
report,
|
|
@@ -435,24 +436,29 @@ async function runSample(args, flags, lang) {
|
|
|
435
436
|
let name;
|
|
436
437
|
let skillPath;
|
|
437
438
|
let samplesPath;
|
|
439
|
+
let existingSamplesPath;
|
|
438
440
|
const fullPath = join(skillDir, entry);
|
|
439
441
|
if (entry.endsWith('.md') && !entry.endsWith('.eval-samples.json')) {
|
|
440
442
|
name = entry.slice(0, -3);
|
|
441
443
|
skillPath = fullPath;
|
|
442
|
-
samplesPath =
|
|
444
|
+
samplesPath = defaultFlatSkillSamplesFile(skillDir, name);
|
|
445
|
+
existingSamplesPath = findFlatSkillSamplesPath(skillDir, name);
|
|
443
446
|
}
|
|
444
447
|
else if (statSync(fullPath).isDirectory()) {
|
|
445
448
|
const skillMd = join(fullPath, 'SKILL.md');
|
|
446
449
|
if (!existsSync(skillMd))
|
|
447
450
|
continue;
|
|
451
|
+
if (existsSync(join(skillDir, `${entry}.md`)))
|
|
452
|
+
continue;
|
|
448
453
|
name = entry;
|
|
449
454
|
skillPath = skillMd;
|
|
450
|
-
samplesPath =
|
|
455
|
+
samplesPath = defaultSkillLocalSamplesFile(fullPath);
|
|
456
|
+
existingSamplesPath = findSkillSamplesPath(fullPath);
|
|
451
457
|
}
|
|
452
458
|
else {
|
|
453
459
|
continue;
|
|
454
460
|
}
|
|
455
|
-
if (
|
|
461
|
+
if (existingSamplesPath) {
|
|
456
462
|
process.stderr.write(tCli('cli.gen.skill_skipped_existing', lang, { name }));
|
|
457
463
|
continue;
|
|
458
464
|
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { CliMessage } from './types.js';
|
|
2
|
-
export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.samples_detected' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
|
|
2
|
+
export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.common.deprecated_skill_samples_path' | 'cli.common.samples_not_found' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.samples_detected' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
|
|
3
3
|
export declare const commonDict: Record<CommonMessageKey, CliMessage>;
|
|
@@ -35,6 +35,14 @@ export const commonDict = {
|
|
|
35
35
|
zh: '⚠ 加载 samples 文件失败 ({path}): {message}\n',
|
|
36
36
|
en: '⚠ Failed to load samples file ({path}): {message}\n',
|
|
37
37
|
},
|
|
38
|
+
'cli.common.deprecated_skill_samples_path': {
|
|
39
|
+
zh: '⚠ 发现旧的目录 skill 用例位置:{oldPath}。目录 skill 的私有用例已改放到 {newPath},请迁移。\n',
|
|
40
|
+
en: '⚠ Found deprecated directory skill samples path: {oldPath}. Directory skill samples now live at {newPath}; please migrate.\n',
|
|
41
|
+
},
|
|
42
|
+
'cli.common.samples_not_found': {
|
|
43
|
+
zh: '未找到评测用例:{path}。请通过 --samples 指定文件,或创建项目级 eval-samples.json;单 treatment 目录 skill 请使用 <skill>/.omk/samples.json。',
|
|
44
|
+
en: 'Eval samples not found: {path}. Pass --samples, create project-level eval-samples.json, or use <skill>/.omk/samples.json for a single-treatment directory skill.',
|
|
45
|
+
},
|
|
38
46
|
'cli.update.new_version_available': {
|
|
39
47
|
zh: '\n💡 新版本可用:{old} → {new},运行 npm i -g oh-my-knowledge@latest 升级\n\n',
|
|
40
48
|
en: '\n💡 New version available: {old} → {new}, run npm i -g oh-my-knowledge@latest to upgrade\n\n',
|
|
@@ -170,20 +170,21 @@ omk sample——生成或补齐 eval-samples 评测用例
|
|
|
170
170
|
omk sample --batch [--skill-dir <dir>] [options]
|
|
171
171
|
|
|
172
172
|
输出位置(默认):
|
|
173
|
-
|
|
174
|
-
|
|
173
|
+
目录 skill(<skill>/SKILL.md) → <skill>/.omk/samples.json(omk 标准约定)
|
|
174
|
+
扁平 .md(单次) → 当前目录的 eval-samples.json(项目级兜底)
|
|
175
|
+
扁平 .md(--batch) → <skill-dir>/<name>.eval-samples.json(兼容 paired 布局)
|
|
175
176
|
|
|
176
177
|
选项:
|
|
177
178
|
--count <n> 强制生成 N 条(不指定时由 LLM 按 skill 类型自动判断:
|
|
178
179
|
工作流型 6-8 条 / 原子型 4-6 条 / 混合型 5-7 条)
|
|
179
180
|
--model <name> 生成模型(默认:opus;lean+effort-low 已自动开,想省钱可改 sonnet/haiku)
|
|
180
181
|
--focus <text> 自然语言指定希望覆盖的场景(追加到 prompt,优先级高于自由发挥)
|
|
181
|
-
--batch 为 skill 目录下缺少
|
|
182
|
+
--batch 为 skill 目录下缺少 samples 的 skill 批量生成
|
|
182
183
|
--skill-dir <path> skill 目录(batch 使用,默认:skills)
|
|
183
184
|
|
|
184
185
|
示例:
|
|
185
|
-
omk
|
|
186
|
-
omk
|
|
186
|
+
omk sample skills/req-tool.md
|
|
187
|
+
omk sample skills/req-tool.md --count 8 \\
|
|
187
188
|
--focus "重点覆盖 tag 查询走 PROJECT 空 → WORKSPACE 兜底的多步流程,以及 search 失败的错误路径"
|
|
188
189
|
`,
|
|
189
190
|
en: `
|
|
@@ -194,20 +195,21 @@ Usage:
|
|
|
194
195
|
omk sample --batch [--skill-dir <dir>] [options]
|
|
195
196
|
|
|
196
197
|
Output path (default):
|
|
197
|
-
<skill>/SKILL.md
|
|
198
|
-
|
|
198
|
+
directory skill (<skill>/SKILL.md) → <skill>/.omk/samples.json (omk standard layout)
|
|
199
|
+
flat .md (single) → ./eval-samples.json in current directory (project fallback)
|
|
200
|
+
flat .md (--batch) → <skill-dir>/<name>.eval-samples.json (compatible paired layout)
|
|
199
201
|
|
|
200
202
|
Options:
|
|
201
203
|
--count <n> Force N samples (omit to let LLM auto-decide by skill type:
|
|
202
204
|
workflow 6-8 / atomic 4-6 / mixed 5-7)
|
|
203
205
|
--model <name> Generation model (default: opus; lean+effort-low applied; pass --model sonnet/haiku to save cost)
|
|
204
206
|
--focus <text> Natural-language scenario hints appended to the prompt (overrides freeform diversity)
|
|
205
|
-
--batch Generate for skills that are missing
|
|
207
|
+
--batch Generate for skills that are missing samples
|
|
206
208
|
--skill-dir <path> Skill directory for batch mode (default: skills)
|
|
207
209
|
|
|
208
210
|
Examples:
|
|
209
|
-
omk
|
|
210
|
-
omk
|
|
211
|
+
omk sample skills/req-tool.md
|
|
212
|
+
omk sample skills/req-tool.md --count 8 \\
|
|
211
213
|
--focus "Cover PROJECT-empty → WORKSPACE-fallback multi-step tag lookup and the search-failure error path"
|
|
212
214
|
`,
|
|
213
215
|
},
|
|
@@ -1,18 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* 当 CLI 未传 `--samples` 时,自动发现 sample 路径。
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* 1. treatment
|
|
6
|
-
* 2.
|
|
7
|
-
* 3. fallback `<skillDir>/<treatmentName>/.omk/`
|
|
8
|
-
* 4. 终极兜底 cwd 下的 `eval-samples.{json,yaml,yml}`
|
|
4
|
+
* 发现顺序:
|
|
5
|
+
* 1. 单 treatment → 找该 skill 私有 samples(`<skill>/.omk/`)或扁平 skill paired 文件
|
|
6
|
+
* 2. 终极兜底 cwd 下的项目级 `eval-samples.{json,yaml,yml}`
|
|
9
7
|
*
|
|
10
8
|
* `loadSamples` 在下游自己处理「文件 vs 目录」—— 目录模式 glob `*.{json,yaml,yml}`
|
|
11
9
|
* 合并,跳过 reserved 前缀(`report-` / `health-` / `_`)。所以一个 skill 可以把
|
|
12
10
|
* sample 拆到多文件(`workflow.json` + `platform.json`),也可以单 `samples.json`,
|
|
13
11
|
* 两种都行。
|
|
14
12
|
*
|
|
15
|
-
* 多 treatment
|
|
16
|
-
* 没有唯一答案)
|
|
13
|
+
* 多 treatment 评测不会触发 skill-local 自动发现(因为「找哪个 skill 的 bundled samples」
|
|
14
|
+
* 没有唯一答案),只回到项目级 samples 兜底。
|
|
17
15
|
*/
|
|
18
16
|
export declare function discoverSamplesPath(values: Record<string, unknown>, skillDir: string): string;
|
|
@@ -1,50 +1,28 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* 当 CLI 未传 `--samples` 时,自动发现 sample 路径。
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* 1. treatment
|
|
6
|
-
* 2.
|
|
7
|
-
* 3. fallback `<skillDir>/<treatmentName>/.omk/`
|
|
8
|
-
* 4. 终极兜底 cwd 下的 `eval-samples.{json,yaml,yml}`
|
|
4
|
+
* 发现顺序:
|
|
5
|
+
* 1. 单 treatment → 找该 skill 私有 samples(`<skill>/.omk/`)或扁平 skill paired 文件
|
|
6
|
+
* 2. 终极兜底 cwd 下的项目级 `eval-samples.{json,yaml,yml}`
|
|
9
7
|
*
|
|
10
8
|
* `loadSamples` 在下游自己处理「文件 vs 目录」—— 目录模式 glob `*.{json,yaml,yml}`
|
|
11
9
|
* 合并,跳过 reserved 前缀(`report-` / `health-` / `_`)。所以一个 skill 可以把
|
|
12
10
|
* sample 拆到多文件(`workflow.json` + `platform.json`),也可以单 `samples.json`,
|
|
13
11
|
* 两种都行。
|
|
14
12
|
*
|
|
15
|
-
* 多 treatment
|
|
16
|
-
* 没有唯一答案)
|
|
13
|
+
* 多 treatment 评测不会触发 skill-local 自动发现(因为「找哪个 skill 的 bundled samples」
|
|
14
|
+
* 没有唯一答案),只回到项目级 samples 兜底。
|
|
17
15
|
*/
|
|
18
|
-
import {
|
|
19
|
-
import { existsSync, statSync } from 'node:fs';
|
|
16
|
+
import { findProjectSamplesFile, findSingleTreatmentSamplesPath } from '../../../inputs/sample-locator.js';
|
|
20
17
|
export function discoverSamplesPath(values, skillDir) {
|
|
21
18
|
const treatmentRaw = values.treatment;
|
|
22
19
|
const treatments = treatmentRaw
|
|
23
20
|
? treatmentRaw.split(',').map((v) => v.trim()).filter(Boolean)
|
|
24
21
|
: [];
|
|
25
22
|
if (treatments.length === 1) {
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
const treatmentDir = statSync(resolved).isDirectory() ? resolved : dirname(resolved);
|
|
30
|
-
const omkDir = join(treatmentDir, '.omk');
|
|
31
|
-
if (existsSync(omkDir))
|
|
32
|
-
return omkDir;
|
|
33
|
-
for (const name of ['eval-samples.json', 'eval-samples.yaml', 'eval-samples.yml']) {
|
|
34
|
-
if (existsSync(join(treatmentDir, name)))
|
|
35
|
-
return join(treatmentDir, name);
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
const omkDir = join(skillDir, expr, '.omk');
|
|
39
|
-
if (existsSync(omkDir))
|
|
40
|
-
return omkDir;
|
|
23
|
+
const samplesPath = findSingleTreatmentSamplesPath(treatments[0], skillDir, process.cwd());
|
|
24
|
+
if (samplesPath)
|
|
25
|
+
return samplesPath;
|
|
41
26
|
}
|
|
42
|
-
|
|
43
|
-
if (!existsSync(resolve(cwdFile))) {
|
|
44
|
-
if (existsSync(resolve('eval-samples.yaml')))
|
|
45
|
-
cwdFile = 'eval-samples.yaml';
|
|
46
|
-
else if (existsSync(resolve('eval-samples.yml')))
|
|
47
|
-
cwdFile = 'eval-samples.yml';
|
|
48
|
-
}
|
|
49
|
-
return cwdFile;
|
|
27
|
+
return findProjectSamplesFile(process.cwd()) ?? 'eval-samples.json';
|
|
50
28
|
}
|
|
@@ -41,10 +41,10 @@ export function parseRunConfig(values) {
|
|
|
41
41
|
? loadEvalConfig(values.config)
|
|
42
42
|
: null;
|
|
43
43
|
const skillDir = resolve(values['skill-dir'] ?? 'skills');
|
|
44
|
-
// 2) Resolve samples path: CLI > config >
|
|
45
|
-
//
|
|
46
|
-
// which skill's bundled samples to use. The dir form (loadSamples handles
|
|
47
|
-
// means a skill can split samples across multiple files
|
|
44
|
+
// 2) Resolve samples path: CLI > config > single-treatment skill-local discovery > cwd project default.
|
|
45
|
+
// Skill-local discovery only fires when exactly one --treatment is given, so omk knows
|
|
46
|
+
// which skill's bundled samples to use. The `<skill>/.omk/` dir form (loadSamples handles
|
|
47
|
+
// both file + dir) means a skill can split samples across multiple files.
|
|
48
48
|
const cliSamples = values.samples;
|
|
49
49
|
let samplesFile;
|
|
50
50
|
if (cliSamples) {
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { resolve, join, dirname, basename } from 'node:path';
|
|
2
2
|
import { existsSync, statSync } from 'node:fs';
|
|
3
3
|
import { tCli } from './i18n.js';
|
|
4
|
+
import { findFlatSkillSamplesPath, findProjectSamplesFile, findSkillSamplesPath, skillLocalSamplesDir, } from '../../inputs/sample-locator.js';
|
|
4
5
|
// 统一 skill 入参解析:既接受 SKILL.md 文件(老式 + flat skill),也接受 directory-skill
|
|
5
|
-
// 目录(skills/foo/ 自动找 foo/SKILL.md)
|
|
6
|
-
//
|
|
7
|
-
// .
|
|
6
|
+
// 目录(skills/foo/ 自动找 foo/SKILL.md)。目录-skill 以 `<skill>/.omk/` 为标准
|
|
7
|
+
// samples 命名空间;扁平 .md 兼容 paired sidecar,找不到时回到项目级
|
|
8
|
+
// `eval-samples.json`。
|
|
8
9
|
//
|
|
9
10
|
// 错误用 tCli 走 i18n,调用方直接 console.error err.message 给用户看,zh/en 都要正确。
|
|
10
11
|
export function resolveSkillInput(input, lang) {
|
|
@@ -26,16 +27,13 @@ export function resolveSkillInput(input, lang) {
|
|
|
26
27
|
skillPath = resolved;
|
|
27
28
|
skillDir = dirname(resolved);
|
|
28
29
|
}
|
|
29
|
-
const omkDir = join(skillDir, '.omk');
|
|
30
|
-
const candidates = [
|
|
31
|
-
...(existsSync(omkDir) ? [omkDir] : []),
|
|
32
|
-
join(skillDir, 'eval-samples.json'),
|
|
33
|
-
join(skillDir, 'eval-samples.yaml'),
|
|
34
|
-
join(skillDir, 'eval-samples.yml'),
|
|
35
|
-
];
|
|
36
|
-
// fallback 到 .omk/ 目录:上游会报 "no sample files found in directory" 引导用户创建
|
|
37
|
-
const samplesPath = candidates.find(existsSync) ?? omkDir;
|
|
38
30
|
// 形态以解析后的 skillPath 命名为准:目录-skill 的 skillPath 总是 `.../SKILL.md`。
|
|
39
31
|
const isDirectorySkill = basename(skillPath) === 'SKILL.md';
|
|
32
|
+
const flatSkillName = basename(skillPath).replace(/\.md$/i, '');
|
|
33
|
+
const samplesPath = isDirectorySkill
|
|
34
|
+
? findSkillSamplesPath(skillDir) ?? skillLocalSamplesDir(skillDir)
|
|
35
|
+
: findFlatSkillSamplesPath(skillDir, flatSkillName)
|
|
36
|
+
?? findProjectSamplesFile(process.cwd())
|
|
37
|
+
?? 'eval-samples.json';
|
|
40
38
|
return { skillPath, skillDir, samplesPath, isDirectorySkill };
|
|
41
39
|
}
|
package/dist/doctor/index.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { existsSync, statSync } from 'node:fs';
|
|
11
11
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
12
|
+
import { runFileSuffix } from '../eval-core/artifact-file-names.js';
|
|
12
13
|
import { discoverVariants, resolveArtifacts } from '../inputs/skill-loader.js';
|
|
13
14
|
import { DOCTOR_REPORT_SCHEMA_VERSION, isComposerRule } from '../types/doctor.js';
|
|
14
15
|
import { getRegisteredRules } from './rules.js';
|
|
@@ -163,14 +164,8 @@ function inferSkillPath(artifact, baseDir) {
|
|
|
163
164
|
// ---------------------------------------------------------------------------
|
|
164
165
|
// Public entry
|
|
165
166
|
// ---------------------------------------------------------------------------
|
|
166
|
-
let reportIdCounter = 0;
|
|
167
167
|
function nextReportId() {
|
|
168
|
-
|
|
169
|
-
const ts = new Date().toISOString().replace(/[-:.]/g, '').slice(0, 15);
|
|
170
|
-
// 进程内计数器只防同进程同秒撞;跨进程同秒(两个 omk doctor 并发 / 不同项目)仍会撞同 id,
|
|
171
|
-
// 经机器级卡片 dedup 当唯一键用时会静默并掉一份。加 4 位随机根治跨进程撞名(id 是标签非测量数)。
|
|
172
|
-
const rand = Math.random().toString(36).slice(2, 6);
|
|
173
|
-
return `doctor-${ts}-${reportIdCounter}-${rand}`;
|
|
168
|
+
return `doctor-${runFileSuffix()}`;
|
|
174
169
|
}
|
|
175
170
|
function readCliVersion() {
|
|
176
171
|
// 不依赖 package.json import(避免 type 解析复杂度);用环境变量或退回 'unknown'
|
package/dist/doctor/messages.js
CHANGED
|
@@ -133,8 +133,8 @@ export const DOCTOR_MESSAGES = {
|
|
|
133
133
|
en: 'directory-skill missing SKILL.md entry file',
|
|
134
134
|
},
|
|
135
135
|
'cli.doctor.skill_metadata.hint.frontmatter': {
|
|
136
|
-
zh: 'front-matter 用 YAML
|
|
137
|
-
en: 'front-matter uses YAML syntax (key: value or - item). See examples/
|
|
136
|
+
zh: 'front-matter 用 YAML 语法,key: value 或 - item 形式。可参考 examples/skill-map-showcase/skills/release-readiness 的目录式 skill 写法',
|
|
137
|
+
en: 'front-matter uses YAML syntax (key: value or - item). See examples/skill-map-showcase/skills/release-readiness for a directory-skill reference',
|
|
138
138
|
},
|
|
139
139
|
'cli.doctor.skill_metadata.hint.hardrules': {
|
|
140
140
|
zh: 'hardRules 必须写在 SKILL.md front-matter 中,格式为 hardRules: [{ id, rule, expectedBehavior }];id 要稳定且唯一,expectedBehavior 写可观察行为。',
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export declare const REPORT_FILE_SUFFIX = ".report.json";
|
|
2
|
+
export declare const GRAPH_FILE_SUFFIX = ".graph.json";
|
|
3
|
+
export declare const CARD_FILE_SUFFIX = ".card.md";
|
|
4
|
+
export declare function safeArtifactFileStem(id: string): string;
|
|
5
|
+
export declare function reportFileName(stem: string): string;
|
|
6
|
+
export declare function reportFilePath(dir: string, stem: string): string;
|
|
7
|
+
export declare function reportFileStem(fileName: string): string | null;
|
|
8
|
+
export declare function isReportFileName(fileName: string): boolean;
|
|
9
|
+
export declare function graphFileName(stem: string): string;
|
|
10
|
+
export declare function cardFileName(stem: string): string;
|
|
11
|
+
export declare function runTimestamp(date?: Date): string;
|
|
12
|
+
export declare function randomRunToken(): string;
|
|
13
|
+
export declare function runFileSuffix(): string;
|
|
14
|
+
export declare function stripDomainPrefix(id: string, domain: string): string;
|
|
15
|
+
export declare function doctorReportFileStem(skillName: string, reportId: string): string;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
export const REPORT_FILE_SUFFIX = '.report.json';
|
|
3
|
+
export const GRAPH_FILE_SUFFIX = '.graph.json';
|
|
4
|
+
export const CARD_FILE_SUFFIX = '.card.md';
|
|
5
|
+
export function safeArtifactFileStem(id) {
|
|
6
|
+
return id.replaceAll(/[/\\:*?"<>|]/g, '_');
|
|
7
|
+
}
|
|
8
|
+
export function reportFileName(stem) {
|
|
9
|
+
return `${safeArtifactFileStem(stem)}${REPORT_FILE_SUFFIX}`;
|
|
10
|
+
}
|
|
11
|
+
export function reportFilePath(dir, stem) {
|
|
12
|
+
return join(dir, reportFileName(stem));
|
|
13
|
+
}
|
|
14
|
+
export function reportFileStem(fileName) {
|
|
15
|
+
return fileName.endsWith(REPORT_FILE_SUFFIX)
|
|
16
|
+
? fileName.slice(0, -REPORT_FILE_SUFFIX.length)
|
|
17
|
+
: null;
|
|
18
|
+
}
|
|
19
|
+
export function isReportFileName(fileName) {
|
|
20
|
+
return reportFileStem(fileName) !== null;
|
|
21
|
+
}
|
|
22
|
+
export function graphFileName(stem) {
|
|
23
|
+
return `${safeArtifactFileStem(stem)}${GRAPH_FILE_SUFFIX}`;
|
|
24
|
+
}
|
|
25
|
+
export function cardFileName(stem) {
|
|
26
|
+
return `${safeArtifactFileStem(stem)}${CARD_FILE_SUFFIX}`;
|
|
27
|
+
}
|
|
28
|
+
export function runTimestamp(date = new Date()) {
|
|
29
|
+
const pad = (n) => String(n).padStart(2, '0');
|
|
30
|
+
return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}T${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`;
|
|
31
|
+
}
|
|
32
|
+
export function randomRunToken() {
|
|
33
|
+
return Math.random().toString(36).slice(2, 6);
|
|
34
|
+
}
|
|
35
|
+
export function runFileSuffix() {
|
|
36
|
+
return `${runTimestamp()}-${randomRunToken()}`;
|
|
37
|
+
}
|
|
38
|
+
export function stripDomainPrefix(id, domain) {
|
|
39
|
+
const safeId = safeArtifactFileStem(id);
|
|
40
|
+
const prefix = `${domain}-`;
|
|
41
|
+
return safeId.startsWith(prefix) ? safeId.slice(prefix.length) : safeId;
|
|
42
|
+
}
|
|
43
|
+
export function doctorReportFileStem(skillName, reportId) {
|
|
44
|
+
return `${safeArtifactFileStem(skillName)}-${stripDomainPrefix(reportId, 'doctor')}`;
|
|
45
|
+
}
|
|
@@ -33,7 +33,7 @@ export interface PersistableReport {
|
|
|
33
33
|
}
|
|
34
34
|
export declare function persistReport(report: PersistableReport, outputDir: string | null): string | null;
|
|
35
35
|
/**
|
|
36
|
-
* run id 的时间戳后缀 `
|
|
36
|
+
* run id 的时间戳后缀 `YYYYMMDDTHHmmss-rand4`。
|
|
37
37
|
* 含秒 + 4 位随机:id 是 run 标签(非测量数),但被 studio 机器级 dedup 与 managed 证据 (reportId,
|
|
38
38
|
* contentHash) 去重当唯一键用。分钟级会让跨项目 / 同分钟重跑撞同 id → 索引静默顶掉一份、managed 错并一条。
|
|
39
39
|
* 秒+随机根治撞名,保证每次 run 全局唯一。供 generateRunId 与 evolve 合并 id 共用,避免靠 split 反解格式。
|
|
@@ -5,6 +5,8 @@ import { createHash } from 'node:crypto';
|
|
|
5
5
|
import { fileURLToPath } from 'node:url';
|
|
6
6
|
import { DEFAULT_REPORTS_DIR } from './default-dirs.js';
|
|
7
7
|
import { indexReportWrite } from './artifact-index.js';
|
|
8
|
+
import { randomRunToken, reportFilePath, runTimestamp } from './artifact-file-names.js';
|
|
9
|
+
import { persistEvalGraphSidecar } from '../artifact-graph/eval.js';
|
|
8
10
|
import { buildVariantSummary } from './schema.js';
|
|
9
11
|
import { buildVariantConfig, resolveExecutionStrategy } from './execution-strategy.js';
|
|
10
12
|
import { getJudgePromptHash } from '../grading/judge.js';
|
|
@@ -275,35 +277,53 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
|
|
|
275
277
|
}])),
|
|
276
278
|
};
|
|
277
279
|
}
|
|
280
|
+
function isEvaluationReport(report) {
|
|
281
|
+
return report['kind'] === 'evaluation';
|
|
282
|
+
}
|
|
283
|
+
function persistEvalGraphSidecarSafely(report, outputDir, sourcePath) {
|
|
284
|
+
if (!isEvaluationReport(report))
|
|
285
|
+
return;
|
|
286
|
+
try {
|
|
287
|
+
persistEvalGraphSidecar({ report, outputDir, sourcePath, fileStem: report.id });
|
|
288
|
+
}
|
|
289
|
+
catch (err) {
|
|
290
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
291
|
+
process.stderr.write(`[omk] 写入 eval 图谱失败:${message}\n`);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
278
294
|
export function persistReport(report, outputDir) {
|
|
279
295
|
if (!outputDir)
|
|
280
296
|
return null;
|
|
281
297
|
if (!existsSync(outputDir))
|
|
282
298
|
mkdirSync(outputDir, { recursive: true });
|
|
283
|
-
const filePath =
|
|
299
|
+
const filePath = reportFilePath(outputDir, report.id);
|
|
284
300
|
writeFileSync(filePath, JSON.stringify(report, null, 2));
|
|
301
|
+
persistEvalGraphSidecarSafely(report, outputDir, filePath);
|
|
285
302
|
// 产物发现索引:报告落项目本地后,best-effort 追加全局轻卡片,让 omk studio 跨项目聚合成机器级总览。
|
|
286
303
|
// 永不抛、永不阻断报告落盘(正文是 source of truth)。
|
|
287
304
|
indexReportWrite(report, filePath, outputDir);
|
|
288
305
|
return filePath;
|
|
289
306
|
}
|
|
290
307
|
/**
|
|
291
|
-
* run id 的时间戳后缀 `
|
|
308
|
+
* run id 的时间戳后缀 `YYYYMMDDTHHmmss-rand4`。
|
|
292
309
|
* 含秒 + 4 位随机:id 是 run 标签(非测量数),但被 studio 机器级 dedup 与 managed 证据 (reportId,
|
|
293
310
|
* contentHash) 去重当唯一键用。分钟级会让跨项目 / 同分钟重跑撞同 id → 索引静默顶掉一份、managed 错并一条。
|
|
294
311
|
* 秒+随机根治撞名,保证每次 run 全局唯一。供 generateRunId 与 evolve 合并 id 共用,避免靠 split 反解格式。
|
|
295
312
|
*/
|
|
296
313
|
export function runIdSuffix() {
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
const
|
|
301
|
-
|
|
302
|
-
|
|
314
|
+
return `${runTimestamp()}-${randomRunToken()}`;
|
|
315
|
+
}
|
|
316
|
+
function safeRunSubject(subject) {
|
|
317
|
+
const sanitized = subject
|
|
318
|
+
.replaceAll(/[\\/:]/g, '-')
|
|
319
|
+
.replaceAll(/[^a-zA-Z0-9._@-]/g, '_')
|
|
320
|
+
.replace(/^-+|-+$/g, '');
|
|
321
|
+
return sanitized || 'run';
|
|
322
|
+
}
|
|
323
|
+
function primaryRunSubject(variants) {
|
|
324
|
+
const nonBaseline = variants.filter((variant) => variant !== 'baseline');
|
|
325
|
+
return nonBaseline.at(-1) ?? variants.at(-1) ?? 'run';
|
|
303
326
|
}
|
|
304
327
|
export function generateRunId(variants) {
|
|
305
|
-
|
|
306
|
-
.map((variant) => variant.replaceAll(/[\\/:]/g, '-').replaceAll(/[^a-zA-Z0-9._@-]/g, '_'))
|
|
307
|
-
.join('-vs-');
|
|
308
|
-
return `${variantPart}-${runIdSuffix()}`;
|
|
328
|
+
return `${safeRunSubject(primaryRunSubject(variants))}-${runIdSuffix()}`;
|
|
309
329
|
}
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
import { existsSync, readdirSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { DEFAULT_OBSERVE_HEALTH_DIR, DEFAULT_DOCTORS_DIR, DEFAULT_REPORTS_DIR } from './default-dirs.js';
|
|
4
|
+
import { isReportFileName } from './artifact-file-names.js';
|
|
5
|
+
import { migrateLegacyReportFiles } from './report-file-migration.js';
|
|
4
6
|
/**
|
|
5
7
|
* 测量产物的「项目优先 → 全局兜底」目录解析,镜像 managed 的 `resolveManagedDir`
|
|
6
8
|
* (src/managed/store.ts)。测量产物绑用例集上下文(construct validity,不可全局化),
|
|
7
9
|
* 默认落项目 `.omk/`,全局作显式 opt-in。
|
|
8
10
|
*
|
|
9
|
-
* 「记录优先」—— 目录里有匹配
|
|
10
|
-
* 避免空项目目录遮蔽全局数据。项目目录按**调用时** `cwd()`
|
|
11
|
-
*
|
|
11
|
+
* 「记录优先」—— 目录里有匹配 report 文件才算数(不是「目录存在」),与 managed 同口径,
|
|
12
|
+
* 避免空项目目录遮蔽全局数据。项目目录按**调用时** `cwd()` 求值(函数,不是 import 时
|
|
13
|
+
* 冻结的常量),studio 长会话 per-request 解析才正确。
|
|
12
14
|
*/
|
|
13
15
|
/** dir 里是否有至少一个满足 match 的文件(短路,不读文件内容)。 */
|
|
14
16
|
function hasMatchingFile(dir, match) {
|
|
@@ -22,7 +24,7 @@ function hasMatchingFile(dir, match) {
|
|
|
22
24
|
}
|
|
23
25
|
}
|
|
24
26
|
// —— observe-health(skill 健康度报告)——
|
|
25
|
-
// 文件名 `{
|
|
27
|
+
// 文件名 `{run}.report.json`,目录已表达 observe-health 域。
|
|
26
28
|
/** 项目级 observe-health 目录(相对调用时 cwd)。 */
|
|
27
29
|
export function projectObserveHealthDir(cwd = process.cwd()) {
|
|
28
30
|
return join(cwd, '.omk', 'observe-health');
|
|
@@ -34,7 +36,9 @@ export function globalObserveHealthDir() {
|
|
|
34
36
|
/** 权威 observe-health 目录:项目有报告取项目,否则全局有取全局,都空回项目(同 resolveManagedDir)。
|
|
35
37
|
* `global` 可注入(默认真实全局目录),仅供测试用受控 temp 目录复现 project↔global 兜底。 */
|
|
36
38
|
export function resolveObserveHealthDir(dir = projectObserveHealthDir(), global = globalObserveHealthDir()) {
|
|
37
|
-
|
|
39
|
+
migrateLegacyReportFiles(dir, 'observe-health');
|
|
40
|
+
migrateLegacyReportFiles(global, 'observe-health');
|
|
41
|
+
const has = (d) => hasMatchingFile(d, isReportFileName);
|
|
38
42
|
if (has(dir))
|
|
39
43
|
return dir;
|
|
40
44
|
if (dir !== global && has(global))
|
|
@@ -42,7 +46,7 @@ export function resolveObserveHealthDir(dir = projectObserveHealthDir(), global
|
|
|
42
46
|
return dir;
|
|
43
47
|
}
|
|
44
48
|
// —— doctors(体检报告)——
|
|
45
|
-
// 文件名 `{skill}-{
|
|
49
|
+
// 文件名 `{skill}-{run}.report.json`,谓词用 report 后缀即可(真解析在下游 scanDoctorReports)。
|
|
46
50
|
/** 项目级 doctors 目录(相对调用时 cwd)。 */
|
|
47
51
|
export function projectDoctorsDir(cwd = process.cwd()) {
|
|
48
52
|
return join(cwd, '.omk', 'doctors');
|
|
@@ -54,7 +58,9 @@ export function globalDoctorsDir() {
|
|
|
54
58
|
/** 权威 doctors 目录:项目有报告取项目,否则全局有取全局,都空回项目(同 resolveManagedDir)。
|
|
55
59
|
* `global` 可注入(默认真实全局目录),仅供测试用受控 temp 目录复现 project↔global 兜底。 */
|
|
56
60
|
export function resolveDoctorsDir(dir = projectDoctorsDir(), global = globalDoctorsDir()) {
|
|
57
|
-
|
|
61
|
+
migrateLegacyReportFiles(dir, 'doctor');
|
|
62
|
+
migrateLegacyReportFiles(global, 'doctor');
|
|
63
|
+
const has = (d) => hasMatchingFile(d, isReportFileName);
|
|
58
64
|
if (has(dir))
|
|
59
65
|
return dir;
|
|
60
66
|
if (dir !== global && has(global))
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export type LegacyReportDomain = 'report' | 'doctor' | 'observe-health' | 'observe-inbox';
|
|
2
|
+
/**
|
|
3
|
+
* One-shot migration for pre-`.report.json` run artifacts.
|
|
4
|
+
*
|
|
5
|
+
* This is deliberately scoped to measurement report directories. It does not make
|
|
6
|
+
* bare `.json` a normal reader format again; it just renames known legacy report
|
|
7
|
+
* files into the canonical name so old local data can be discovered, rotated, and
|
|
8
|
+
* eventually garbage-collected by the new code paths.
|
|
9
|
+
*/
|
|
10
|
+
export declare function migrateLegacyReportFiles(dir: string, domain: LegacyReportDomain): void;
|