oh-my-knowledge 0.42.0 → 0.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh.md +1 -1
- package/dist/artifact-graph/doctor.d.ts +21 -0
- package/dist/artifact-graph/doctor.js +569 -0
- package/dist/assets/agent-skills/omk/SKILL.md +9 -9
- package/dist/assets/agent-skills/omk/references/commands.md +1 -1
- package/dist/cli/commands/doctor.js +62 -61
- package/dist/cli/commands/eval/index.js +20 -2
- package/dist/cli/commands/init.js +1 -0
- package/dist/cli/commands/observe/index.d.ts +2 -2
- package/dist/cli/commands/observe/index.js +8 -7
- package/dist/cli/commands/sample.js +22 -16
- package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/common.js +8 -0
- package/dist/cli/lib/i18n-dict/help.js +12 -10
- package/dist/cli/lib/parse-run-config/samples-discovery.d.ts +5 -7
- package/dist/cli/lib/parse-run-config/samples-discovery.js +10 -32
- package/dist/cli/lib/parse-run-config.js +4 -4
- package/dist/cli/lib/resolve-skill-input.js +10 -12
- package/dist/doctor/messages.js +2 -2
- package/dist/eval-core/artifact-file-names.d.ts +15 -0
- package/dist/eval-core/artifact-file-names.js +46 -0
- package/dist/eval-core/evaluation-reporting.js +2 -1
- package/dist/eval-core/measurement-dirs.js +13 -7
- package/dist/eval-core/report-file-migration.d.ts +10 -0
- package/dist/eval-core/report-file-migration.js +90 -0
- package/dist/inputs/sample-locator.d.ts +23 -0
- package/dist/inputs/sample-locator.js +195 -0
- package/dist/inputs/skill-loader.js +7 -17
- package/dist/observability/inbox.js +7 -3
- package/dist/server/report-server.js +10 -4
- package/dist/server/report-store.js +17 -9
- package/dist/server/skill-index.js +16 -11
- package/dist/types/artifact-graph.d.ts +93 -0
- package/dist/types/artifact-graph.js +1 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/package.json +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { mkdirSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join, resolve } from 'node:path';
|
|
3
3
|
import { Args, Flags } from '@oclif/core';
|
|
4
4
|
import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
|
|
5
5
|
import { BaseCommand } from '../oclif/base-command.js';
|
|
@@ -9,50 +9,11 @@ import { tCli } from '../lib/i18n.js';
|
|
|
9
9
|
import { makeDoctorProgress } from '../lib/progress.js';
|
|
10
10
|
import { DEFAULT_DOCTORS_DIR } from '../../eval-core/default-dirs.js';
|
|
11
11
|
import { indexDoctorWrite, removeDoctorCard } from '../../eval-core/artifact-index.js';
|
|
12
|
+
import { doctorReportFileStem, isReportFileName, reportFilePath } from '../../eval-core/artifact-file-names.js';
|
|
13
|
+
import { migrateLegacyReportFiles } from '../../eval-core/report-file-migration.js';
|
|
12
14
|
import { projectDoctorsDir, globalDoctorsDir } from '../../eval-core/measurement-dirs.js';
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
for (const name of DEFAULT_SAMPLE_FILENAMES) {
|
|
16
|
-
const candidate = join(dir, name);
|
|
17
|
-
if (existsSync(candidate))
|
|
18
|
-
return candidate;
|
|
19
|
-
}
|
|
20
|
-
return null;
|
|
21
|
-
}
|
|
22
|
-
function sampleSearchDirs(target, cwd) {
|
|
23
|
-
const dirs = [];
|
|
24
|
-
const add = (dir) => {
|
|
25
|
-
const abs = resolve(dir);
|
|
26
|
-
if (!dirs.includes(abs))
|
|
27
|
-
dirs.push(abs);
|
|
28
|
-
};
|
|
29
|
-
if (target) {
|
|
30
|
-
const absTarget = resolve(target);
|
|
31
|
-
if (existsSync(absTarget)) {
|
|
32
|
-
const stat = statSync(absTarget);
|
|
33
|
-
if (stat.isDirectory()) {
|
|
34
|
-
add(absTarget);
|
|
35
|
-
add(dirname(absTarget));
|
|
36
|
-
add(dirname(dirname(absTarget)));
|
|
37
|
-
}
|
|
38
|
-
else {
|
|
39
|
-
const parent = dirname(absTarget);
|
|
40
|
-
add(parent);
|
|
41
|
-
add(dirname(parent));
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
add(cwd);
|
|
46
|
-
return dirs;
|
|
47
|
-
}
|
|
48
|
-
function findDefaultSamplesPath(target, cwd) {
|
|
49
|
-
for (const dir of sampleSearchDirs(target, cwd)) {
|
|
50
|
-
const samplesPath = findSamplesInDir(dir);
|
|
51
|
-
if (samplesPath)
|
|
52
|
-
return samplesPath;
|
|
53
|
-
}
|
|
54
|
-
return null;
|
|
55
|
-
}
|
|
15
|
+
import { findDoctorDeprecatedSamplesHint, findDoctorSamplesPath } from '../../inputs/sample-locator.js';
|
|
16
|
+
import { persistDoctorGraphSidecars, removeDoctorGraphSidecars } from '../../artifact-graph/doctor.js';
|
|
56
17
|
export default class Doctor extends BaseCommand {
|
|
57
18
|
static description = bilingual({
|
|
58
19
|
zh: '体检 omk 工作目录,检查 skill 配置 / 依赖 / executor 连通性。',
|
|
@@ -186,7 +147,16 @@ export default class Doctor extends BaseCommand {
|
|
|
186
147
|
const timeoutSec = flags.timeout != null ? Number(flags.timeout) : defaultTimeoutSec;
|
|
187
148
|
const timeoutMs = Math.max(1000, Math.floor((Number.isFinite(timeoutSec) ? timeoutSec : defaultTimeoutSec) * 1000));
|
|
188
149
|
const cwd = process.cwd();
|
|
189
|
-
const samplesPath = flags.samples ? resolve(flags.samples) :
|
|
150
|
+
const samplesPath = flags.samples ? resolve(flags.samples) : findDoctorSamplesPath(target, cwd);
|
|
151
|
+
const deprecatedSamplesHint = !flags.samples && !samplesPath
|
|
152
|
+
? findDoctorDeprecatedSamplesHint(target, cwd)
|
|
153
|
+
: null;
|
|
154
|
+
if (deprecatedSamplesHint) {
|
|
155
|
+
process.stderr.write(tCli('cli.common.deprecated_skill_samples_path', lang, {
|
|
156
|
+
oldPath: deprecatedSamplesHint.oldPath,
|
|
157
|
+
newPath: deprecatedSamplesHint.newPath,
|
|
158
|
+
}));
|
|
159
|
+
}
|
|
190
160
|
let samples;
|
|
191
161
|
let requires;
|
|
192
162
|
if (samplesPath) {
|
|
@@ -270,7 +240,7 @@ export default class Doctor extends BaseCommand {
|
|
|
270
240
|
}
|
|
271
241
|
persistDoctorReport(report, flags['output-dir']
|
|
272
242
|
? resolve(flags['output-dir'])
|
|
273
|
-
: (flags.global ? globalDoctorsDir() : projectDoctorsDir()));
|
|
243
|
+
: (flags.global ? globalDoctorsDir() : projectDoctorsDir()), lang);
|
|
274
244
|
if (flags.fix) {
|
|
275
245
|
const existing = report;
|
|
276
246
|
if (existing.outcome !== 'failed') {
|
|
@@ -288,47 +258,76 @@ export default class Doctor extends BaseCommand {
|
|
|
288
258
|
// 每个 skill 最多保留多少份历史 doctor 报告(避免无界增长拖慢 studio 启动 +
|
|
289
259
|
// scanDoctorReports 扫盘成本)。50 = ~每天 1 跑撑 1.5 个月 sparkline,够用。
|
|
290
260
|
const DOCTOR_HISTORY_MAX_PER_SKILL = 50;
|
|
291
|
-
function persistDoctorReport(report, outputDir) {
|
|
261
|
+
function persistDoctorReport(report, outputDir, lang = 'zh') {
|
|
292
262
|
const dir = outputDir ?? DEFAULT_DOCTORS_DIR;
|
|
293
263
|
mkdirSync(dir, { recursive: true });
|
|
294
|
-
|
|
264
|
+
migrateLegacyReportFiles(dir, 'doctor');
|
|
295
265
|
for (const skill of report.skills) {
|
|
296
|
-
const counts = {
|
|
266
|
+
const counts = {
|
|
267
|
+
pass: 0,
|
|
268
|
+
warn: 0,
|
|
269
|
+
fail: 0,
|
|
270
|
+
skipped: 0,
|
|
271
|
+
};
|
|
297
272
|
for (const r of skill.results) {
|
|
298
273
|
const s = r.status;
|
|
299
274
|
if (s in counts)
|
|
300
275
|
counts[s]++;
|
|
301
276
|
}
|
|
277
|
+
const outcome = skill.status === 'fail' ? 'failed' : skill.status === 'warn' ? 'warnings_only' : 'passed';
|
|
302
278
|
const perSkill = {
|
|
303
279
|
...report,
|
|
304
280
|
skills: [skill],
|
|
305
|
-
ruleStats: {
|
|
281
|
+
ruleStats: {
|
|
282
|
+
pass: counts.pass,
|
|
283
|
+
warn: counts.warn,
|
|
284
|
+
fail: counts.fail,
|
|
285
|
+
skipped: counts.skipped,
|
|
286
|
+
total: skill.results.length,
|
|
287
|
+
},
|
|
306
288
|
totals: {
|
|
307
289
|
pass: skill.status === 'pass' ? 1 : 0,
|
|
308
290
|
warn: skill.status === 'warn' ? 1 : 0,
|
|
309
291
|
fail: skill.status === 'fail' ? 1 : 0,
|
|
310
292
|
},
|
|
311
|
-
outcome
|
|
293
|
+
outcome,
|
|
312
294
|
};
|
|
313
|
-
const
|
|
314
|
-
const
|
|
315
|
-
const filePath = join(dir, `${cardId}.json`);
|
|
295
|
+
const cardId = doctorReportFileStem(skill.skillName, report.id);
|
|
296
|
+
const filePath = reportFilePath(dir, cardId);
|
|
316
297
|
writeFileSync(filePath, JSON.stringify(perSkill, null, 2), 'utf8');
|
|
317
298
|
// 产物发现索引:per-skill 报告落项目本地后,best-effort 追加全局轻卡片,让 studio 跨项目聚合。
|
|
318
299
|
indexDoctorWrite({
|
|
319
300
|
id: cardId, path: filePath, skillName: skill.skillName, reportId: report.id, timestamp: report.timestamp,
|
|
320
301
|
status: skill.status, passCount: counts.pass, warnCount: counts.warn, failCount: counts.fail,
|
|
321
302
|
}, dir);
|
|
303
|
+
try {
|
|
304
|
+
persistDoctorGraphSidecars({
|
|
305
|
+
report: perSkill,
|
|
306
|
+
skill,
|
|
307
|
+
sourcePath: filePath,
|
|
308
|
+
outputDir: dir,
|
|
309
|
+
fileStem: cardId,
|
|
310
|
+
lang,
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
catch (err) {
|
|
314
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
315
|
+
const warning = lang === 'zh'
|
|
316
|
+
? `⚠️ doctor graph sidecar 写入失败:${message}\n`
|
|
317
|
+
: `⚠️ failed to write doctor graph sidecar: ${message}\n`;
|
|
318
|
+
process.stderr.write(warning);
|
|
319
|
+
}
|
|
322
320
|
pruneDoctorHistory(dir, skill.skillName, DOCTOR_HISTORY_MAX_PER_SKILL);
|
|
323
321
|
}
|
|
324
322
|
}
|
|
325
|
-
// 写入新报告后调用:扫 dir 里属于该 skill 的所有 single-skill doctor
|
|
323
|
+
// 写入新报告后调用:扫 dir 里属于该 skill 的所有 single-skill doctor report,
|
|
326
324
|
// 按 timestamp 倒排,保留 maxKeep 份最近的,其余删。按 content 匹配 skillName 不
|
|
327
|
-
//
|
|
325
|
+
// 看文件名,所以清理逻辑不依赖 readdir 顺序或 stem 推断 skill 名。
|
|
328
326
|
export function pruneDoctorHistory(dir, skillName, maxKeep) {
|
|
327
|
+
migrateLegacyReportFiles(dir, 'doctor');
|
|
329
328
|
const candidates = [];
|
|
330
329
|
for (const file of readdirSync(dir)) {
|
|
331
|
-
if (!file
|
|
330
|
+
if (!isReportFileName(file))
|
|
332
331
|
continue;
|
|
333
332
|
try {
|
|
334
333
|
const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
|
|
@@ -337,20 +336,22 @@ export function pruneDoctorHistory(dir, skillName, maxKeep) {
|
|
|
337
336
|
continue;
|
|
338
337
|
if (data.skills[0].skillName !== skillName)
|
|
339
338
|
continue;
|
|
340
|
-
candidates.push({ file, timestamp: data.timestamp });
|
|
339
|
+
candidates.push({ file, graphStem: doctorReportFileStem(skillName, data.id), timestamp: data.timestamp });
|
|
341
340
|
}
|
|
342
341
|
catch { /* skip corrupt / unrelated json */ }
|
|
343
342
|
}
|
|
344
343
|
if (candidates.length <= maxKeep)
|
|
345
344
|
return;
|
|
346
345
|
candidates.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
|
347
|
-
for (const { file } of candidates.slice(maxKeep)) {
|
|
346
|
+
for (const { file, graphStem } of candidates.slice(maxKeep)) {
|
|
348
347
|
try {
|
|
349
348
|
unlinkSync(join(dir, file));
|
|
350
349
|
}
|
|
351
350
|
catch { /* ignore */ }
|
|
352
351
|
// 连带删卡片:否则被 prune 掉的报告会经 listDoctorCards 合并在本项目 studio「复活」(正文已删、卡片还在)。
|
|
353
352
|
// 卡片 id = 文件 stem(`{name}-{id}`),与 indexDoctorWrite 写入口径一致。
|
|
354
|
-
|
|
353
|
+
const doctorStem = file.replace(/\.report\.json$/, '');
|
|
354
|
+
removeDoctorCard(doctorStem);
|
|
355
|
+
removeDoctorGraphSidecars(dir, graphStem);
|
|
355
356
|
}
|
|
356
357
|
}
|
|
@@ -10,6 +10,7 @@ import { computeRunTally } from '../../lib/run-tally.js';
|
|
|
10
10
|
import { DEFAULT_BOOTSTRAP_SAMPLES } from '../../../eval-core/bootstrap.js';
|
|
11
11
|
import { DEFAULT_GATE_THRESHOLD } from '../../../eval-core/verdict.js';
|
|
12
12
|
import { EVALUATION_REPORT_SCHEMA_VERSION } from '../../../eval-core/evaluation-reporting.js';
|
|
13
|
+
import { findSingleTreatmentDeprecatedSamplesHint, hasUsableSamplesPath, } from '../../../inputs/sample-locator.js';
|
|
13
14
|
function isDryRunReport(report) {
|
|
14
15
|
return Boolean(report && typeof report === 'object' && report.dryRun === true);
|
|
15
16
|
}
|
|
@@ -180,6 +181,23 @@ async function announceSavedReport({ report, filePath, reportsDir, values, lang,
|
|
|
180
181
|
}
|
|
181
182
|
async function runEval(_args, flags, lang) {
|
|
182
183
|
const { values, config, evalConfig } = parseRunConfig({ ...flags });
|
|
184
|
+
if (!values.batch && !hasUsableSamplesPath(config.samplesPath)) {
|
|
185
|
+
const treatmentRaw = typeof values.treatment === 'string' ? values.treatment : '';
|
|
186
|
+
const treatments = treatmentRaw.split(',').map((v) => v.trim()).filter(Boolean);
|
|
187
|
+
const deprecatedSamplesHint = !values.samples && !evalConfig?.samples && treatments.length === 1
|
|
188
|
+
? findSingleTreatmentDeprecatedSamplesHint(treatments[0], config.skillDir, process.cwd())
|
|
189
|
+
: null;
|
|
190
|
+
if (deprecatedSamplesHint) {
|
|
191
|
+
process.stderr.write(tCli('cli.common.deprecated_skill_samples_path', lang, {
|
|
192
|
+
oldPath: deprecatedSamplesHint.oldPath,
|
|
193
|
+
newPath: deprecatedSamplesHint.newPath,
|
|
194
|
+
}));
|
|
195
|
+
}
|
|
196
|
+
console.error(tCli('cli.common.error_prefix', lang, {
|
|
197
|
+
message: tCli('cli.common.samples_not_found', lang, { path: config.samplesPath }),
|
|
198
|
+
}));
|
|
199
|
+
throw new CliExit(1);
|
|
200
|
+
}
|
|
183
201
|
const { runEvaluation, runMultiple, runBatchEvaluation } = await import('../../../eval-workflows/run-evaluation.js');
|
|
184
202
|
config.onProgress = makeOnProgress(lang);
|
|
185
203
|
const repeatRaw = values.repeat;
|
|
@@ -377,8 +395,8 @@ export default class Eval extends BaseCommand {
|
|
|
377
395
|
}),
|
|
378
396
|
samples: Flags.string({
|
|
379
397
|
description: bilingual({
|
|
380
|
-
zh: '
|
|
381
|
-
en: 'Samples
|
|
398
|
+
zh: '用例文件路径。默认项目级 eval-samples.json,也接受 .yaml/.yml;单 treatment 时可自动发现 <skill>/.omk/。',
|
|
399
|
+
en: 'Samples path. Defaults to project-level eval-samples.json (also .yaml/.yml); single-treatment runs can auto-discover <skill>/.omk/.',
|
|
382
400
|
}),
|
|
383
401
|
}),
|
|
384
402
|
'skill-dir': Flags.string({
|
|
@@ -2,8 +2,8 @@ import { BaseCommand } from '../../oclif/base-command.js';
|
|
|
2
2
|
import type { SkillHealthReport } from '../../../observability/skill-health-analyzer.js';
|
|
3
3
|
/**
|
|
4
4
|
* observe-health 报告落盘:id / 文件名加 4 位随机段,根治「同秒两次 omk observe 直接覆盖、数据丢失」的 bug。
|
|
5
|
-
*
|
|
6
|
-
*
|
|
5
|
+
* 文件名统一为 `{run}.report.json`:目录表达 observe-health 域,文件表达这是可读报告。
|
|
6
|
+
* 落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
|
|
7
7
|
*/
|
|
8
8
|
export declare function persistObserveHealthReport(report: SkillHealthReport, outDir: string): {
|
|
9
9
|
id: string;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
2
|
-
import { resolve
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
3
|
import { Args, Flags } from '@oclif/core';
|
|
4
4
|
import { LANG_FLAG, bilingual } from '../../oclif/i18n.js';
|
|
5
5
|
import { BaseCommand } from '../../oclif/base-command.js';
|
|
@@ -8,17 +8,18 @@ import { tCli } from '../../lib/i18n.js';
|
|
|
8
8
|
import { parseLastWindow } from '../../lib/shared.js';
|
|
9
9
|
import { projectObserveHealthDir, globalObserveHealthDir } from '../../../eval-core/measurement-dirs.js';
|
|
10
10
|
import { indexObserveWrite } from '../../../eval-core/artifact-index.js';
|
|
11
|
+
import { reportFilePath, runFileSuffix } from '../../../eval-core/artifact-file-names.js';
|
|
12
|
+
import { migrateLegacyReportFiles } from '../../../eval-core/report-file-migration.js';
|
|
11
13
|
/**
|
|
12
14
|
* observe-health 报告落盘:id / 文件名加 4 位随机段,根治「同秒两次 omk observe 直接覆盖、数据丢失」的 bug。
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
+
* 文件名统一为 `{run}.report.json`:目录表达 observe-health 域,文件表达这是可读报告。
|
|
16
|
+
* 落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
|
|
15
17
|
*/
|
|
16
18
|
export function persistObserveHealthReport(report, outDir) {
|
|
17
19
|
mkdirSync(outDir, { recursive: true });
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
const
|
|
21
|
-
const jsonPath = join(outDir, `${id}.json`);
|
|
20
|
+
migrateLegacyReportFiles(outDir, 'observe-health');
|
|
21
|
+
const id = runFileSuffix();
|
|
22
|
+
const jsonPath = reportFilePath(outDir, id);
|
|
22
23
|
writeFileSync(jsonPath, JSON.stringify(report, null, 2));
|
|
23
24
|
indexObserveWrite(report, jsonPath, outDir, id);
|
|
24
25
|
return { id, jsonPath };
|
|
@@ -9,6 +9,7 @@ import { CliExit } from '../lib/cli-exit.js';
|
|
|
9
9
|
import { tCli } from '../lib/i18n.js';
|
|
10
10
|
import { projectReportsDir, globalReportsDir } from '../../eval-core/measurement-dirs.js';
|
|
11
11
|
import { loadSamples, parseYaml, listSampleFilesInDir } from '../../inputs/load-samples.js';
|
|
12
|
+
import { defaultFlatSkillSamplesFile, defaultSkillLocalSamplesFile, findFlatSkillSamplesPath, findSkillSamplesPath, } from '../../inputs/sample-locator.js';
|
|
12
13
|
import { hashSample } from '../../eval-core/evaluation-reporting.js';
|
|
13
14
|
import { hashArtifactSource } from '../../inputs/content-hash.js';
|
|
14
15
|
function isRecord(value) {
|
|
@@ -233,23 +234,23 @@ async function runSampleFix(args, flags, lang) {
|
|
|
233
234
|
console.error(lang === 'zh' ? '请指定 skill 路径,如: omk sample skills/my-skill/SKILL.md --fix' : 'Specify skill path: omk sample skills/my-skill/SKILL.md --fix');
|
|
234
235
|
throw new CliExit(1);
|
|
235
236
|
}
|
|
236
|
-
const
|
|
237
|
-
|
|
238
|
-
|
|
237
|
+
const { resolveSkillInput } = await import('../lib/resolve-skill-input.js');
|
|
238
|
+
let resolvedInput;
|
|
239
|
+
try {
|
|
240
|
+
resolvedInput = resolveSkillInput(skillPath, lang);
|
|
241
|
+
}
|
|
242
|
+
catch (err) {
|
|
243
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
239
244
|
throw new CliExit(1);
|
|
240
245
|
}
|
|
241
|
-
const
|
|
242
|
-
const skillDir = isDir ? dirname(resolvedSkillPath) : dirname(resolvedSkillPath);
|
|
243
|
-
const samplesInput = isDir
|
|
244
|
-
? join(skillDir, '.omk')
|
|
245
|
-
: resolve('eval-samples.json');
|
|
246
|
+
const samplesInput = resolvedInput.samplesPath;
|
|
246
247
|
if (!existsSync(samplesInput)) {
|
|
247
248
|
console.error(lang === 'zh' ? `samples 路径不存在: ${samplesInput},先运行 omk sample 生成` : `Samples path not found: ${samplesInput}, run omk sample first`);
|
|
248
249
|
throw new CliExit(1);
|
|
249
250
|
}
|
|
250
|
-
const defaultTreatmentName =
|
|
251
|
-
? basename(skillDir)
|
|
252
|
-
: basename(
|
|
251
|
+
const defaultTreatmentName = resolvedInput.isDirectorySkill
|
|
252
|
+
? basename(resolvedInput.skillDir)
|
|
253
|
+
: basename(resolvedInput.skillPath, extname(resolvedInput.skillPath));
|
|
253
254
|
const treatmentName = flags.treatment ?? defaultTreatmentName;
|
|
254
255
|
process.stderr.write(lang === 'zh' ? `🔍 正在查找 ${treatmentName} 的最新评测报告...\n` : `🔍 Scanning latest report for ${treatmentName}...\n`);
|
|
255
256
|
// 显式 --reports-dir 固定该目录;默认 overlay(项目 .omk/reports 盖全局),findByVariant 记录优先看项目、
|
|
@@ -277,7 +278,7 @@ async function runSampleFix(args, flags, lang) {
|
|
|
277
278
|
throw new CliExit(1);
|
|
278
279
|
}
|
|
279
280
|
const samples = loadedSamples.samples;
|
|
280
|
-
const skillContent = readFileSync(
|
|
281
|
+
const skillContent = readFileSync(resolvedInput.skillPath, 'utf-8');
|
|
281
282
|
const sampleDesignIds = collectSampleDesignFailureIds(report, treatmentName);
|
|
282
283
|
const sampleDesignCount = sampleDesignIds.size;
|
|
283
284
|
if (sampleDesignCount === 0) {
|
|
@@ -286,7 +287,7 @@ async function runSampleFix(args, flags, lang) {
|
|
|
286
287
|
}
|
|
287
288
|
// 当前内容指纹走整树哈,与 eval 报告口径一致:dir-skill(用户传 .../SKILL.md)哈整棵 skill 目录、
|
|
288
289
|
// 单文件 .md 哈单文件字节。
|
|
289
|
-
const currentContentHash = hashArtifactSource(
|
|
290
|
+
const currentContentHash = hashArtifactSource(resolvedInput.isDirectorySkill ? resolvedInput.skillDir : resolvedInput.skillPath, resolvedInput.isDirectorySkill);
|
|
290
291
|
try {
|
|
291
292
|
assertFixReportMatchesCurrentInputs({
|
|
292
293
|
report,
|
|
@@ -435,24 +436,29 @@ async function runSample(args, flags, lang) {
|
|
|
435
436
|
let name;
|
|
436
437
|
let skillPath;
|
|
437
438
|
let samplesPath;
|
|
439
|
+
let existingSamplesPath;
|
|
438
440
|
const fullPath = join(skillDir, entry);
|
|
439
441
|
if (entry.endsWith('.md') && !entry.endsWith('.eval-samples.json')) {
|
|
440
442
|
name = entry.slice(0, -3);
|
|
441
443
|
skillPath = fullPath;
|
|
442
|
-
samplesPath =
|
|
444
|
+
samplesPath = defaultFlatSkillSamplesFile(skillDir, name);
|
|
445
|
+
existingSamplesPath = findFlatSkillSamplesPath(skillDir, name);
|
|
443
446
|
}
|
|
444
447
|
else if (statSync(fullPath).isDirectory()) {
|
|
445
448
|
const skillMd = join(fullPath, 'SKILL.md');
|
|
446
449
|
if (!existsSync(skillMd))
|
|
447
450
|
continue;
|
|
451
|
+
if (existsSync(join(skillDir, `${entry}.md`)))
|
|
452
|
+
continue;
|
|
448
453
|
name = entry;
|
|
449
454
|
skillPath = skillMd;
|
|
450
|
-
samplesPath =
|
|
455
|
+
samplesPath = defaultSkillLocalSamplesFile(fullPath);
|
|
456
|
+
existingSamplesPath = findSkillSamplesPath(fullPath);
|
|
451
457
|
}
|
|
452
458
|
else {
|
|
453
459
|
continue;
|
|
454
460
|
}
|
|
455
|
-
if (
|
|
461
|
+
if (existingSamplesPath) {
|
|
456
462
|
process.stderr.write(tCli('cli.gen.skill_skipped_existing', lang, { name }));
|
|
457
463
|
continue;
|
|
458
464
|
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { CliMessage } from './types.js';
|
|
2
|
-
export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.samples_detected' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
|
|
2
|
+
export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.common.deprecated_skill_samples_path' | 'cli.common.samples_not_found' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.samples_detected' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
|
|
3
3
|
export declare const commonDict: Record<CommonMessageKey, CliMessage>;
|
|
@@ -35,6 +35,14 @@ export const commonDict = {
|
|
|
35
35
|
zh: '⚠ 加载 samples 文件失败 ({path}): {message}\n',
|
|
36
36
|
en: '⚠ Failed to load samples file ({path}): {message}\n',
|
|
37
37
|
},
|
|
38
|
+
'cli.common.deprecated_skill_samples_path': {
|
|
39
|
+
zh: '⚠ 发现旧的目录 skill 用例位置:{oldPath}。目录 skill 的私有用例已改放到 {newPath},请迁移。\n',
|
|
40
|
+
en: '⚠ Found deprecated directory skill samples path: {oldPath}. Directory skill samples now live at {newPath}; please migrate.\n',
|
|
41
|
+
},
|
|
42
|
+
'cli.common.samples_not_found': {
|
|
43
|
+
zh: '未找到评测用例:{path}。请通过 --samples 指定文件,或创建项目级 eval-samples.json;单 treatment 目录 skill 请使用 <skill>/.omk/samples.json。',
|
|
44
|
+
en: 'Eval samples not found: {path}. Pass --samples, create project-level eval-samples.json, or use <skill>/.omk/samples.json for a single-treatment directory skill.',
|
|
45
|
+
},
|
|
38
46
|
'cli.update.new_version_available': {
|
|
39
47
|
zh: '\n💡 新版本可用:{old} → {new},运行 npm i -g oh-my-knowledge@latest 升级\n\n',
|
|
40
48
|
en: '\n💡 New version available: {old} → {new}, run npm i -g oh-my-knowledge@latest to upgrade\n\n',
|
|
@@ -170,20 +170,21 @@ omk sample——生成或补齐 eval-samples 评测用例
|
|
|
170
170
|
omk sample --batch [--skill-dir <dir>] [options]
|
|
171
171
|
|
|
172
172
|
输出位置(默认):
|
|
173
|
-
|
|
174
|
-
|
|
173
|
+
目录 skill(<skill>/SKILL.md) → <skill>/.omk/samples.json(omk 标准约定)
|
|
174
|
+
扁平 .md(单次) → 当前目录的 eval-samples.json(项目级兜底)
|
|
175
|
+
扁平 .md(--batch) → <skill-dir>/<name>.eval-samples.json(兼容 paired 布局)
|
|
175
176
|
|
|
176
177
|
选项:
|
|
177
178
|
--count <n> 强制生成 N 条(不指定时由 LLM 按 skill 类型自动判断:
|
|
178
179
|
工作流型 6-8 条 / 原子型 4-6 条 / 混合型 5-7 条)
|
|
179
180
|
--model <name> 生成模型(默认:opus;lean+effort-low 已自动开,想省钱可改 sonnet/haiku)
|
|
180
181
|
--focus <text> 自然语言指定希望覆盖的场景(追加到 prompt,优先级高于自由发挥)
|
|
181
|
-
--batch 为 skill 目录下缺少
|
|
182
|
+
--batch 为 skill 目录下缺少 samples 的 skill 批量生成
|
|
182
183
|
--skill-dir <path> skill 目录(batch 使用,默认:skills)
|
|
183
184
|
|
|
184
185
|
示例:
|
|
185
|
-
omk
|
|
186
|
-
omk
|
|
186
|
+
omk sample skills/req-tool.md
|
|
187
|
+
omk sample skills/req-tool.md --count 8 \\
|
|
187
188
|
--focus "重点覆盖 tag 查询走 PROJECT 空 → WORKSPACE 兜底的多步流程,以及 search 失败的错误路径"
|
|
188
189
|
`,
|
|
189
190
|
en: `
|
|
@@ -194,20 +195,21 @@ Usage:
|
|
|
194
195
|
omk sample --batch [--skill-dir <dir>] [options]
|
|
195
196
|
|
|
196
197
|
Output path (default):
|
|
197
|
-
<skill>/SKILL.md
|
|
198
|
-
|
|
198
|
+
directory skill (<skill>/SKILL.md) → <skill>/.omk/samples.json (omk standard layout)
|
|
199
|
+
flat .md (single) → ./eval-samples.json in current directory (project fallback)
|
|
200
|
+
flat .md (--batch) → <skill-dir>/<name>.eval-samples.json (compatible paired layout)
|
|
199
201
|
|
|
200
202
|
Options:
|
|
201
203
|
--count <n> Force N samples (omit to let LLM auto-decide by skill type:
|
|
202
204
|
workflow 6-8 / atomic 4-6 / mixed 5-7)
|
|
203
205
|
--model <name> Generation model (default: opus; lean+effort-low applied; pass --model sonnet/haiku to save cost)
|
|
204
206
|
--focus <text> Natural-language scenario hints appended to the prompt (overrides freeform diversity)
|
|
205
|
-
--batch Generate for skills that are missing
|
|
207
|
+
--batch Generate for skills that are missing samples
|
|
206
208
|
--skill-dir <path> Skill directory for batch mode (default: skills)
|
|
207
209
|
|
|
208
210
|
Examples:
|
|
209
|
-
omk
|
|
210
|
-
omk
|
|
211
|
+
omk sample skills/req-tool.md
|
|
212
|
+
omk sample skills/req-tool.md --count 8 \\
|
|
211
213
|
--focus "Cover PROJECT-empty → WORKSPACE-fallback multi-step tag lookup and the search-failure error path"
|
|
212
214
|
`,
|
|
213
215
|
},
|
|
@@ -1,18 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* 当 CLI 未传 `--samples` 时,自动发现 sample 路径。
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* 1. treatment
|
|
6
|
-
* 2.
|
|
7
|
-
* 3. fallback `<skillDir>/<treatmentName>/.omk/`
|
|
8
|
-
* 4. 终极兜底 cwd 下的 `eval-samples.{json,yaml,yml}`
|
|
4
|
+
* 发现顺序:
|
|
5
|
+
* 1. 单 treatment → 找该 skill 私有 samples(`<skill>/.omk/`)或扁平 skill paired 文件
|
|
6
|
+
* 2. 终极兜底 cwd 下的项目级 `eval-samples.{json,yaml,yml}`
|
|
9
7
|
*
|
|
10
8
|
* `loadSamples` 在下游自己处理「文件 vs 目录」—— 目录模式 glob `*.{json,yaml,yml}`
|
|
11
9
|
* 合并,跳过 reserved 前缀(`report-` / `health-` / `_`)。所以一个 skill 可以把
|
|
12
10
|
* sample 拆到多文件(`workflow.json` + `platform.json`),也可以单 `samples.json`,
|
|
13
11
|
* 两种都行。
|
|
14
12
|
*
|
|
15
|
-
* 多 treatment
|
|
16
|
-
* 没有唯一答案)
|
|
13
|
+
* 多 treatment 评测不会触发 skill-local 自动发现(因为「找哪个 skill 的 bundled samples」
|
|
14
|
+
* 没有唯一答案),只回到项目级 samples 兜底。
|
|
17
15
|
*/
|
|
18
16
|
export declare function discoverSamplesPath(values: Record<string, unknown>, skillDir: string): string;
|
|
@@ -1,50 +1,28 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* 当 CLI 未传 `--samples` 时,自动发现 sample 路径。
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* 1. treatment
|
|
6
|
-
* 2.
|
|
7
|
-
* 3. fallback `<skillDir>/<treatmentName>/.omk/`
|
|
8
|
-
* 4. 终极兜底 cwd 下的 `eval-samples.{json,yaml,yml}`
|
|
4
|
+
* 发现顺序:
|
|
5
|
+
* 1. 单 treatment → 找该 skill 私有 samples(`<skill>/.omk/`)或扁平 skill paired 文件
|
|
6
|
+
* 2. 终极兜底 cwd 下的项目级 `eval-samples.{json,yaml,yml}`
|
|
9
7
|
*
|
|
10
8
|
* `loadSamples` 在下游自己处理「文件 vs 目录」—— 目录模式 glob `*.{json,yaml,yml}`
|
|
11
9
|
* 合并,跳过 reserved 前缀(`report-` / `health-` / `_`)。所以一个 skill 可以把
|
|
12
10
|
* sample 拆到多文件(`workflow.json` + `platform.json`),也可以单 `samples.json`,
|
|
13
11
|
* 两种都行。
|
|
14
12
|
*
|
|
15
|
-
* 多 treatment
|
|
16
|
-
* 没有唯一答案)
|
|
13
|
+
* 多 treatment 评测不会触发 skill-local 自动发现(因为「找哪个 skill 的 bundled samples」
|
|
14
|
+
* 没有唯一答案),只回到项目级 samples 兜底。
|
|
17
15
|
*/
|
|
18
|
-
import {
|
|
19
|
-
import { existsSync, statSync } from 'node:fs';
|
|
16
|
+
import { findProjectSamplesFile, findSingleTreatmentSamplesPath } from '../../../inputs/sample-locator.js';
|
|
20
17
|
export function discoverSamplesPath(values, skillDir) {
|
|
21
18
|
const treatmentRaw = values.treatment;
|
|
22
19
|
const treatments = treatmentRaw
|
|
23
20
|
? treatmentRaw.split(',').map((v) => v.trim()).filter(Boolean)
|
|
24
21
|
: [];
|
|
25
22
|
if (treatments.length === 1) {
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
const treatmentDir = statSync(resolved).isDirectory() ? resolved : dirname(resolved);
|
|
30
|
-
const omkDir = join(treatmentDir, '.omk');
|
|
31
|
-
if (existsSync(omkDir))
|
|
32
|
-
return omkDir;
|
|
33
|
-
for (const name of ['eval-samples.json', 'eval-samples.yaml', 'eval-samples.yml']) {
|
|
34
|
-
if (existsSync(join(treatmentDir, name)))
|
|
35
|
-
return join(treatmentDir, name);
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
const omkDir = join(skillDir, expr, '.omk');
|
|
39
|
-
if (existsSync(omkDir))
|
|
40
|
-
return omkDir;
|
|
23
|
+
const samplesPath = findSingleTreatmentSamplesPath(treatments[0], skillDir, process.cwd());
|
|
24
|
+
if (samplesPath)
|
|
25
|
+
return samplesPath;
|
|
41
26
|
}
|
|
42
|
-
|
|
43
|
-
if (!existsSync(resolve(cwdFile))) {
|
|
44
|
-
if (existsSync(resolve('eval-samples.yaml')))
|
|
45
|
-
cwdFile = 'eval-samples.yaml';
|
|
46
|
-
else if (existsSync(resolve('eval-samples.yml')))
|
|
47
|
-
cwdFile = 'eval-samples.yml';
|
|
48
|
-
}
|
|
49
|
-
return cwdFile;
|
|
27
|
+
return findProjectSamplesFile(process.cwd()) ?? 'eval-samples.json';
|
|
50
28
|
}
|
|
@@ -41,10 +41,10 @@ export function parseRunConfig(values) {
|
|
|
41
41
|
? loadEvalConfig(values.config)
|
|
42
42
|
: null;
|
|
43
43
|
const skillDir = resolve(values['skill-dir'] ?? 'skills');
|
|
44
|
-
// 2) Resolve samples path: CLI > config >
|
|
45
|
-
//
|
|
46
|
-
// which skill's bundled samples to use. The dir form (loadSamples handles
|
|
47
|
-
// means a skill can split samples across multiple files
|
|
44
|
+
// 2) Resolve samples path: CLI > config > single-treatment skill-local discovery > cwd project default.
|
|
45
|
+
// Skill-local discovery only fires when exactly one --treatment is given, so omk knows
|
|
46
|
+
// which skill's bundled samples to use. The `<skill>/.omk/` dir form (loadSamples handles
|
|
47
|
+
// both file + dir) means a skill can split samples across multiple files.
|
|
48
48
|
const cliSamples = values.samples;
|
|
49
49
|
let samplesFile;
|
|
50
50
|
if (cliSamples) {
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { resolve, join, dirname, basename } from 'node:path';
|
|
2
2
|
import { existsSync, statSync } from 'node:fs';
|
|
3
3
|
import { tCli } from './i18n.js';
|
|
4
|
+
import { findFlatSkillSamplesPath, findProjectSamplesFile, findSkillSamplesPath, skillLocalSamplesDir, } from '../../inputs/sample-locator.js';
|
|
4
5
|
// 统一 skill 入参解析:既接受 SKILL.md 文件(老式 + flat skill),也接受 directory-skill
|
|
5
|
-
// 目录(skills/foo/ 自动找 foo/SKILL.md)
|
|
6
|
-
//
|
|
7
|
-
// .
|
|
6
|
+
// 目录(skills/foo/ 自动找 foo/SKILL.md)。目录-skill 以 `<skill>/.omk/` 为标准
|
|
7
|
+
// samples 命名空间;扁平 .md 兼容 paired sidecar,找不到时回到项目级
|
|
8
|
+
// `eval-samples.json`。
|
|
8
9
|
//
|
|
9
10
|
// 错误用 tCli 走 i18n,调用方直接 console.error err.message 给用户看,zh/en 都要正确。
|
|
10
11
|
export function resolveSkillInput(input, lang) {
|
|
@@ -26,16 +27,13 @@ export function resolveSkillInput(input, lang) {
|
|
|
26
27
|
skillPath = resolved;
|
|
27
28
|
skillDir = dirname(resolved);
|
|
28
29
|
}
|
|
29
|
-
const omkDir = join(skillDir, '.omk');
|
|
30
|
-
const candidates = [
|
|
31
|
-
...(existsSync(omkDir) ? [omkDir] : []),
|
|
32
|
-
join(skillDir, 'eval-samples.json'),
|
|
33
|
-
join(skillDir, 'eval-samples.yaml'),
|
|
34
|
-
join(skillDir, 'eval-samples.yml'),
|
|
35
|
-
];
|
|
36
|
-
// fallback 到 .omk/ 目录:上游会报 "no sample files found in directory" 引导用户创建
|
|
37
|
-
const samplesPath = candidates.find(existsSync) ?? omkDir;
|
|
38
30
|
// 形态以解析后的 skillPath 命名为准:目录-skill 的 skillPath 总是 `.../SKILL.md`。
|
|
39
31
|
const isDirectorySkill = basename(skillPath) === 'SKILL.md';
|
|
32
|
+
const flatSkillName = basename(skillPath).replace(/\.md$/i, '');
|
|
33
|
+
const samplesPath = isDirectorySkill
|
|
34
|
+
? findSkillSamplesPath(skillDir) ?? skillLocalSamplesDir(skillDir)
|
|
35
|
+
: findFlatSkillSamplesPath(skillDir, flatSkillName)
|
|
36
|
+
?? findProjectSamplesFile(process.cwd())
|
|
37
|
+
?? 'eval-samples.json';
|
|
40
38
|
return { skillPath, skillDir, samplesPath, isDirectorySkill };
|
|
41
39
|
}
|
package/dist/doctor/messages.js
CHANGED
|
@@ -133,8 +133,8 @@ export const DOCTOR_MESSAGES = {
|
|
|
133
133
|
en: 'directory-skill missing SKILL.md entry file',
|
|
134
134
|
},
|
|
135
135
|
'cli.doctor.skill_metadata.hint.frontmatter': {
|
|
136
|
-
zh: 'front-matter 用 YAML
|
|
137
|
-
en: 'front-matter uses YAML syntax (key: value or - item). See examples/
|
|
136
|
+
zh: 'front-matter 用 YAML 语法,key: value 或 - item 形式。可参考 examples/skill-map-showcase/skills/release-readiness 的目录式 skill 写法',
|
|
137
|
+
en: 'front-matter uses YAML syntax (key: value or - item). See examples/skill-map-showcase/skills/release-readiness for a directory-skill reference',
|
|
138
138
|
},
|
|
139
139
|
'cli.doctor.skill_metadata.hint.hardrules': {
|
|
140
140
|
zh: 'hardRules 必须写在 SKILL.md front-matter 中,格式为 hardRules: [{ id, rule, expectedBehavior }];id 要稳定且唯一,expectedBehavior 写可观察行为。',
|