oh-my-knowledge 0.37.0 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh.md +1 -1
- package/dist/assets/agent-skills/omk/SKILL.md +1 -1
- package/dist/assets/agent-skills/omk/references/commands.md +21 -12
- package/dist/authoring/evolver.d.ts +4 -1
- package/dist/authoring/evolver.js +13 -8
- package/dist/cli/commands/doctor.d.ts +1 -0
- package/dist/cli/commands/doctor.js +26 -6
- package/dist/cli/commands/eval/gold/compare.js +7 -4
- package/dist/cli/commands/eval/index.d.ts +1 -0
- package/dist/cli/commands/eval/index.js +14 -3
- package/dist/cli/commands/evolve.d.ts +1 -0
- package/dist/cli/commands/evolve.js +45 -0
- package/dist/cli/commands/init.js +18 -6
- package/dist/cli/commands/list.d.ts +0 -2
- package/dist/cli/commands/list.js +2 -7
- package/dist/cli/commands/observe/inbox.d.ts +1 -0
- package/dist/cli/commands/observe/inbox.js +14 -4
- package/dist/cli/commands/observe/index.d.ts +11 -0
- package/dist/cli/commands/observe/index.js +35 -12
- package/dist/cli/commands/observe/ingest.d.ts +1 -0
- package/dist/cli/commands/observe/ingest.js +13 -4
- package/dist/cli/commands/observe/show.d.ts +1 -0
- package/dist/cli/commands/observe/show.js +12 -2
- package/dist/cli/commands/promote.js +1 -2
- package/dist/cli/commands/sample.js +15 -9
- package/dist/cli/commands/studio.d.ts +2 -0
- package/dist/cli/commands/studio.js +62 -12
- package/dist/cli/lib/cmd-flags.d.ts +8 -0
- package/dist/cli/lib/i18n-dict/common.js +1 -1
- package/dist/cli/lib/i18n-dict/evolve.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/evolve.js +8 -0
- package/dist/cli/lib/i18n-dict/help.js +27 -45
- package/dist/cli/lib/i18n-dict/init.js +2 -2
- package/dist/cli/lib/parse-run-config.d.ts +0 -1
- package/dist/cli/lib/parse-run-config.js +6 -4
- package/dist/cli/lib/record-evolve-outcome.d.ts +31 -0
- package/dist/cli/lib/record-evolve-outcome.js +110 -0
- package/dist/cli/lib/resolve-skill-input.d.ts +6 -0
- package/dist/cli/lib/resolve-skill-input.js +4 -2
- package/dist/diagnosis/types.js +1 -1
- package/dist/doctor/index.js +4 -1
- package/dist/eval-core/artifact-index.d.ts +106 -0
- package/dist/eval-core/artifact-index.js +277 -0
- package/dist/eval-core/default-dirs.d.ts +34 -0
- package/dist/eval-core/default-dirs.js +36 -0
- package/dist/eval-core/evaluation-execution.js +3 -3
- package/dist/eval-core/evaluation-reporting.d.ts +7 -0
- package/dist/eval-core/evaluation-reporting.js +20 -5
- package/dist/eval-core/execution-strategy.js +3 -3
- package/dist/eval-core/measurement-dirs.d.ts +18 -0
- package/dist/eval-core/measurement-dirs.js +76 -0
- package/dist/eval-workflows/batch-evaluation-workflow.js +2 -1
- package/dist/eval-workflows/evaluation-pipeline/run-state.js +2 -1
- package/dist/eval-workflows/run-evaluation.js +5 -2
- package/dist/executors/codex-cli.js +2 -2
- package/dist/executors/script.js +43 -4
- package/dist/inputs/materialize-copy.d.ts +1 -1
- package/dist/inputs/materialize-copy.js +3 -3
- package/dist/inputs/skill-loader.d.ts +1 -1
- package/dist/inputs/skill-loader.js +1 -1
- package/dist/managed/index.d.ts +2 -0
- package/dist/managed/index.js +2 -0
- package/dist/managed/list-query.d.ts +7 -0
- package/dist/managed/list-query.js +11 -0
- package/dist/managed/source-probe.d.ts +15 -0
- package/dist/managed/source-probe.js +129 -0
- package/dist/managed/store.d.ts +10 -0
- package/dist/managed/store.js +25 -2
- package/dist/observability/inbox.js +6 -3
- package/dist/observability/skill-health-analyzer.d.ts +3 -0
- package/dist/observability/skill-health-analyzer.js +2 -0
- package/dist/renderer/html-renderer.js +4 -2
- package/dist/renderer/managed-history-renderer.d.ts +5 -0
- package/dist/renderer/managed-history-renderer.js +255 -0
- package/dist/renderer/observation-inbox-renderer.js +19 -19
- package/dist/renderer/skill-detail-renderer.js +17 -8
- package/dist/renderer/skill-list-renderer.js +1 -0
- package/dist/server/indexed-report-store.d.ts +8 -0
- package/dist/server/indexed-report-store.js +130 -0
- package/dist/server/job-store.d.ts +0 -1
- package/dist/server/job-store.js +0 -2
- package/dist/server/report-server.d.ts +17 -3
- package/dist/server/report-server.js +231 -85
- package/dist/server/report-store.d.ts +14 -8
- package/dist/server/report-store.js +46 -0
- package/dist/server/skill-index.d.ts +7 -1
- package/dist/server/skill-index.js +48 -12
- package/dist/types/executor.d.ts +4 -2
- package/dist/types/report.d.ts +15 -0
- package/dist/types/skill-index.d.ts +3 -0
- package/package.json +1 -1
|
@@ -20,8 +20,11 @@ function pickSkillToolCounts(value, skillName) {
|
|
|
20
20
|
}
|
|
21
21
|
// runObserveInbox export:test/cli/observe.test.ts in-process import 验证 by-skill 聚合行为。
|
|
22
22
|
export async function runObserveInbox(_args, flags, lang) {
|
|
23
|
-
const { queryObservationInbox, selectExploreInboxItems, loadLatestObservationInboxReports, summarizeObservationInboxBySkill, DEFAULT_OBSERVATIONS_DIR } = await import('../../../observability/inbox.js');
|
|
24
|
-
|
|
23
|
+
const { queryObservationInbox, selectExploreInboxItems, loadLatestObservationInboxReports, summarizeObservationInboxBySkill, DEFAULT_OBSERVATIONS_DIR, DEFAULT_GLOBAL_OBSERVATIONS_DIR } = await import('../../../observability/inbox.js');
|
|
24
|
+
// 显式 --input-dir 最高;否则 --global 直读全局、默认读项目(空则 loadObservationInboxReports 兜底全局)。
|
|
25
|
+
const dir = flags['input-dir']
|
|
26
|
+
? resolve(flags['input-dir'])
|
|
27
|
+
: (flags.global ? DEFAULT_GLOBAL_OBSERVATIONS_DIR : DEFAULT_OBSERVATIONS_DIR);
|
|
25
28
|
if (flags['llm-enhanced-review']) {
|
|
26
29
|
const { buildObservationInboxViewModel } = await import('../../../observability/inbox-view-model.js');
|
|
27
30
|
const { extractSkillSoftStandards, DEFAULT_LLM_ENHANCED_REVIEW_MODEL } = await import('../../../observability/soft-standards/index.js');
|
|
@@ -186,10 +189,17 @@ export default class ObserveInbox extends BaseCommand {
|
|
|
186
189
|
lang: LANG_FLAG,
|
|
187
190
|
'input-dir': Flags.string({
|
|
188
191
|
description: bilingual({
|
|
189
|
-
zh: 'inbox 数据目录,默认 .omk/
|
|
190
|
-
en: 'Inbox data dir, default .omk/
|
|
192
|
+
zh: 'inbox 数据目录,默认 .omk/observe-inbox(项目级,相对于 cwd);目录不存在时兜底读 ~/.oh-my-knowledge/observe-inbox。',
|
|
193
|
+
en: 'Inbox data dir, default .omk/observe-inbox (project-local); falls back to ~/.oh-my-knowledge/observe-inbox when missing.',
|
|
191
194
|
}),
|
|
192
195
|
}),
|
|
196
|
+
global: Flags.boolean({
|
|
197
|
+
description: bilingual({
|
|
198
|
+
zh: '直接读取全局 ~/.oh-my-knowledge/observe-inbox(跳过项目级与兜底)。',
|
|
199
|
+
en: 'Read directly from global ~/.oh-my-knowledge/observe-inbox (skip project-local and fallback).',
|
|
200
|
+
}),
|
|
201
|
+
default: false,
|
|
202
|
+
}),
|
|
193
203
|
skill: Flags.string({
|
|
194
204
|
description: bilingual({ zh: '只看指定 skill', en: 'Filter to specific skill' }),
|
|
195
205
|
}),
|
|
@@ -1,4 +1,14 @@
|
|
|
1
1
|
import { BaseCommand } from '../../oclif/base-command.js';
|
|
2
|
+
import type { SkillHealthReport } from '../../../observability/skill-health-analyzer.js';
|
|
3
|
+
/**
|
|
4
|
+
* observe-health 报告落盘:id / 文件名加 4 位随机段,根治「同秒两次 omk observe 直接覆盖、数据丢失」的 bug。
|
|
5
|
+
* 保留 `-observe-health.json` 后缀 —— resolveObserveHealthDir 靠它判项目优先、listAnalyses 靠它取 id stem、
|
|
6
|
+
* loadAnalysis 靠 `${id}.json` 读真身。落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
|
|
7
|
+
*/
|
|
8
|
+
export declare function persistObserveHealthReport(report: SkillHealthReport, outDir: string): {
|
|
9
|
+
id: string;
|
|
10
|
+
jsonPath: string;
|
|
11
|
+
};
|
|
2
12
|
export default class Observe extends BaseCommand {
|
|
3
13
|
static description: string;
|
|
4
14
|
static examples: {
|
|
@@ -16,6 +26,7 @@ export default class Observe extends BaseCommand {
|
|
|
16
26
|
to: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
17
27
|
skills: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
18
28
|
'output-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
29
|
+
global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
19
30
|
};
|
|
20
31
|
run(): Promise<void>;
|
|
21
32
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
1
2
|
import { resolve, join } from 'node:path';
|
|
2
3
|
import { Args, Flags } from '@oclif/core';
|
|
3
4
|
import { LANG_FLAG, bilingual } from '../../oclif/i18n.js';
|
|
@@ -5,12 +6,29 @@ import { BaseCommand } from '../../oclif/base-command.js';
|
|
|
5
6
|
import { CliExit } from '../../lib/cli-exit.js';
|
|
6
7
|
import { tCli } from '../../lib/i18n.js';
|
|
7
8
|
import { parseLastWindow } from '../../lib/shared.js';
|
|
8
|
-
|
|
9
|
-
|
|
9
|
+
import { projectObserveHealthDir, globalObserveHealthDir } from '../../../eval-core/measurement-dirs.js';
|
|
10
|
+
import { indexObserveWrite } from '../../../eval-core/artifact-index.js';
|
|
11
|
+
/**
|
|
12
|
+
* observe-health 报告落盘:id / 文件名加 4 位随机段,根治「同秒两次 omk observe 直接覆盖、数据丢失」的 bug。
|
|
13
|
+
* 保留 `-observe-health.json` 后缀 —— resolveObserveHealthDir 靠它判项目优先、listAnalyses 靠它取 id stem、
|
|
14
|
+
* loadAnalysis 靠 `${id}.json` 读真身。落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
|
|
15
|
+
*/
|
|
16
|
+
export function persistObserveHealthReport(report, outDir) {
|
|
17
|
+
mkdirSync(outDir, { recursive: true });
|
|
18
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
19
|
+
const rand = Math.random().toString(36).slice(2, 6);
|
|
20
|
+
const id = `${timestamp}-${rand}-observe-health`;
|
|
21
|
+
const jsonPath = join(outDir, `${id}.json`);
|
|
22
|
+
writeFileSync(jsonPath, JSON.stringify(report, null, 2));
|
|
23
|
+
indexObserveWrite(report, jsonPath, outDir, id);
|
|
24
|
+
return { id, jsonPath };
|
|
25
|
+
}
|
|
26
|
+
// `omk observe <sessions-dir>` 是默认命令 —— 分析 sessions 目录的 skill 调用健康度,产出 observe-health 报告(JSON),
|
|
27
|
+
// 由 Studio 健康报告页按需渲染。observe 这条线的另一条产物是观测收件箱(observe-inbox),走子命令 ingest / inbox / show。
|
|
10
28
|
export default class Observe extends BaseCommand {
|
|
11
29
|
static description = bilingual({
|
|
12
30
|
zh: '分析 sessions 目录的 skill 调用健康度(默认行为)。子命令:ingest / inbox / show。',
|
|
13
|
-
en: 'Analyze skill invocation health from sessions dir (default). Subcommands: ingest / inbox / show.',
|
|
31
|
+
en: 'Analyze skill invocation health from a sessions dir (default). Subcommands: ingest / inbox / show.',
|
|
14
32
|
});
|
|
15
33
|
static examples = [
|
|
16
34
|
{
|
|
@@ -52,8 +70,14 @@ export default class Observe extends BaseCommand {
|
|
|
52
70
|
}),
|
|
53
71
|
'output-dir': Flags.string({
|
|
54
72
|
description: bilingual({
|
|
55
|
-
zh: '
|
|
56
|
-
en: '
|
|
73
|
+
zh: '健康报告输出目录,默认项目级 .omk/observe-health(--global 写全局)',
|
|
74
|
+
en: 'Health report output dir, default project-level .omk/observe-health (--global for global)',
|
|
75
|
+
}),
|
|
76
|
+
}),
|
|
77
|
+
global: Flags.boolean({
|
|
78
|
+
description: bilingual({
|
|
79
|
+
zh: '写全局 ~/.oh-my-knowledge/observe-health,而非项目 .omk/observe-health',
|
|
80
|
+
en: 'Write to global ~/.oh-my-knowledge/observe-health instead of project .omk/observe-health',
|
|
57
81
|
}),
|
|
58
82
|
}),
|
|
59
83
|
};
|
|
@@ -67,7 +91,7 @@ export default class Observe extends BaseCommand {
|
|
|
67
91
|
throw new CliExit(1);
|
|
68
92
|
}
|
|
69
93
|
const tracePath = resolve(dir);
|
|
70
|
-
const { existsSync
|
|
94
|
+
const { existsSync } = await import('node:fs');
|
|
71
95
|
if (!existsSync(tracePath)) {
|
|
72
96
|
console.error(`Trace path does not exist: ${tracePath}`);
|
|
73
97
|
throw new CliExit(1);
|
|
@@ -92,12 +116,11 @@ export default class Observe extends BaseCommand {
|
|
|
92
116
|
to,
|
|
93
117
|
skills,
|
|
94
118
|
});
|
|
95
|
-
// JSON 是主产物;HTML 由 report server
|
|
96
|
-
const outDir =
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
const jsonPath =
|
|
100
|
-
writeFileSync(jsonPath, JSON.stringify(report, null, 2));
|
|
119
|
+
// JSON 是主产物;HTML 由 report server 的健康报告详情页按需渲染。
|
|
120
|
+
const outDir = flags['output-dir']
|
|
121
|
+
? resolve(flags['output-dir'])
|
|
122
|
+
: (flags.global ? globalObserveHealthDir() : projectObserveHealthDir());
|
|
123
|
+
const { jsonPath } = persistObserveHealthReport(report, outDir);
|
|
101
124
|
const { sessionCount, segmentCount, toolCallCount, toolFailureRate } = report.meta;
|
|
102
125
|
console.log('');
|
|
103
126
|
console.log(`sessions: ${sessionCount} · segments: ${segmentCount} · tool calls: ${toolCallCount} · fail rate: ${(toolFailureRate * 100).toFixed(1)}%`);
|
|
@@ -7,6 +7,7 @@ export default class ObserveIngest extends BaseCommand {
|
|
|
7
7
|
static flags: {
|
|
8
8
|
lang: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
9
9
|
'output-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
10
|
+
global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
10
11
|
};
|
|
11
12
|
run(): Promise<void>;
|
|
12
13
|
}
|
|
@@ -21,10 +21,17 @@ export default class ObserveIngest extends BaseCommand {
|
|
|
21
21
|
lang: LANG_FLAG,
|
|
22
22
|
'output-dir': Flags.string({
|
|
23
23
|
description: bilingual({
|
|
24
|
-
zh: '输出目录,默认 .omk/
|
|
25
|
-
en: 'Output dir, default .omk/
|
|
24
|
+
zh: '输出目录,默认 .omk/observe-inbox(项目级,相对于 cwd;--global 写全局)。',
|
|
25
|
+
en: 'Output dir, default .omk/observe-inbox (project-local, relative to cwd; --global writes global).',
|
|
26
26
|
}),
|
|
27
27
|
}),
|
|
28
|
+
global: Flags.boolean({
|
|
29
|
+
description: bilingual({
|
|
30
|
+
zh: '写入全局 ~/.oh-my-knowledge/observe-inbox,而非项目 .omk/observe-inbox。',
|
|
31
|
+
en: 'Write to global ~/.oh-my-knowledge/observe-inbox instead of project .omk/observe-inbox.',
|
|
32
|
+
}),
|
|
33
|
+
default: false,
|
|
34
|
+
}),
|
|
28
35
|
};
|
|
29
36
|
async run() {
|
|
30
37
|
const { args, flags } = await this.parse(ObserveIngest);
|
|
@@ -54,8 +61,10 @@ export default class ObserveIngest extends BaseCommand {
|
|
|
54
61
|
: 'Error: --output-dir must not be an empty string.');
|
|
55
62
|
throw new CliExit(2);
|
|
56
63
|
}
|
|
57
|
-
const { buildObservationInboxReport, saveObservationInboxReport,
|
|
58
|
-
|
|
64
|
+
const { buildObservationInboxReport, saveObservationInboxReport, DEFAULT_PROJECT_OBSERVATIONS_DIR, DEFAULT_GLOBAL_OBSERVATIONS_DIR } = await import('../../../observability/inbox.js');
|
|
65
|
+
// 显式 --output-dir 最高;否则 --global 写全局、默认写项目(读侧 loadObservationInboxReports 会从项目兜底到全局)。
|
|
66
|
+
const defaultDir = flags.global ? DEFAULT_GLOBAL_OBSERVATIONS_DIR : DEFAULT_PROJECT_OBSERVATIONS_DIR;
|
|
67
|
+
const outDir = resolve(outDirRaw ?? defaultDir);
|
|
59
68
|
const { loadObservationReviewState } = await import('../../../observability/review-state.js');
|
|
60
69
|
const { buildObserveDiagnosticsFromReport } = await import('../../../diagnosis/observe-producer.js');
|
|
61
70
|
const report = buildObservationInboxReport(tracePath, { reviewState: loadObservationReviewState(outDir) });
|
|
@@ -7,6 +7,7 @@ export default class ObserveShow extends BaseCommand {
|
|
|
7
7
|
static flags: {
|
|
8
8
|
lang: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
9
9
|
'input-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
10
|
+
global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
10
11
|
};
|
|
11
12
|
run(): Promise<void>;
|
|
12
13
|
}
|
|
@@ -25,6 +25,13 @@ export default class ObserveShow extends BaseCommand {
|
|
|
25
25
|
en: 'Inbox data dir',
|
|
26
26
|
}),
|
|
27
27
|
}),
|
|
28
|
+
global: Flags.boolean({
|
|
29
|
+
description: bilingual({
|
|
30
|
+
zh: '直接读取全局 ~/.oh-my-knowledge/observe-inbox(跳过项目级与兜底)。',
|
|
31
|
+
en: 'Read directly from global ~/.oh-my-knowledge/observe-inbox (skip project-local and fallback).',
|
|
32
|
+
}),
|
|
33
|
+
default: false,
|
|
34
|
+
}),
|
|
28
35
|
};
|
|
29
36
|
async run() {
|
|
30
37
|
const { args, flags } = await this.parse(ObserveShow);
|
|
@@ -35,8 +42,11 @@ export default class ObserveShow extends BaseCommand {
|
|
|
35
42
|
console.error(lang === 'zh' ? '用法:omk observe show <inbox_id> [--input-dir <path>]' : 'Usage: omk observe show <inbox_id> [--input-dir <path>]');
|
|
36
43
|
throw new CliExit(1);
|
|
37
44
|
}
|
|
38
|
-
const { findObservationInboxItem, formatObservationShow, DEFAULT_OBSERVATIONS_DIR } = await import('../../../observability/inbox.js');
|
|
39
|
-
|
|
45
|
+
const { findObservationInboxItem, formatObservationShow, DEFAULT_OBSERVATIONS_DIR, DEFAULT_GLOBAL_OBSERVATIONS_DIR } = await import('../../../observability/inbox.js');
|
|
46
|
+
// 显式 --input-dir 最高;否则 --global 直读全局、默认读项目(空则 findObservationInboxItem 兜底全局)。与 observe inbox 同口径。
|
|
47
|
+
const dir = flags['input-dir']
|
|
48
|
+
? resolve(flags['input-dir'])
|
|
49
|
+
: (flags.global ? DEFAULT_GLOBAL_OBSERVATIONS_DIR : DEFAULT_OBSERVATIONS_DIR);
|
|
40
50
|
const item = findObservationInboxItem(id, dir);
|
|
41
51
|
if (!item) {
|
|
42
52
|
console.error(lang === 'zh' ? `未找到 observation:${id}` : `Observation not found: ${id}`);
|
|
@@ -4,10 +4,9 @@ import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
|
|
|
4
4
|
import { BaseCommand } from '../oclif/base-command.js';
|
|
5
5
|
import { tCli } from '../lib/i18n.js';
|
|
6
6
|
import { CliExit } from '../lib/cli-exit.js';
|
|
7
|
-
import { probeSourceState } from '../lib/source-probe.js';
|
|
8
7
|
import { sanitizeCell } from '../lib/cell-format.js';
|
|
9
8
|
import { getJudgePromptHash } from '../../grading/judge.js';
|
|
10
|
-
import { appendManagedDecision, evaluatePromoteGate, globalManagedDir, isCurrentlyPromoted, loadManagedRecord, managedDir, managedRecordId, resolveManagedDir, } from '../../managed/index.js';
|
|
9
|
+
import { appendManagedDecision, evaluatePromoteGate, globalManagedDir, isCurrentlyPromoted, loadManagedRecord, probeSourceState, managedDir, managedRecordId, resolveManagedDir, } from '../../managed/index.js';
|
|
11
10
|
const SUPPORTED_KINDS = ['skill'];
|
|
12
11
|
/** 决定的 actor:--actor > git config user.name > $USER / $LOGNAME > unknown。git 参数无用户输入,安全。 */
|
|
13
12
|
function resolveActor(flagActor) {
|
|
@@ -7,7 +7,7 @@ import { BaseCommand } from '../oclif/base-command.js';
|
|
|
7
7
|
import { integerStringParser } from '../oclif/parsers.js';
|
|
8
8
|
import { CliExit } from '../lib/cli-exit.js';
|
|
9
9
|
import { tCli } from '../lib/i18n.js';
|
|
10
|
-
import {
|
|
10
|
+
import { projectReportsDir, globalReportsDir } from '../../eval-core/measurement-dirs.js';
|
|
11
11
|
import { loadSamples, parseYaml } from '../../inputs/load-samples.js';
|
|
12
12
|
import { hashSample } from '../../eval-core/evaluation-reporting.js';
|
|
13
13
|
import { hashArtifactSource } from '../../inputs/content-hash.js';
|
|
@@ -154,9 +154,8 @@ export function writeFixedSamplesToSources(loaded, samples, changedIds) {
|
|
|
154
154
|
}
|
|
155
155
|
async function runSampleFix(args, flags, lang) {
|
|
156
156
|
const { fixSamples } = await import('../../authoring/sample-fixer.js');
|
|
157
|
-
const { createFileStore } = await import('../../server/report-store.js');
|
|
157
|
+
const { createFileStore, createOverlayReportStore } = await import('../../server/report-store.js');
|
|
158
158
|
const model = flags.model;
|
|
159
|
-
const reportsDir = resolve(flags['reports-dir'] ?? DEFAULT_REPORTS_DIR);
|
|
160
159
|
const skillPath = args.skillPath;
|
|
161
160
|
if (!skillPath) {
|
|
162
161
|
console.error(lang === 'zh' ? '请指定 skill 路径,如: omk sample skills/my-skill/SKILL.md --fix' : 'Specify skill path: omk sample skills/my-skill/SKILL.md --fix');
|
|
@@ -181,10 +180,17 @@ async function runSampleFix(args, flags, lang) {
|
|
|
181
180
|
: basename(resolvedSkillPath, extname(resolvedSkillPath));
|
|
182
181
|
const treatmentName = flags.treatment ?? defaultTreatmentName;
|
|
183
182
|
process.stderr.write(lang === 'zh' ? `🔍 正在查找 ${treatmentName} 的最新评测报告...\n` : `🔍 Scanning latest report for ${treatmentName}...\n`);
|
|
184
|
-
|
|
183
|
+
// 显式 --reports-dir 固定该目录;默认 overlay(项目 .omk/reports 盖全局),findByVariant 记录优先看项目、
|
|
184
|
+
// 空则全局兜底,不因 eval 写默认翻项目而查不到报告。
|
|
185
|
+
const store = flags['reports-dir']
|
|
186
|
+
? createFileStore(resolve(flags['reports-dir']))
|
|
187
|
+
: createOverlayReportStore(projectReportsDir(), globalReportsDir());
|
|
185
188
|
const reports = await store.findByVariant(treatmentName);
|
|
186
189
|
if (reports.length === 0) {
|
|
187
|
-
|
|
190
|
+
const where = flags['reports-dir']
|
|
191
|
+
? resolve(flags['reports-dir'])
|
|
192
|
+
: (lang === 'zh' ? '项目 .omk/reports 或全局 ~/.oh-my-knowledge/reports' : 'project .omk/reports or global ~/.oh-my-knowledge/reports');
|
|
193
|
+
console.error(lang === 'zh' ? `未找到 ${treatmentName} 的评测报告(报告目录: ${where})` : `No eval report found for ${treatmentName} in ${where}`);
|
|
188
194
|
throw new CliExit(1);
|
|
189
195
|
}
|
|
190
196
|
const report = reports[0];
|
|
@@ -273,8 +279,8 @@ async function runSampleFromTraces(flags, lang) {
|
|
|
273
279
|
const obsDir = resolve(flags['observations-dir'] ?? DEFAULT_OBSERVATIONS_DIR);
|
|
274
280
|
if (!existsSync(obsDir)) {
|
|
275
281
|
console.error(lang === 'zh'
|
|
276
|
-
? `
|
|
277
|
-
: `
|
|
282
|
+
? `observe-inbox 目录不存在: ${obsDir}(先运行 omk observe ingest 生成)`
|
|
283
|
+
: `Observe-inbox dir not found: ${obsDir} (run omk observe ingest first)`);
|
|
278
284
|
throw new CliExit(1);
|
|
279
285
|
}
|
|
280
286
|
// Drop noise-tier signals up front: they're exactly what the generator is told to
|
|
@@ -582,8 +588,8 @@ export default class Sample extends BaseCommand {
|
|
|
582
588
|
}),
|
|
583
589
|
'observations-dir': Flags.string({
|
|
584
590
|
description: bilingual({
|
|
585
|
-
zh: 'observe inbox 目录(from-traces 模式用),默认项目 .omk/
|
|
586
|
-
en: 'Observe inbox dir (from-traces mode), default project .omk/
|
|
591
|
+
zh: 'observe inbox 目录(from-traces 模式用),默认项目 .omk/observe-inbox。',
|
|
592
|
+
en: 'Observe inbox dir (from-traces mode), default project .omk/observe-inbox.',
|
|
587
593
|
}),
|
|
588
594
|
}),
|
|
589
595
|
};
|
|
@@ -14,7 +14,9 @@ export default class Studio extends BaseCommand {
|
|
|
14
14
|
host: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
15
15
|
'reports-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
16
16
|
'analyses-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
17
|
+
'doctors-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
17
18
|
'observations-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
19
|
+
global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
18
20
|
'no-open': import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
19
21
|
dev: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
20
22
|
};
|
|
@@ -4,7 +4,9 @@ import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
|
|
|
4
4
|
import { BaseCommand } from '../oclif/base-command.js';
|
|
5
5
|
import { integerStringParser } from '../oclif/parsers.js';
|
|
6
6
|
import { tCli } from '../lib/i18n.js';
|
|
7
|
-
import {
|
|
7
|
+
import { resolveManagedDir, managedDir } from '../../managed/index.js';
|
|
8
|
+
import { resolveObserveHealthDir, projectObserveHealthDir, globalObserveHealthDir, resolveDoctorsDir, projectDoctorsDir, globalDoctorsDir, globalReportsDir, } from '../../eval-core/measurement-dirs.js';
|
|
9
|
+
import { DEFAULT_GLOBAL_OBSERVATIONS_DIR } from '../../observability/inbox.js';
|
|
8
10
|
async function openWorkbench(url, lang) {
|
|
9
11
|
const { execFile } = await import('node:child_process');
|
|
10
12
|
const { platform } = await import('node:os');
|
|
@@ -29,7 +31,13 @@ async function openWorkbench(url, lang) {
|
|
|
29
31
|
// import 直接调用。把业务作为 module-level helper export 从 Command file 暴露,
|
|
30
32
|
// 既保留 test 兼容又让产品命令树语义干净(无 legacy commands directory)。
|
|
31
33
|
export async function runStudio(_args, flags, lang) {
|
|
32
|
-
|
|
34
|
+
// reports 读取目录:显式 --reports-dir 固定该目录;--global 钉全局;默认 = undefined,交给
|
|
35
|
+
// createReportServer 建 indexed(机器级聚合:当前项目 + 全局 live ∪ 别项目索引卡片)。
|
|
36
|
+
const reportsDirOpt = flags['reports-dir']
|
|
37
|
+
? resolve(flags['reports-dir'])
|
|
38
|
+
: flags.global
|
|
39
|
+
? globalReportsDir()
|
|
40
|
+
: undefined;
|
|
33
41
|
if (flags.dev && !process.env.__OMK_DEV_CHILD) {
|
|
34
42
|
const { spawn } = await import('node:child_process');
|
|
35
43
|
const { fileURLToPath } = await import('node:url');
|
|
@@ -44,14 +52,22 @@ export async function runStudio(_args, flags, lang) {
|
|
|
44
52
|
'studio',
|
|
45
53
|
'--port', flags.port,
|
|
46
54
|
...(flags.host ? ['--host', flags.host] : []),
|
|
47
|
-
'--reports-dir', reportsDir,
|
|
48
55
|
];
|
|
56
|
+
if (flags['reports-dir']) {
|
|
57
|
+
childArgs.push('--reports-dir', flags['reports-dir']);
|
|
58
|
+
}
|
|
49
59
|
if (flags['analyses-dir']) {
|
|
50
60
|
childArgs.push('--analyses-dir', flags['analyses-dir']);
|
|
51
61
|
}
|
|
62
|
+
if (flags['doctors-dir']) {
|
|
63
|
+
childArgs.push('--doctors-dir', flags['doctors-dir']);
|
|
64
|
+
}
|
|
52
65
|
if (flags['observations-dir']) {
|
|
53
66
|
childArgs.push('--observations-dir', flags['observations-dir']);
|
|
54
67
|
}
|
|
68
|
+
if (flags.global) {
|
|
69
|
+
childArgs.push('--global');
|
|
70
|
+
}
|
|
55
71
|
if (flags['no-open']) {
|
|
56
72
|
childArgs.push('--no-open');
|
|
57
73
|
}
|
|
@@ -69,9 +85,31 @@ export async function runStudio(_args, flags, lang) {
|
|
|
69
85
|
const server = createReportServer({
|
|
70
86
|
port: Number(flags.port),
|
|
71
87
|
...(flags.host ? { host: flags.host } : {}),
|
|
72
|
-
reportsDir
|
|
73
|
-
|
|
74
|
-
...(
|
|
88
|
+
// reportsDir 缺省 undefined → server 建 indexed(机器级:当前项目 + 全局 + 别项目索引卡片);
|
|
89
|
+
// 显式 --reports-dir / --global 固定单目录(逃生舱,不走聚合)。
|
|
90
|
+
...(reportsDirOpt !== undefined ? { reportsDir: reportsDirOpt } : {}),
|
|
91
|
+
// 测量产物(observe-health / doctors)默认按请求项目优先→全局兜底(同 managed);
|
|
92
|
+
// 显式 --analyses-dir/--doctors-dir 固定该目录;--global 钉全局目录。
|
|
93
|
+
analysesDir: flags['analyses-dir']
|
|
94
|
+
? resolve(flags['analyses-dir'])
|
|
95
|
+
: (flags.global ? globalObserveHealthDir : () => resolveObserveHealthDir(projectObserveHealthDir())),
|
|
96
|
+
doctorsDir: flags['doctors-dir']
|
|
97
|
+
? resolve(flags['doctors-dir'])
|
|
98
|
+
: (flags.global ? globalDoctorsDir : () => resolveDoctorsDir(projectDoctorsDir())),
|
|
99
|
+
// 只有默认机器级模式才合别项目的索引卡片;--global / 显式 --analyses-dir / --doctors-dir 是逃生舱,
|
|
100
|
+
// 只看固定目录(与 reports 的 --reports-dir 关闭 indexed store 对称)。observe / doctor 各自独立判定。
|
|
101
|
+
includeObserveCards: !flags.global && !flags['analyses-dir'],
|
|
102
|
+
includeDoctorCards: !flags.global && !flags['doctors-dir'],
|
|
103
|
+
// observe-inbox 缺省按请求项目优先→全局兜底(report-server 默认即此);显式 --observations-dir 固定该目录;
|
|
104
|
+
// --global 钉全局(与 observe-health / doctors 的 --global 一致)。
|
|
105
|
+
...(flags['observations-dir']
|
|
106
|
+
? { observationsDir: resolve(flags['observations-dir']) }
|
|
107
|
+
: flags.global
|
|
108
|
+
? { observationsDir: DEFAULT_GLOBAL_OBSERVATIONS_DIR }
|
|
109
|
+
: {}),
|
|
110
|
+
// 传解析器而非解析结果:Studio 是长会话,受管根目录要按请求解析(项目首次 install 后从 global 切回
|
|
111
|
+
// project),与 omk list 同口径;若在此处一次性解析、冻结进 server,长会话里会与 CLI 分叉。
|
|
112
|
+
managedDir: () => resolveManagedDir(managedDir()),
|
|
75
113
|
});
|
|
76
114
|
const url = await server.start();
|
|
77
115
|
console.log(tCli('cli.studio.started', lang, { url }));
|
|
@@ -116,20 +154,32 @@ export default class Studio extends BaseCommand {
|
|
|
116
154
|
}),
|
|
117
155
|
'reports-dir': Flags.string({
|
|
118
156
|
description: bilingual({
|
|
119
|
-
zh: '
|
|
120
|
-
en: '
|
|
157
|
+
zh: '只看指定报告目录(可选;默认机器级聚合:当前项目 + 全局 + 别项目索引)',
|
|
158
|
+
en: 'View only this reports dir (optional; default aggregates machine-wide: current project + global + other projects via index)',
|
|
121
159
|
}),
|
|
122
160
|
}),
|
|
123
161
|
'analyses-dir': Flags.string({
|
|
124
162
|
description: bilingual({
|
|
125
|
-
zh: '
|
|
126
|
-
en: '
|
|
163
|
+
zh: '观测健康报告目录(可选,默认项目级 .omk/observe-health,空则全局兜底)',
|
|
164
|
+
en: 'Observe-health reports dir (optional, default project .omk/observe-health, falls back to global)',
|
|
165
|
+
}),
|
|
166
|
+
}),
|
|
167
|
+
'doctors-dir': Flags.string({
|
|
168
|
+
description: bilingual({
|
|
169
|
+
zh: '体检报告目录(可选,默认项目级 .omk/doctors,空则全局兜底)',
|
|
170
|
+
en: 'Doctor reports dir (optional, default project .omk/doctors, falls back to global)',
|
|
127
171
|
}),
|
|
128
172
|
}),
|
|
129
173
|
'observations-dir': Flags.string({
|
|
130
174
|
description: bilingual({
|
|
131
|
-
zh: '
|
|
132
|
-
en: '
|
|
175
|
+
zh: '观测收件箱数据目录(可选,默认 .omk/observe-inbox)',
|
|
176
|
+
en: 'Observe-inbox data dir (optional, default .omk/observe-inbox)',
|
|
177
|
+
}),
|
|
178
|
+
}),
|
|
179
|
+
global: Flags.boolean({
|
|
180
|
+
description: bilingual({
|
|
181
|
+
zh: '只看全局 reports / observe-health / doctors / observe-inbox 目录(~/.oh-my-knowledge/*),而非机器级聚合 / 项目优先;managed 不受影响',
|
|
182
|
+
en: 'View only global reports / observe-health / doctors / observe-inbox dirs (~/.oh-my-knowledge/*) instead of machine-wide / project-first; does not affect managed',
|
|
133
183
|
}),
|
|
134
184
|
}),
|
|
135
185
|
'no-open': Flags.boolean({
|
|
@@ -76,6 +76,8 @@ export interface EvalFlags {
|
|
|
76
76
|
executor?: string;
|
|
77
77
|
'judge-models'?: string;
|
|
78
78
|
'output-dir'?: string;
|
|
79
|
+
/** 报告写全局而非项目 .omk/reports(写入侧 escape;读取侧 studio / 复用走 overlay 兜底)。 */
|
|
80
|
+
global?: boolean;
|
|
79
81
|
'no-judge'?: boolean;
|
|
80
82
|
'no-cache'?: boolean;
|
|
81
83
|
'dry-run'?: boolean;
|
|
@@ -140,6 +142,7 @@ export interface EvolveFlags {
|
|
|
140
142
|
'edit-budget': string;
|
|
141
143
|
'no-edit-budget': boolean;
|
|
142
144
|
'no-reject-memory': boolean;
|
|
145
|
+
'snapshot-only': boolean;
|
|
143
146
|
}
|
|
144
147
|
export interface SampleArgs {
|
|
145
148
|
skillPath?: string;
|
|
@@ -200,11 +203,13 @@ export interface ObserveIngestArgs {
|
|
|
200
203
|
export interface ObserveIngestFlags {
|
|
201
204
|
lang: Lang;
|
|
202
205
|
'output-dir'?: string;
|
|
206
|
+
global: boolean;
|
|
203
207
|
}
|
|
204
208
|
export type ObserveInboxArgs = Record<string, never>;
|
|
205
209
|
export interface ObserveInboxFlags {
|
|
206
210
|
lang: Lang;
|
|
207
211
|
'input-dir'?: string;
|
|
212
|
+
global: boolean;
|
|
208
213
|
skill?: string;
|
|
209
214
|
limit?: string;
|
|
210
215
|
explore?: string;
|
|
@@ -222,6 +227,7 @@ export interface ObserveShowArgs {
|
|
|
222
227
|
export interface ObserveShowFlags {
|
|
223
228
|
lang: Lang;
|
|
224
229
|
'input-dir'?: string;
|
|
230
|
+
global: boolean;
|
|
225
231
|
}
|
|
226
232
|
export type StudioArgs = Record<string, never>;
|
|
227
233
|
export interface StudioFlags {
|
|
@@ -230,7 +236,9 @@ export interface StudioFlags {
|
|
|
230
236
|
host?: string;
|
|
231
237
|
'reports-dir'?: string;
|
|
232
238
|
'analyses-dir'?: string;
|
|
239
|
+
'doctors-dir'?: string;
|
|
233
240
|
'observations-dir'?: string;
|
|
241
|
+
global?: boolean;
|
|
234
242
|
'no-open': boolean;
|
|
235
243
|
dev: boolean;
|
|
236
244
|
}
|
|
@@ -56,7 +56,7 @@ export const commonDict = {
|
|
|
56
56
|
en: 'Silence: set {env} to disable',
|
|
57
57
|
},
|
|
58
58
|
'cli.observe.view_hint': {
|
|
59
|
-
zh: '分析 JSON 已写入 output-dir;后续可用 omk observe
|
|
59
|
+
zh: '分析 JSON 已写入 output-dir;后续可用 omk observe 持续生成健康报告。',
|
|
60
60
|
en: 'Analysis JSON written to output-dir; use omk observe to keep producing health reports.',
|
|
61
61
|
},
|
|
62
62
|
'cli.studio.started': {
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { CliMessage } from './types.js';
|
|
2
|
-
export type EvolveMessageKey = 'cli.evolve.specify_skill_path' | 'cli.evolve.section_header' | 'cli.evolve.round_baseline' | 'cli.evolve.round_baseline_reused' | 'cli.evolve.round_error' | 'cli.evolve.round_done' | 'cli.evolve.summary' | 'cli.evolve.best_path' | 'cli.evolve.versions_saved' | 'cli.evolve.report_link' | 'cli.evolve.holdout_active' | 'cli.evolve.holdout_disabled' | 'cli.evolve.gate_underpowered' | 'cli.evolve.generalization' | 'cli.evolve.test_disabled' | 'cli.evolve.reject_not_significant';
|
|
2
|
+
export type EvolveMessageKey = 'cli.evolve.specify_skill_path' | 'cli.evolve.section_header' | 'cli.evolve.round_baseline' | 'cli.evolve.round_baseline_reused' | 'cli.evolve.round_error' | 'cli.evolve.round_done' | 'cli.evolve.summary' | 'cli.evolve.best_path' | 'cli.evolve.versions_saved' | 'cli.evolve.report_link' | 'cli.evolve.holdout_active' | 'cli.evolve.holdout_disabled' | 'cli.evolve.gate_underpowered' | 'cli.evolve.generalization' | 'cli.evolve.test_disabled' | 'cli.evolve.reject_not_significant' | 'cli.evolve.evidence_recorded_managed' | 'cli.evolve.snapshot_only_hint';
|
|
3
3
|
export declare const evolveDict: Record<EvolveMessageKey, CliMessage>;
|
|
@@ -39,6 +39,14 @@ export const evolveDict = {
|
|
|
39
39
|
zh: '📊 查看报告:omk studio(报告 ID:{id})\n',
|
|
40
40
|
en: '📊 View report: omk studio (report id: {id})\n',
|
|
41
41
|
},
|
|
42
|
+
'cli.evolve.evidence_recorded_managed': {
|
|
43
|
+
zh: '📌 已记入受管记录:{name}(verdict={verdict})→ measurable。运行 omk promote {name} 接受为 promoted。\n',
|
|
44
|
+
en: '📌 Recorded to managed record: {name} (verdict={verdict}) → measurable. Run omk promote {name} to accept as promoted.\n',
|
|
45
|
+
},
|
|
46
|
+
'cli.evolve.snapshot_only_hint': {
|
|
47
|
+
zh: '📎 --snapshot-only:未写回 source,候选留在 {dir}/。挑选后手动应用,再 omk eval / omk promote。\n',
|
|
48
|
+
en: '📎 --snapshot-only: source not modified; candidates kept in {dir}/. Apply your pick manually, then omk eval / omk promote.\n',
|
|
49
|
+
},
|
|
42
50
|
'cli.evolve.holdout_active': {
|
|
43
51
|
zh: '🔀 holdout 切分:训练 {train} / 验收 {holdout},accept 看 holdout 分(防 train-on-test)\n',
|
|
44
52
|
en: '🔀 Holdout split: {train} train / {holdout} holdout; accepted on holdout score (guards train-on-test)\n',
|