oh-my-knowledge 0.26.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -346
- package/README.zh.md +142 -375
- package/dist/src/analysis/report-diagnostics.d.ts +2 -2
- package/dist/src/analysis/report-diagnostics.js +2 -2
- package/dist/src/analysis/sample-diagnostics.d.ts +3 -3
- package/dist/src/analysis/sample-diagnostics.js +3 -3
- package/dist/src/authoring/generator.d.ts.map +1 -1
- package/dist/src/authoring/generator.js +9 -8
- package/dist/src/authoring/generator.js.map +1 -1
- package/dist/src/cli/cli-exit.d.ts +15 -0
- package/dist/src/cli/cli-exit.d.ts.map +1 -0
- package/dist/src/cli/cli-exit.js +19 -0
- package/dist/src/cli/cli-exit.js.map +1 -0
- package/dist/src/cli/commands/_shared.d.ts +12 -0
- package/dist/src/cli/commands/_shared.d.ts.map +1 -0
- package/dist/src/cli/commands/_shared.js +26 -0
- package/dist/src/cli/commands/_shared.js.map +1 -0
- package/dist/src/cli/commands/doctor.d.ts +2 -0
- package/dist/src/cli/commands/doctor.d.ts.map +1 -0
- package/dist/src/cli/commands/doctor.js +174 -0
- package/dist/src/cli/commands/doctor.js.map +1 -0
- package/dist/src/cli/commands/eval-gold.d.ts +2 -0
- package/dist/src/cli/commands/eval-gold.d.ts.map +1 -0
- package/dist/src/cli/commands/eval-gold.js +137 -0
- package/dist/src/cli/commands/eval-gold.js.map +1 -0
- package/dist/src/cli/commands/eval-runner.d.ts +2 -0
- package/dist/src/cli/commands/eval-runner.d.ts.map +1 -0
- package/dist/src/cli/commands/eval-runner.js +299 -0
- package/dist/src/cli/commands/eval-runner.js.map +1 -0
- package/dist/src/cli/commands/eval.d.ts +2 -0
- package/dist/src/cli/commands/eval.d.ts.map +1 -0
- package/dist/src/cli/commands/eval.js +11 -0
- package/dist/src/cli/commands/eval.js.map +1 -0
- package/dist/src/cli/commands/evolve.d.ts +2 -0
- package/dist/src/cli/commands/evolve.d.ts.map +1 -0
- package/dist/src/cli/commands/evolve.js +115 -0
- package/dist/src/cli/commands/evolve.js.map +1 -0
- package/dist/src/cli/commands/init.d.ts +2 -0
- package/dist/src/cli/commands/init.d.ts.map +1 -0
- package/dist/src/cli/commands/init.js +110 -0
- package/dist/src/cli/commands/init.js.map +1 -0
- package/dist/src/cli/commands/observe.d.ts +2 -0
- package/dist/src/cli/commands/observe.d.ts.map +1 -0
- package/dist/src/cli/commands/observe.js +77 -0
- package/dist/src/cli/commands/observe.js.map +1 -0
- package/dist/src/cli/commands/registry.d.ts +11 -0
- package/dist/src/cli/commands/registry.d.ts.map +1 -0
- package/dist/src/cli/commands/registry.js +24 -0
- package/dist/src/cli/commands/registry.js.map +1 -0
- package/dist/src/cli/commands/sample.d.ts +2 -0
- package/dist/src/cli/commands/sample.d.ts.map +1 -0
- package/dist/src/cli/commands/sample.js +121 -0
- package/dist/src/cli/commands/sample.js.map +1 -0
- package/dist/src/cli/commands/studio.d.ts +2 -0
- package/dist/src/cli/commands/studio.d.ts.map +1 -0
- package/dist/src/cli/commands/studio.js +76 -0
- package/dist/src/cli/commands/studio.js.map +1 -0
- package/dist/src/cli/i18n-dict.d.ts +2 -4
- package/dist/src/cli/i18n-dict.d.ts.map +1 -1
- package/dist/src/cli/i18n-dict.js +461 -675
- package/dist/src/cli/i18n-dict.js.map +1 -1
- package/dist/src/cli/index.js +43 -1516
- package/dist/src/cli/index.js.map +1 -1
- package/dist/src/cli/parse-run-config.d.ts +3 -4
- package/dist/src/cli/parse-run-config.d.ts.map +1 -1
- package/dist/src/cli/parse-run-config.js +5 -5
- package/dist/src/cli/parse-run-config.js.map +1 -1
- package/dist/src/cli/parse-strict.d.ts +0 -13
- package/dist/src/cli/parse-strict.d.ts.map +1 -1
- package/dist/src/cli/parse-strict.js +2 -1
- package/dist/src/cli/parse-strict.js.map +1 -1
- package/dist/src/doctor/health/builtin-dimensions.d.ts +11 -0
- package/dist/src/doctor/health/builtin-dimensions.d.ts.map +1 -0
- package/dist/src/doctor/health/builtin-dimensions.js +94 -0
- package/dist/src/doctor/health/builtin-dimensions.js.map +1 -0
- package/dist/src/doctor/health/composer.d.ts +19 -0
- package/dist/src/doctor/health/composer.d.ts.map +1 -0
- package/dist/src/doctor/health/composer.js +289 -0
- package/dist/src/doctor/health/composer.js.map +1 -0
- package/dist/src/doctor/health/dimension-registry.d.ts +13 -0
- package/dist/src/doctor/health/dimension-registry.d.ts.map +1 -0
- package/dist/src/doctor/health/dimension-registry.js +28 -0
- package/dist/src/doctor/health/dimension-registry.js.map +1 -0
- package/dist/src/doctor/health/dimension-spec.d.ts +46 -0
- package/dist/src/doctor/health/dimension-spec.d.ts.map +1 -0
- package/dist/src/doctor/health/dimension-spec.js +12 -0
- package/dist/src/doctor/health/dimension-spec.js.map +1 -0
- package/dist/src/doctor/health/parser.d.ts +27 -0
- package/dist/src/doctor/health/parser.d.ts.map +1 -0
- package/dist/src/doctor/health/parser.js +190 -0
- package/dist/src/doctor/health/parser.js.map +1 -0
- package/dist/src/doctor/health/prompt-builder.d.ts +22 -0
- package/dist/src/doctor/health/prompt-builder.d.ts.map +1 -0
- package/dist/src/doctor/health/prompt-builder.js +162 -0
- package/dist/src/doctor/health/prompt-builder.js.map +1 -0
- package/dist/src/doctor/health/register.d.ts +13 -0
- package/dist/src/doctor/health/register.d.ts.map +1 -0
- package/dist/src/doctor/health/register.js +20 -0
- package/dist/src/doctor/health/register.js.map +1 -0
- package/dist/src/doctor/html-renderer.d.ts +20 -0
- package/dist/src/doctor/html-renderer.d.ts.map +1 -0
- package/dist/src/doctor/html-renderer.js +366 -0
- package/dist/src/doctor/html-renderer.js.map +1 -0
- package/dist/src/doctor/index.d.ts +1 -1
- package/dist/src/doctor/index.d.ts.map +1 -1
- package/dist/src/doctor/index.js +57 -8
- package/dist/src/doctor/index.js.map +1 -1
- package/dist/src/doctor/preflight.d.ts +2 -2
- package/dist/src/doctor/preflight.js +2 -2
- package/dist/src/doctor/renderer.d.ts +4 -0
- package/dist/src/doctor/renderer.d.ts.map +1 -1
- package/dist/src/doctor/renderer.js +72 -18
- package/dist/src/doctor/renderer.js.map +1 -1
- package/dist/src/doctor/rules.d.ts +6 -5
- package/dist/src/doctor/rules.d.ts.map +1 -1
- package/dist/src/doctor/rules.js +4 -3
- package/dist/src/doctor/rules.js.map +1 -1
- package/dist/src/eval-core/fact-checker.js +1 -1
- package/dist/src/eval-core/fact-checker.js.map +1 -1
- package/dist/src/eval-core/layer-gates.d.ts +1 -1
- package/dist/src/eval-core/layer-gates.js +1 -1
- package/dist/src/eval-core/verdict.d.ts +4 -4
- package/dist/src/eval-core/verdict.d.ts.map +1 -1
- package/dist/src/eval-core/verdict.js +2 -2
- package/dist/src/eval-workflows/batch-evaluation-workflow.d.ts +15 -2
- package/dist/src/eval-workflows/batch-evaluation-workflow.d.ts.map +1 -1
- package/dist/src/eval-workflows/batch-evaluation-workflow.js +13 -2
- package/dist/src/eval-workflows/batch-evaluation-workflow.js.map +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts +6 -3
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.js +10 -9
- package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -1
- package/dist/src/eval-workflows/run-evaluation.d.ts +1 -1
- package/dist/src/eval-workflows/run-evaluation.d.ts.map +1 -1
- package/dist/src/eval-workflows/run-evaluation.js +13 -6
- package/dist/src/eval-workflows/run-evaluation.js.map +1 -1
- package/dist/src/executors/script.d.ts.map +1 -1
- package/dist/src/executors/script.js +16 -3
- package/dist/src/executors/script.js.map +1 -1
- package/dist/src/grading/debias-validate.d.ts +2 -2
- package/dist/src/grading/debias-validate.js +2 -2
- package/dist/src/grading/gold-cli.d.ts +2 -5
- package/dist/src/grading/gold-cli.d.ts.map +1 -1
- package/dist/src/grading/gold-cli.js +4 -8
- package/dist/src/grading/gold-cli.js.map +1 -1
- package/dist/src/grading/judge.d.ts +1 -1
- package/dist/src/renderer/html-renderer.js +1 -1
- package/dist/src/renderer/layout.js +5 -5
- package/dist/src/renderer/layout.js.map +1 -1
- package/dist/src/renderer/skill-health-renderer.d.ts +1 -1
- package/dist/src/renderer/skill-health-renderer.js +1 -1
- package/dist/src/renderer/summary.js +8 -8
- package/dist/src/server/report-server.d.ts.map +1 -1
- package/dist/src/server/report-server.js +8 -7
- package/dist/src/server/report-server.js.map +1 -1
- package/dist/src/types/doctor.d.ts +44 -6
- package/dist/src/types/doctor.d.ts.map +1 -1
- package/dist/src/types/doctor.js +3 -0
- package/dist/src/types/doctor.js.map +1 -1
- package/dist/src/types/eval.d.ts +1 -1
- package/dist/src/types/report.d.ts +2 -2
- package/dist/src/types/report.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/src/cli/coverage-renderer.d.ts +0 -15
- package/dist/src/cli/coverage-renderer.d.ts.map +0 -1
- package/dist/src/cli/coverage-renderer.js +0 -74
- package/dist/src/cli/coverage-renderer.js.map +0 -1
|
@@ -26,8 +26,8 @@ export declare function analyzeResults(report: Report, opts?: AnalyzeResultsOpti
|
|
|
26
26
|
* Missing fields are bucketed under the `unspecified` key in the relevant
|
|
27
27
|
* distribution map, so users see "I have N samples without difficulty declared".
|
|
28
28
|
*
|
|
29
|
-
*
|
|
30
|
-
* in grading / judge / verdict. See docs/sample-design-spec.md.
|
|
29
|
+
* Persisted on the report so studio can surface coverage gaps. Does NOT
|
|
30
|
+
* participate in grading / judge / verdict. See docs/sample-design-spec.md.
|
|
31
31
|
*/
|
|
32
32
|
export declare function buildSampleQualityAggregate(samples: Sample[]): SampleQualityAggregate;
|
|
33
33
|
export declare function generateAnalysisSummary(report: Report, lang?: Lang): string | undefined;
|
|
@@ -56,8 +56,8 @@ export function analyzeResults(report, opts = {}) {
|
|
|
56
56
|
* Missing fields are bucketed under the `unspecified` key in the relevant
|
|
57
57
|
* distribution map, so users see "I have N samples without difficulty declared".
|
|
58
58
|
*
|
|
59
|
-
*
|
|
60
|
-
* in grading / judge / verdict. See docs/sample-design-spec.md.
|
|
59
|
+
* Persisted on the report so studio can surface coverage gaps. Does NOT
|
|
60
|
+
* participate in grading / judge / verdict. See docs/sample-design-spec.md.
|
|
61
61
|
*/
|
|
62
62
|
export function buildSampleQualityAggregate(samples) {
|
|
63
63
|
const capabilityCoverage = {};
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
* (d) are cost / latency outliers (one sample spikes the budget by itself), or
|
|
17
17
|
* (e) fail across the board (likely broken sample, not a model deficiency).
|
|
18
18
|
*
|
|
19
|
-
*
|
|
20
|
-
* the SAMPLE SET
|
|
19
|
+
* Persisted on the report so studio can surface these to users iterating
|
|
20
|
+
* the SAMPLE SET (alongside iterating the SKILL).
|
|
21
21
|
*
|
|
22
22
|
* Output shape: a flat list of issues, each tagged with severity and the
|
|
23
23
|
* minimal evidence needed to act ("here are the matching prompts"). Issues are
|
|
@@ -75,7 +75,7 @@ export declare function normalizeCapability(raw: string): string;
|
|
|
75
75
|
type DiagnosticLang = 'zh' | 'en';
|
|
76
76
|
export declare function formatSampleIssue(issue: SampleIssue, lang?: DiagnosticLang): string;
|
|
77
77
|
/**
|
|
78
|
-
* Plain-text formatter for
|
|
78
|
+
* Plain-text formatter for sample-quality diagnostics output.
|
|
79
79
|
*/
|
|
80
80
|
export declare function formatSampleDiagnostics(diag: SampleDiagnosticReport, options?: {
|
|
81
81
|
topN?: number;
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
* (d) are cost / latency outliers (one sample spikes the budget by itself), or
|
|
17
17
|
* (e) fail across the board (likely broken sample, not a model deficiency).
|
|
18
18
|
*
|
|
19
|
-
*
|
|
20
|
-
* the SAMPLE SET
|
|
19
|
+
* Persisted on the report so studio can surface these to users iterating
|
|
20
|
+
* the SAMPLE SET (alongside iterating the SKILL).
|
|
21
21
|
*
|
|
22
22
|
* Output shape: a flat list of issues, each tagged with severity and the
|
|
23
23
|
* minimal evidence needed to act ("here are the matching prompts"). Issues are
|
|
@@ -376,7 +376,7 @@ export function formatSampleIssue(issue, lang = 'zh') {
|
|
|
376
376
|
}
|
|
377
377
|
}
|
|
378
378
|
/**
|
|
379
|
-
* Plain-text formatter for
|
|
379
|
+
* Plain-text formatter for sample-quality diagnostics output.
|
|
380
380
|
*/
|
|
381
381
|
export function formatSampleDiagnostics(diag, options = {}) {
|
|
382
382
|
const lines = [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../../../src/authoring/generator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../../../src/authoring/generator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AA+BhD,UAAU,sBAAsB;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,wBAAsB,eAAe,CAAC,EAAE,YAAY,EAAE,KAAS,EAAE,KAAqB,EAAE,YAAuB,EAAE,EAAE,sBAAsB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CA2C1L;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG;IAAE,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAoClF"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createExecutor, DEFAULT_MODEL } from '../executors/index.js';
|
|
2
|
-
const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据用户提供的 skill
|
|
2
|
+
const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据用户提供的 skill(系统提示词)内容,生成高质量的评测用例。
|
|
3
3
|
|
|
4
4
|
每个用例需包含以下字段:
|
|
5
5
|
- sample_id: 唯一标识,格式为 s001, s002, ...
|
|
@@ -7,16 +7,17 @@ const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据
|
|
|
7
7
|
- context: 可选,附加上下文信息(如代码片段、文档段落等),仅在需要时提供
|
|
8
8
|
- rubric: 评分标准,描述一个好的回答应该具备什么特征(1-2 句话)
|
|
9
9
|
- assertions: 2-3 个断言检查,可选类型:
|
|
10
|
-
- { "type": "contains", "value": "
|
|
11
|
-
- { "type": "not_contains", "value": "
|
|
12
|
-
- { "type": "regex", "pattern": "
|
|
10
|
+
- { "type": "contains", "value": "English keyword or code token", "weight": 1 }
|
|
11
|
+
- { "type": "not_contains", "value": "English phrase that should not appear", "weight": 0.5 }
|
|
12
|
+
- { "type": "regex", "pattern": "English|code|token", "weight": 1 }
|
|
13
13
|
|
|
14
14
|
要求:
|
|
15
|
-
1.
|
|
15
|
+
1. 评测用例应覆盖 skill 的不同能力维度
|
|
16
16
|
2. prompt 要贴近真实用户的使用场景
|
|
17
17
|
3. rubric 要具体,不要泛泛而谈
|
|
18
18
|
4. assertions 要有区分度,能检测出有无 skill 的差异
|
|
19
|
-
5.
|
|
19
|
+
5. assertions 的 value / pattern / values / reference 必须使用英文、数字或代码 token,不要使用中文关键词。
|
|
20
|
+
6. 断言应检测 skill 文档中的具体细节(如特定参数名、配置值、工作流步骤),而非通用知识。
|
|
20
21
|
避免使用 baseline 凭常识或搜索文件也能答对的断言(如 not_contains 通用错误写法)。
|
|
21
22
|
优先使用 contains 检测文档独有的术语、参数组合或特定值
|
|
22
23
|
|
|
@@ -32,7 +33,7 @@ export async function generateSamples({ skillContent, count = 5, model = DEFAULT
|
|
|
32
33
|
|
|
33
34
|
${skillContent}
|
|
34
35
|
|
|
35
|
-
请根据这个 skill 生成 ${count}
|
|
36
|
+
请根据这个 skill 生成 ${count} 个评测用例。直接输出 JSON 数组。`;
|
|
36
37
|
const result = await executor({ model, system: SYSTEM_PROMPT, prompt });
|
|
37
38
|
if (!result.ok) {
|
|
38
39
|
throw new Error(`generation failed: ${result.error || 'unknown error'}`);
|
|
@@ -57,7 +58,7 @@ ${skillContent}
|
|
|
57
58
|
// (see sanitizeGeneratedSamples).
|
|
58
59
|
const { stripped } = sanitizeGeneratedSamples(samples);
|
|
59
60
|
if (stripped.length > 0) {
|
|
60
|
-
process.stderr.write(`[omk
|
|
61
|
+
process.stderr.write(`[omk sample] LLM-output 含 ${stripped.length} 个非法元数据字段,已剥离避免污染:\n - ${stripped.join('\n - ')}\n`);
|
|
61
62
|
}
|
|
62
63
|
return { samples, costUSD: result.costUSD };
|
|
63
64
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generator.js","sourceRoot":"","sources":["../../../src/authoring/generator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAGtE,MAAM,aAAa,GAAG
|
|
1
|
+
{"version":3,"file":"generator.js","sourceRoot":"","sources":["../../../src/authoring/generator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAGtE,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;uCA2BiB,CAAC;AASxC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,aAAa,EAAE,YAAY,GAAG,QAAQ,EAA0B;IACvI,MAAM,QAAQ,GAAG,cAAc,CAAC,YAAY,CAAC,CAAC;IAE9C,MAAM,MAAM,GAAG;;EAEf,YAAY;;iBAEG,KAAK,sBAAsB,CAAC;IAE3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,CAAC;IAExE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,kEAAkE;IAClE,IAAI,OAAO,GAAG,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC;IACpC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAChE,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAChC,CAAC;IAED,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IAED,8EAA8E;IAC9E,kCAAkC;IAClC,MAAM,EAAE,QAAQ,EAAE,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACvD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6BAA6B,QAAQ,CAAC,MAAM,2BAA2B,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CACnG,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;AAC9C,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,wBAAwB,CAAC,OAAiB;IACxD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,kBAAkB,CAAC,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACvC,kEAAkE;QAClE,0DAA0D;QAC1D,IAAI,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChE,CAAC,CAAC,SAAS,GAAG,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QACrD,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,mDAAmD,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACrG,CAAC;QAED,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC/B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;gBACtG,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,iBAAiB,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC;gBACnE,OAAQ,CAA8B,CAAC,UAAU,CAAC;YACpD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,UAAoB,CAAC,EAAE,CAAC;YAChF,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC1E,OAAQ,CAA8B,CAAC,UAAU,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;YACnF,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,OAAO,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;YACjE,OAAQ,CAA6B,CAAC,SAAS,CAAC;QAClD,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,UAAoB,CAAC,EAAE,CAAC;YAChF,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC1E,OAAQ,CAA8B,CAAC,UAAU,CAAC;QACpD,CAAC;QACD,kFAAkF;QAClF,IAAI,CAAC,CAAC,CAAC,UAAU;YAAE,CAAC,CAAC,UAAU,GAAG,eAAe,CAAC;IACpD,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 命令退出信号。execute() / parse-strict / requireEvaluationReport 等想以
|
|
3
|
+
* 特定 exit code 终止时,throw 这个,不要直接 process.exit。
|
|
4
|
+
*
|
|
5
|
+
* dispatcher (`main()` 顶层 catch) 负责把 CliExit 转成 process.exit(code) —
|
|
6
|
+
* 这样 execute() 可以在单测里被 try/catch 捕获,不 kill 整个测试进程。
|
|
7
|
+
*
|
|
8
|
+
* 异步回调内的「子进程 exit code 透传」。
|
|
9
|
+
* 不走这层,继续 process.exit — 那是子进程退出后的 cleanup,不在 main 调用栈。
|
|
10
|
+
*/
|
|
11
|
+
export declare class CliExit extends Error {
|
|
12
|
+
readonly code: number;
|
|
13
|
+
constructor(code: number);
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=cli-exit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli-exit.d.ts","sourceRoot":"","sources":["../../../src/cli/cli-exit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,qBAAa,OAAQ,SAAQ,KAAK;aACJ,IAAI,EAAE,MAAM;gBAAZ,IAAI,EAAE,MAAM;CAIzC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 命令退出信号。execute() / parse-strict / requireEvaluationReport 等想以
|
|
3
|
+
* 特定 exit code 终止时,throw 这个,不要直接 process.exit。
|
|
4
|
+
*
|
|
5
|
+
* dispatcher (`main()` 顶层 catch) 负责把 CliExit 转成 process.exit(code) —
|
|
6
|
+
* 这样 execute() 可以在单测里被 try/catch 捕获,不 kill 整个测试进程。
|
|
7
|
+
*
|
|
8
|
+
* 异步回调内的「子进程 exit code 透传」。
|
|
9
|
+
* 不走这层,继续 process.exit — 那是子进程退出后的 cleanup,不在 main 调用栈。
|
|
10
|
+
*/
|
|
11
|
+
export class CliExit extends Error {
|
|
12
|
+
code;
|
|
13
|
+
constructor(code) {
|
|
14
|
+
super(`CliExit(${code})`);
|
|
15
|
+
this.code = code;
|
|
16
|
+
this.name = 'CliExit';
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=cli-exit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli-exit.js","sourceRoot":"","sources":["../../../src/cli/cli-exit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,MAAM,OAAO,OAAQ,SAAQ,KAAK;IACJ;IAA5B,YAA4B,IAAY;QACtC,KAAK,CAAC,WAAW,IAAI,GAAG,CAAC,CAAC;QADA,SAAI,GAAJ,IAAI,CAAQ;QAEtC,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;IACxB,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type CliLang } from '../i18n.js';
|
|
2
|
+
import type { EvaluationReport, ReportDocument } from '../../types/index.js';
|
|
3
|
+
export interface EvalResult {
|
|
4
|
+
report: ReportDocument;
|
|
5
|
+
filePath: string | null;
|
|
6
|
+
}
|
|
7
|
+
export interface ReportServer {
|
|
8
|
+
start: () => Promise<string>;
|
|
9
|
+
}
|
|
10
|
+
export declare function requireEvaluationReport(report: ReportDocument | null, id: string, lang: CliLang): EvaluationReport;
|
|
11
|
+
export declare function parseLastWindow(spec: string): string | null;
|
|
12
|
+
//# sourceMappingURL=_shared.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"_shared.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/_shared.ts"],"names":[],"mappings":"AACA,OAAO,EAAQ,KAAK,OAAO,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE7E,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,cAAc,CAAC;IACvB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,gBAAgB,CAYlH;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAQ3D"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { CliExit } from '../cli-exit.js';
|
|
2
|
+
import { tCli } from '../i18n.js';
|
|
3
|
+
export function requireEvaluationReport(report, id, lang) {
|
|
4
|
+
if (!report) {
|
|
5
|
+
console.error(tCli('cli.common.report_not_found', lang, { id }));
|
|
6
|
+
throw new CliExit(1);
|
|
7
|
+
}
|
|
8
|
+
if (report.kind === 'batch-evaluation') {
|
|
9
|
+
console.error(lang === 'zh'
|
|
10
|
+
? `报告 ${id} 是 BatchEvaluationReport。该命令需要单次 EvaluationReport;请使用其中的 child reportId。`
|
|
11
|
+
: `Report ${id} is a BatchEvaluationReport. This command requires an EvaluationReport; use a child reportId from the batch.`);
|
|
12
|
+
throw new CliExit(1);
|
|
13
|
+
}
|
|
14
|
+
return report;
|
|
15
|
+
}
|
|
16
|
+
export function parseLastWindow(spec) {
|
|
17
|
+
// "7d" / "24h" / "30m" → ISO timestamp (from = now - spec)
|
|
18
|
+
const m = /^(\d+)([dhm])$/.exec(spec);
|
|
19
|
+
if (!m)
|
|
20
|
+
return null;
|
|
21
|
+
const n = Number(m[1]);
|
|
22
|
+
const unit = m[2];
|
|
23
|
+
const ms = unit === 'd' ? n * 86400_000 : unit === 'h' ? n * 3600_000 : n * 60_000;
|
|
24
|
+
return new Date(Date.now() - ms).toISOString();
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=_shared.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"_shared.js","sourceRoot":"","sources":["../../../../src/cli/commands/_shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAgB,MAAM,YAAY,CAAC;AAYhD,MAAM,UAAU,uBAAuB,CAAC,MAA6B,EAAE,EAAU,EAAE,IAAa;IAC9F,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,6BAA6B,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QACjE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;QACvC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI;YACzB,CAAC,CAAC,MAAM,EAAE,0EAA0E;YACpF,CAAC,CAAC,UAAU,EAAE,8GAA8G,CAAC,CAAC;QAChI,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,2DAA2D;IAC3D,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAClB,MAAM,EAAE,GAAG,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;IACnF,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACjD,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAoDA,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAgI3D"}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { existsSync, statSync } from 'node:fs';
|
|
2
|
+
import { dirname, join, resolve } from 'node:path';
|
|
3
|
+
import { CliExit } from '../cli-exit.js';
|
|
4
|
+
import { tCli, langFromArgv } from '../i18n.js';
|
|
5
|
+
import { COMMON_OPTIONS } from '../parse-run-config.js';
|
|
6
|
+
import { parseArgsStrictOrExit } from '../parse-strict.js';
|
|
7
|
+
const DEFAULT_SAMPLE_FILENAMES = ['eval-samples.json', 'eval-samples.yaml', 'eval-samples.yml'];
|
|
8
|
+
function findSamplesInDir(dir) {
|
|
9
|
+
for (const name of DEFAULT_SAMPLE_FILENAMES) {
|
|
10
|
+
const candidate = join(dir, name);
|
|
11
|
+
if (existsSync(candidate))
|
|
12
|
+
return candidate;
|
|
13
|
+
}
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
function sampleSearchDirs(target, cwd) {
|
|
17
|
+
const dirs = [];
|
|
18
|
+
const add = (dir) => {
|
|
19
|
+
const abs = resolve(dir);
|
|
20
|
+
if (!dirs.includes(abs))
|
|
21
|
+
dirs.push(abs);
|
|
22
|
+
};
|
|
23
|
+
if (target) {
|
|
24
|
+
const absTarget = resolve(target);
|
|
25
|
+
if (existsSync(absTarget)) {
|
|
26
|
+
const stat = statSync(absTarget);
|
|
27
|
+
if (stat.isDirectory()) {
|
|
28
|
+
add(absTarget);
|
|
29
|
+
add(dirname(absTarget));
|
|
30
|
+
add(dirname(dirname(absTarget)));
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
const parent = dirname(absTarget);
|
|
34
|
+
add(parent);
|
|
35
|
+
add(dirname(parent));
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
add(cwd);
|
|
40
|
+
return dirs;
|
|
41
|
+
}
|
|
42
|
+
function findDefaultSamplesPath(target, cwd) {
|
|
43
|
+
for (const dir of sampleSearchDirs(target, cwd)) {
|
|
44
|
+
const samplesPath = findSamplesInDir(dir);
|
|
45
|
+
if (samplesPath)
|
|
46
|
+
return samplesPath;
|
|
47
|
+
}
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
export async function execute(argv) {
|
|
51
|
+
const lang = langFromArgv(argv);
|
|
52
|
+
const { values, positionals } = parseArgsStrictOrExit({
|
|
53
|
+
args: argv,
|
|
54
|
+
allowPositionals: true,
|
|
55
|
+
options: {
|
|
56
|
+
...COMMON_OPTIONS,
|
|
57
|
+
json: { type: 'boolean', default: false },
|
|
58
|
+
gate: { type: 'boolean', default: false },
|
|
59
|
+
executor: { type: 'string' },
|
|
60
|
+
model: { type: 'string' },
|
|
61
|
+
samples: { type: 'string' },
|
|
62
|
+
timeout: { type: 'string' },
|
|
63
|
+
html: { type: 'string' },
|
|
64
|
+
'static-only': { type: 'boolean', default: false },
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
const target = positionals[0] ?? null;
|
|
68
|
+
const executorName = values.executor ?? 'claude';
|
|
69
|
+
const model = values.model ?? 'sonnet';
|
|
70
|
+
const timeoutRaw = values.timeout;
|
|
71
|
+
// 默认 LLM 健康度审计(7 内置维度 + 用户注册的自定义维度);--static-only 切到
|
|
72
|
+
// 离线静态模式:只跑 4 条静态 rule(skill_readable / skill_metadata /
|
|
73
|
+
// dependencies_present / samples_contract_aligned),不调 LLM。后者用于
|
|
74
|
+
// CI 节点没装 claude/codex、本地断网调试等场景。
|
|
75
|
+
const staticOnly = values['static-only'];
|
|
76
|
+
const runHealthCheck = !staticOnly;
|
|
77
|
+
// 单次 LLM 会话(7+N 维度,内部多 turn)默认 timeout 600s(10 min)。
|
|
78
|
+
const defaultTimeoutSec = 600;
|
|
79
|
+
const timeoutSec = timeoutRaw != null ? Number(timeoutRaw) : defaultTimeoutSec;
|
|
80
|
+
const timeoutMs = Math.max(1000, Math.floor((Number.isFinite(timeoutSec) ? timeoutSec : defaultTimeoutSec) * 1000));
|
|
81
|
+
const cwd = process.cwd();
|
|
82
|
+
const samplesPath = values.samples ? resolve(values.samples) : findDefaultSamplesPath(target, cwd);
|
|
83
|
+
let samples;
|
|
84
|
+
let requires;
|
|
85
|
+
if (samplesPath) {
|
|
86
|
+
try {
|
|
87
|
+
const { loadSamples } = await import('../../inputs/load-samples.js');
|
|
88
|
+
const loaded = loadSamples(samplesPath);
|
|
89
|
+
samples = loaded.samples;
|
|
90
|
+
requires = loaded.requires;
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
94
|
+
process.stderr.write(tCli('cli.common.warn_load_samples_failed', lang, { path: samplesPath, message: msg }));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
// 副作用 import: 注册 7 内置维度 spec + skill_health composer rule。
|
|
98
|
+
// 用户在自己代码 / plugin 里 import dimension-registry 注册自定义维度,
|
|
99
|
+
// 同样会被 composer 拼到 prompt(顺序 = 注册顺序)。
|
|
100
|
+
await import('../../doctor/health/register.js');
|
|
101
|
+
const { runDoctor } = await import('../../doctor/index.js');
|
|
102
|
+
const { renderDoctorReportText, renderDoctorReportJson } = await import('../../doctor/renderer.js');
|
|
103
|
+
// 默认模式过滤掉静态 rule 只留 composer;--static-only 反过来:只留静态 rule
|
|
104
|
+
// 跳过 composer。静态 rule 在 omk eval 里继续当强制 gate,doctor 这条线只
|
|
105
|
+
// 选择性暴露给用户。
|
|
106
|
+
const { getRegisteredRules } = await import('../../doctor/rules.js');
|
|
107
|
+
const { isComposerRule } = await import('../../types/doctor.js');
|
|
108
|
+
const rulesOverride = staticOnly
|
|
109
|
+
? getRegisteredRules().filter((r) => !isComposerRule(r))
|
|
110
|
+
: getRegisteredRules().filter(isComposerRule);
|
|
111
|
+
let report;
|
|
112
|
+
try {
|
|
113
|
+
report = await runDoctor({
|
|
114
|
+
target,
|
|
115
|
+
cwd,
|
|
116
|
+
executorName,
|
|
117
|
+
model,
|
|
118
|
+
timeoutMs,
|
|
119
|
+
lang,
|
|
120
|
+
runHealthCheck,
|
|
121
|
+
rules: rulesOverride,
|
|
122
|
+
samples,
|
|
123
|
+
requires,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
catch (err) {
|
|
127
|
+
// CliExit 透传(防御性,目前 runDoctor 不抛 CliExit,但保持四个 catch 一致)。
|
|
128
|
+
if (err instanceof CliExit)
|
|
129
|
+
throw err;
|
|
130
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
131
|
+
console.error(tCli('cli.doctor.no_skill_found', lang, { path: target ?? cwd }));
|
|
132
|
+
console.error(`(${msg})`);
|
|
133
|
+
throw new CliExit(1);
|
|
134
|
+
}
|
|
135
|
+
if (report.skills.length === 0) {
|
|
136
|
+
console.error(tCli('cli.doctor.no_skill_found', lang, { path: target ?? cwd }));
|
|
137
|
+
throw new CliExit(1);
|
|
138
|
+
}
|
|
139
|
+
const isJson = values.json;
|
|
140
|
+
const isGate = values.gate;
|
|
141
|
+
const htmlPath = values.html;
|
|
142
|
+
// --html 与 --json/--gate 可以共存:--html 写文件;同时 stdout/stderr 仍按
|
|
143
|
+
// 主输出格式跑(--json → JSON 到 stdout / --gate → 静默 / 默认 → text 到 stderr)。
|
|
144
|
+
if (htmlPath) {
|
|
145
|
+
const { renderDoctorReportHtml } = await import('../../doctor/html-renderer.js');
|
|
146
|
+
const { writeFileSync, mkdirSync } = await import('node:fs');
|
|
147
|
+
const { dirname, resolve } = await import('node:path');
|
|
148
|
+
const abs = resolve(htmlPath);
|
|
149
|
+
mkdirSync(dirname(abs), { recursive: true });
|
|
150
|
+
writeFileSync(abs, renderDoctorReportHtml(report, lang), 'utf8');
|
|
151
|
+
// 通知用户(stderr,不污染 --json 的 stdout)
|
|
152
|
+
console.error(lang === 'zh' ? `HTML 报告已写入: ${abs}` : `HTML report written to: ${abs}`);
|
|
153
|
+
}
|
|
154
|
+
if (isJson) {
|
|
155
|
+
console.log(renderDoctorReportJson(report));
|
|
156
|
+
}
|
|
157
|
+
else if (isGate) {
|
|
158
|
+
// gate 模式: 静默 stdout, fail 时简短 stderr 摘要(供 CI 抓 exit code)
|
|
159
|
+
if (report.outcome === 'failed') {
|
|
160
|
+
const summary = lang === 'zh'
|
|
161
|
+
? `doctor failed: ${report.totals.fail} 个 skill 未通过 (${report.totals.warn} warn / ${report.totals.pass} pass)`
|
|
162
|
+
: `doctor failed: ${report.totals.fail} skills did not pass (${report.totals.warn} warn / ${report.totals.pass} pass)`;
|
|
163
|
+
console.error(summary);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
if (samplesPath && samples) {
|
|
168
|
+
process.stderr.write(tCli('cli.doctor.samples_detected', lang, { path: samplesPath }) + '\n');
|
|
169
|
+
}
|
|
170
|
+
renderDoctorReportText(report, lang);
|
|
171
|
+
}
|
|
172
|
+
throw new CliExit(report.outcome === 'failed' ? 1 : 0);
|
|
173
|
+
}
|
|
174
|
+
//# sourceMappingURL=doctor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor.js","sourceRoot":"","sources":["../../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAI3D,MAAM,wBAAwB,GAAG,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,kBAAkB,CAAU,CAAC;AAEzG,SAAS,gBAAgB,CAAC,GAAW;IACnC,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAClC,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IAC9C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAqB,EAAE,GAAW;IAC1D,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,GAAG,GAAG,CAAC,GAAW,EAAQ,EAAE;QAChC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC;IACF,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,GAAG,CAAC,SAAS,CAAC,CAAC;gBACf,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;gBACxB,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;gBAClC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACZ,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IACD,GAAG,CAAC,GAAG,CAAC,CAAC;IACT,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAqB,EAAE,GAAW;IAChE,KAAK,MAAM,GAAG,IAAI,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,WAAW,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,WAAW;YAAE,OAAO,WAAW,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAc;IAC1C,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,qBAAqB,CAAC;QACpD,IAAI,EAAE,IAAI;QACV,gBAAgB,EAAE,IAAI;QACtB,OAAO,EAAE;YACP,GAAG,cAAc;YACjB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YACzC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YACzC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC5B,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzB,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;SACnD;KACF,CAAC,CAAC;IAEH,MAAM,MAAM,GAAkB,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IACrD,MAAM,YAAY,GAAI,MAAM,CAAC,QAA+B,IAAI,QAAQ,CAAC;IACzE,MAAM,KAAK,GAAI,MAAM,CAAC,KAA4B,IAAI,QAAQ,CAAC;IAC/D,MAAM,UAAU,GAAG,MAAM,CAAC,OAA6B,CAAC;IACxD,qDAAqD;IACrD,yDAAyD;IACzD,+DAA+D;IAC/D,kCAAkC;IAClC,MAAM,UAAU,GAAG,MAAM,CAAC,aAAa,CAAY,CAAC;IACpD,MAAM,cAAc,GAAG,CAAC,UAAU,CAAC;IACnC,qDAAqD;IACrD,MAAM,iBAAiB,GAAG,GAAG,CAAC;IAC9B,MAAM,UAAU,GAAG,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC;IAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,iBAAiB,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IACpH,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAiB,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7G,IAAI,OAA6B,CAAC;IAClC,IAAI,QAA4C,CAAC;IACjD,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,CAAC;YACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAC;YACrE,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;YACxC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YACzB,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,qCAAqC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAC/G,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,wDAAwD;IACxD,sCAAsC;IACtC,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAC;IAEhD,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;IAC5D,MAAM,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC,0BAA0B,CAAC,CAAC;IAEpG,yDAAyD;IACzD,yDAAyD;IACzD,YAAY;IACZ,MAAM,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;IACrE,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;IACjE,MAAM,aAAa,GAAG,UAAU;QAC9B,CAAC,CAAC,kBAAkB,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,kBAAkB,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAEhD,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,SAAS,CAAC;YACvB,MAAM;YACN,GAAG;YACH,YAAY;YACZ,KAAK;YACL,SAAS;YACT,IAAI;YACJ,cAAc;YACd,KAAK,EAAE,aAAa;YACpB,OAAO;YACP,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,0DAA0D;QAC1D,IAAI,GAAG,YAAY,OAAO;YAAE,MAAM,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QAChF,OAAO,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC;QAC1B,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAe,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAe,CAAC;IACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAA0B,CAAC;IAEnD,6DAA6D;IAC7D,qEAAqE;IACrE,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,EAAE,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QACjF,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7D,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC9B,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,aAAa,CAAC,GAAG,EAAE,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;QACjE,mCAAmC;QACnC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;IACzF,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,MAAM,EAAE,CAAC;QAClB,2DAA2D;QAC3D,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,IAAI,KAAK,IAAI;gBAC3B,CAAC,CAAC,kBAAkB,MAAM,CAAC,MAAM,CAAC,IAAI,iBAAiB,MAAM,CAAC,MAAM,CAAC,IAAI,WAAW,MAAM,CAAC,MAAM,CAAC,IAAI,QAAQ;gBAC9G,CAAC,CAAC,kBAAkB,MAAM,CAAC,MAAM,CAAC,IAAI,yBAAyB,MAAM,CAAC,MAAM,CAAC,IAAI,WAAW,MAAM,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC;YACzH,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,WAAW,IAAI,OAAO,EAAE,CAAC;YAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,6BAA6B,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QAChG,CAAC;QACD,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-gold.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval-gold.ts"],"names":[],"mappings":"AAQA,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA+G3D"}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
2
|
+
import { CliExit } from '../cli-exit.js';
|
|
3
|
+
import { langFromArgv } from '../i18n.js';
|
|
4
|
+
import { COMMON_OPTIONS, DEFAULT_REPORTS_DIR } from '../parse-run-config.js';
|
|
5
|
+
import { parseArgsStrictOrExit } from '../parse-strict.js';
|
|
6
|
+
import { requireEvaluationReport } from './_shared.js';
|
|
7
|
+
export async function execute(argv) {
|
|
8
|
+
const lang = langFromArgv(argv);
|
|
9
|
+
const [sub, ...rest] = argv;
|
|
10
|
+
if (!sub) {
|
|
11
|
+
console.log(usage(lang));
|
|
12
|
+
throw new CliExit(1);
|
|
13
|
+
}
|
|
14
|
+
if (sub === 'init') {
|
|
15
|
+
const { values } = parseArgsStrictOrExit({
|
|
16
|
+
args: rest,
|
|
17
|
+
options: {
|
|
18
|
+
...COMMON_OPTIONS,
|
|
19
|
+
out: { type: 'string', default: './gold-dataset' },
|
|
20
|
+
annotator: { type: 'string' },
|
|
21
|
+
},
|
|
22
|
+
});
|
|
23
|
+
const { initGoldDataset } = await import('../../grading/gold-cli.js');
|
|
24
|
+
try {
|
|
25
|
+
const written = initGoldDataset(values.out, {
|
|
26
|
+
annotator: values.annotator,
|
|
27
|
+
});
|
|
28
|
+
console.log(lang === 'zh'
|
|
29
|
+
? `已在 ${values.out} 创建 ${written.length} 个文件:`
|
|
30
|
+
: `Created ${written.length} files in ${values.out}:`);
|
|
31
|
+
for (const p of written)
|
|
32
|
+
console.log(` ${p}`);
|
|
33
|
+
console.log(lang === 'zh'
|
|
34
|
+
? '\n下一步:编辑 annotations.yaml 加入真实标注,然后运行 omk eval gold validate'
|
|
35
|
+
: '\nNext step: edit annotations.yaml with real annotations, then run omk eval gold validate');
|
|
36
|
+
}
|
|
37
|
+
catch (err) {
|
|
38
|
+
console.error(err.message);
|
|
39
|
+
throw new CliExit(1);
|
|
40
|
+
}
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
if (sub === 'validate') {
|
|
44
|
+
const { positionals } = parseArgsStrictOrExit({
|
|
45
|
+
args: rest,
|
|
46
|
+
allowPositionals: true,
|
|
47
|
+
options: { ...COMMON_OPTIONS },
|
|
48
|
+
});
|
|
49
|
+
const dir = positionals[0];
|
|
50
|
+
if (!dir) {
|
|
51
|
+
console.error('Usage: omk eval gold validate <dir>');
|
|
52
|
+
throw new CliExit(1);
|
|
53
|
+
}
|
|
54
|
+
const { validateGoldDataset } = await import('../../grading/gold-cli.js');
|
|
55
|
+
const result = validateGoldDataset(dir);
|
|
56
|
+
if (result.ok) {
|
|
57
|
+
console.log(lang === 'zh'
|
|
58
|
+
? `✓ gold dataset OK,共 ${result.sampleCount} 条标注`
|
|
59
|
+
: `✓ gold dataset OK — ${result.sampleCount} annotations`);
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
console.error(`✗ gold dataset has ${result.issues.length} issue(s):`);
|
|
63
|
+
for (const msg of result.issues)
|
|
64
|
+
console.error(` - ${msg}`);
|
|
65
|
+
throw new CliExit(1);
|
|
66
|
+
}
|
|
67
|
+
if (sub === 'compare') {
|
|
68
|
+
const reportId = rest[0];
|
|
69
|
+
if (!reportId) {
|
|
70
|
+
console.error('Usage: omk eval gold compare <reportId> --gold-dir <dir>');
|
|
71
|
+
throw new CliExit(1);
|
|
72
|
+
}
|
|
73
|
+
const { values } = parseArgsStrictOrExit({
|
|
74
|
+
args: rest.slice(1),
|
|
75
|
+
options: {
|
|
76
|
+
...COMMON_OPTIONS,
|
|
77
|
+
'gold-dir': { type: 'string' },
|
|
78
|
+
variant: { type: 'string' },
|
|
79
|
+
'reports-dir': { type: 'string', default: DEFAULT_REPORTS_DIR },
|
|
80
|
+
'bootstrap-samples': { type: 'string', default: '1000' },
|
|
81
|
+
seed: { type: 'string' },
|
|
82
|
+
},
|
|
83
|
+
});
|
|
84
|
+
const goldDir = values['gold-dir'];
|
|
85
|
+
if (!goldDir) {
|
|
86
|
+
console.error('--gold-dir is required');
|
|
87
|
+
throw new CliExit(1);
|
|
88
|
+
}
|
|
89
|
+
const { loadGoldDataset } = await import('../../grading/gold-dataset.js');
|
|
90
|
+
const { compareGoldToReport, formatGoldCompare } = await import('../../grading/gold-cli.js');
|
|
91
|
+
const { createFileStore } = await import('../../server/report-store.js');
|
|
92
|
+
const { dataset, issues } = loadGoldDataset(goldDir);
|
|
93
|
+
if (!dataset) {
|
|
94
|
+
console.error('Cannot load gold dataset:');
|
|
95
|
+
for (const i of issues)
|
|
96
|
+
console.error(` - ${i.message}`);
|
|
97
|
+
throw new CliExit(1);
|
|
98
|
+
}
|
|
99
|
+
for (const i of issues)
|
|
100
|
+
console.error(`warn: ${i.message}`);
|
|
101
|
+
const store = createFileStore(resolve(values['reports-dir']));
|
|
102
|
+
const report = requireEvaluationReport(await store.get(reportId), reportId, lang);
|
|
103
|
+
const samples = Math.max(100, Number(values['bootstrap-samples']) || 1000);
|
|
104
|
+
const seedVal = values.seed != null ? Number(values.seed) : undefined;
|
|
105
|
+
const result = compareGoldToReport({
|
|
106
|
+
report,
|
|
107
|
+
gold: dataset,
|
|
108
|
+
variant: values.variant,
|
|
109
|
+
samples,
|
|
110
|
+
seed: Number.isFinite(seedVal) ? seedVal : undefined,
|
|
111
|
+
});
|
|
112
|
+
console.log(formatGoldCompare(result, dataset));
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
console.error(`Unknown subcommand: eval gold ${sub}. Use init / validate / compare.`);
|
|
116
|
+
throw new CliExit(1);
|
|
117
|
+
}
|
|
118
|
+
function usage(lang) {
|
|
119
|
+
return lang === 'zh'
|
|
120
|
+
? [
|
|
121
|
+
'omk eval gold — 管理 human-gold 标注集',
|
|
122
|
+
'',
|
|
123
|
+
'用法:',
|
|
124
|
+
' omk eval gold init [--out <dir>] [--annotator <name>]',
|
|
125
|
+
' omk eval gold validate <dir>',
|
|
126
|
+
' omk eval gold compare <reportId> --gold-dir <dir>',
|
|
127
|
+
].join('\n')
|
|
128
|
+
: [
|
|
129
|
+
'omk eval gold — manage human-gold annotation datasets',
|
|
130
|
+
'',
|
|
131
|
+
'Usage:',
|
|
132
|
+
' omk eval gold init [--out <dir>] [--annotator <name>]',
|
|
133
|
+
' omk eval gold validate <dir>',
|
|
134
|
+
' omk eval gold compare <reportId> --gold-dir <dir>',
|
|
135
|
+
].join('\n');
|
|
136
|
+
}
|
|
137
|
+
//# sourceMappingURL=eval-gold.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-gold.js","sourceRoot":"","sources":["../../../../src/cli/commands/eval-gold.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAEvD,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAc;IAC1C,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACzB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACnB,MAAM,EAAE,MAAM,EAAE,GAAG,qBAAqB,CAAC;YACvC,IAAI,EAAE,IAAI;YACV,OAAO,EAAE;gBACP,GAAG,cAAc;gBACjB,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,EAAE;gBAClD,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC9B;SACF,CAAC,CAAC;QACH,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACtE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,GAAa,EAAE;gBACpD,SAAS,EAAE,MAAM,CAAC,SAA+B;aAClD,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI;gBACvB,CAAC,CAAC,MAAM,MAAM,CAAC,GAAa,OAAO,OAAO,CAAC,MAAM,OAAO;gBACxD,CAAC,CAAC,WAAW,OAAO,CAAC,MAAM,aAAa,MAAM,CAAC,GAAa,GAAG,CAAC,CAAC;YACnE,KAAK,MAAM,CAAC,IAAI,OAAO;gBAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI;gBACvB,CAAC,CAAC,8DAA8D;gBAChE,CAAC,CAAC,2FAA2F,CAAC,CAAC;QACnG,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAE,GAAa,CAAC,OAAO,CAAC,CAAC;YACtC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,GAAG,KAAK,UAAU,EAAE,CAAC;QACvB,MAAM,EAAE,WAAW,EAAE,GAAG,qBAAqB,CAAC;YAC5C,IAAI,EAAE,IAAI;YACV,gBAAgB,EAAE,IAAI;YACtB,OAAO,EAAE,EAAE,GAAG,cAAc,EAAE;SAC/B,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACrD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QAC1E,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI;gBACvB,CAAC,CAAC,uBAAuB,MAAM,CAAC,WAAW,MAAM;gBACjD,CAAC,CAAC,uBAAuB,MAAM,CAAC,WAAW,cAAc,CAAC,CAAC;YAC7D,OAAO;QACT,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,sBAAsB,MAAM,CAAC,MAAM,CAAC,MAAM,YAAY,CAAC,CAAC;QACtE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM;YAAE,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,0DAA0D,CAAC,CAAC;YAC1E,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,MAAM,EAAE,MAAM,EAAE,GAAG,qBAAqB,CAAC;YACvC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YACnB,OAAO,EAAE;gBACP,GAAG,cAAc;gBACjB,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC9B,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC3B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,mBAAmB,EAAE;gBAC/D,mBAAmB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;gBACxD,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aACzB;SACF,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAuB,CAAC;QACzD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;YACxC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QAC1E,MAAM,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QAC7F,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAC;QAEzE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAC3C,KAAK,MAAM,CAAC,IAAI,MAAM;gBAAE,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAC1D,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,MAAM;YAAE,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAE5D,MAAM,KAAK,GAAgB,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAW,CAAC,CAAC,CAAC;QACrF,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;QAC3E,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC;YACjC,MAAM;YACN,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,MAAM,CAAC,OAA6B;YAC7C,OAAO;YACP,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;SACrD,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QAChD,OAAO;IACT,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,iCAAiC,GAAG,kCAAkC,CAAC,CAAC;IACtF,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,KAAK,CAAC,IAAiB;IAC9B,OAAO,IAAI,KAAK,IAAI;QAClB,CAAC,CAAC;YACE,mCAAmC;YACnC,EAAE;YACF,KAAK;YACL,yDAAyD;YACzD,gCAAgC;YAChC,qDAAqD;SACtD,CAAC,IAAI,CAAC,IAAI,CAAC;QACd,CAAC,CAAC;YACE,uDAAuD;YACvD,EAAE;YACF,QAAQ;YACR,yDAAyD;YACzD,gCAAgC;YAChC,qDAAqD;SACtD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval-runner.ts"],"names":[],"mappings":"AA4KA,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA2L3D"}
|