oh-my-knowledge 0.44.0 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh.md +1 -1
- package/dist/artifact-graph/doctor.js +1 -1
- package/dist/artifact-graph/eval.js +120 -0
- package/dist/assets/agent-skills/omk/SKILL.md +3 -3
- package/dist/assets/agent-skills/omk/references/commands.md +13 -6
- package/dist/cli/commands/doctor.d.ts +3 -2
- package/dist/cli/commands/doctor.js +66 -63
- package/dist/cli/lib/cmd-flags.d.ts +3 -2
- package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/common.js +0 -4
- package/dist/doctor/endpoint-rule.js +1 -1
- package/dist/doctor/health/composer.js +162 -50
- package/dist/doctor/health/consensus.d.ts +43 -0
- package/dist/doctor/health/consensus.js +224 -0
- package/dist/doctor/health/dimension-spec.d.ts +6 -0
- package/dist/doctor/health/register.d.ts +0 -1
- package/dist/doctor/health/register.js +0 -1
- package/dist/doctor/index.js +26 -3
- package/dist/doctor/messages.d.ts +1 -1
- package/dist/doctor/messages.js +8 -0
- package/dist/doctor/rules.d.ts +4 -2
- package/dist/doctor/rules.js +4 -2
- package/dist/eval-core/evaluation-reporting.js +1 -0
- package/dist/executors/claude-cli.js +1 -1
- package/dist/inputs/load-samples.js +27 -0
- package/dist/observability/skill-chain-advisories.js +2 -2
- package/dist/renderer/doctor-detail-renderer.js +30 -2
- package/dist/renderer/report-shell.d.ts +1 -1
- package/dist/renderer/report-shell.js +5 -0
- package/dist/renderer/skill-detail-renderer.js +908 -94
- package/dist/renderer/skill-list-renderer.js +1 -1
- package/dist/server/skill-index.js +58 -1
- package/dist/shared/llm-prompts/registry.js +2 -1
- package/dist/shared/llm-prompts/skill-health-merge.d.ts +22 -0
- package/dist/shared/llm-prompts/skill-health-merge.js +79 -0
- package/dist/shared/llm-prompts/skill-health.js +1 -1
- package/dist/types/artifact-graph.d.ts +1 -1
- package/dist/types/doctor.d.ts +32 -8
- package/dist/types/eval.d.ts +10 -0
- package/dist/types/report.d.ts +3 -1
- package/dist/types/skill-index.d.ts +14 -0
- package/package.json +6 -5
|
@@ -134,7 +134,7 @@ function renderPanel(a, lang) {
|
|
|
134
134
|
// renderSkillModal 已移除:首页点行直接进 skill 详情页(doctor/eval/observe 已在该页合一),弹框预览冗余。
|
|
135
135
|
function renderRow(entry, insights, langQ, lang) {
|
|
136
136
|
const h = assessHealth(entry, insights, lang);
|
|
137
|
-
// 点行先进入 skill hub:Skill Map / 三阶段状态 /
|
|
137
|
+
// 点行先进入 skill hub:Skill Map / 三阶段状态 / 图谱摘要是用户第一视角。
|
|
138
138
|
const href = `/skills/${encodeURIComponent(entry.skillName)}${langQ}`;
|
|
139
139
|
const dT = entry.doctor ? entry.doctor.passCount + entry.doctor.warnCount + entry.doctor.failCount : 0;
|
|
140
140
|
const dP = dT > 0 ? Math.round(((entry.doctor.passCount + entry.doctor.warnCount * 0.5) / dT) * 100) : null;
|
|
@@ -412,6 +412,7 @@ function countGraphNodes(graph, nodeKind) {
|
|
|
412
412
|
}
|
|
413
413
|
function graphNodePreview(node) {
|
|
414
414
|
return {
|
|
415
|
+
stableKey: node.stableKey,
|
|
415
416
|
nodeKind: node.nodeKind,
|
|
416
417
|
label: node.label,
|
|
417
418
|
...(node.status ? { status: node.status } : {}),
|
|
@@ -474,6 +475,8 @@ function projectEvalStage(graph, path, entry) {
|
|
|
474
475
|
const nodesById = graphNodeMap(graph);
|
|
475
476
|
const projectedNodeIds = new Set([variantNode.id]);
|
|
476
477
|
const projectedEdgeIds = new Set();
|
|
478
|
+
const sampleStatusByNodeId = new Map();
|
|
479
|
+
const assertionStatusByNodeId = new Map();
|
|
477
480
|
if (skillNode)
|
|
478
481
|
projectedNodeIds.add(skillNode.id);
|
|
479
482
|
const addEdge = (edge) => {
|
|
@@ -484,6 +487,9 @@ function projectEvalStage(graph, path, entry) {
|
|
|
484
487
|
for (const edge of graph.edges) {
|
|
485
488
|
if (edge.fromNodeId === variantNode.id && (edge.edgeKind === 'evaluates' || edge.edgeKind === 'derived_from')) {
|
|
486
489
|
addEdge(edge);
|
|
490
|
+
const to = nodesById.get(edge.toNodeId);
|
|
491
|
+
if (edge.status && to?.nodeKind === 'sample')
|
|
492
|
+
sampleStatusByNodeId.set(to.id, edge.status);
|
|
487
493
|
}
|
|
488
494
|
}
|
|
489
495
|
const evalResultNodes = graph.edges
|
|
@@ -499,6 +505,36 @@ function projectEvalStage(graph, path, entry) {
|
|
|
499
505
|
addEdge(edge);
|
|
500
506
|
}
|
|
501
507
|
}
|
|
508
|
+
const declaredCoverageStableKeys = new Set();
|
|
509
|
+
const declaredCoverageEdges = [];
|
|
510
|
+
let coverageEdges = 0;
|
|
511
|
+
for (const edge of graph.edges) {
|
|
512
|
+
if (edge.edgeKind !== 'covers')
|
|
513
|
+
continue;
|
|
514
|
+
const from = nodesById.get(edge.fromNodeId);
|
|
515
|
+
const to = nodesById.get(edge.toNodeId);
|
|
516
|
+
if (from?.nodeKind !== 'sample' || !to)
|
|
517
|
+
continue;
|
|
518
|
+
if (!projectedNodeIds.has(from.id))
|
|
519
|
+
continue;
|
|
520
|
+
if (artifactHash && nodeArtifactHash(to) !== artifactHash)
|
|
521
|
+
continue;
|
|
522
|
+
addEdge(edge);
|
|
523
|
+
coverageEdges += 1;
|
|
524
|
+
if (to.stableKey)
|
|
525
|
+
declaredCoverageStableKeys.add(to.stableKey);
|
|
526
|
+
if (to.stableKey) {
|
|
527
|
+
const sampleStatus = sampleStatusByNodeId.get(from.id);
|
|
528
|
+
declaredCoverageEdges.push({
|
|
529
|
+
...(from.stableKey ? { sampleStableKey: from.stableKey } : {}),
|
|
530
|
+
sampleLabel: from.label,
|
|
531
|
+
...(sampleStatus ? { sampleStatus } : {}),
|
|
532
|
+
targetStableKey: to.stableKey,
|
|
533
|
+
targetNodeKind: to.nodeKind,
|
|
534
|
+
targetLabel: to.label,
|
|
535
|
+
});
|
|
536
|
+
}
|
|
537
|
+
}
|
|
502
538
|
const sampleIds = new Set();
|
|
503
539
|
const assertionIds = new Set();
|
|
504
540
|
const diagnosticIds = new Set();
|
|
@@ -522,13 +558,31 @@ function projectEvalStage(graph, path, entry) {
|
|
|
522
558
|
diagnosticIds.add(to.id);
|
|
523
559
|
if (edge.edgeKind === 'fails')
|
|
524
560
|
failedAssertionEdges += 1;
|
|
561
|
+
if ((edge.edgeKind === 'passes' || edge.edgeKind === 'fails') && edge.status) {
|
|
562
|
+
if (from?.nodeKind === 'assertion')
|
|
563
|
+
assertionStatusByNodeId.set(from.id, edge.status);
|
|
564
|
+
if (to?.nodeKind === 'assertion')
|
|
565
|
+
assertionStatusByNodeId.set(to.id, edge.status);
|
|
566
|
+
}
|
|
525
567
|
}
|
|
526
568
|
const measurementNodes = graphNodePreviews(graph, [
|
|
527
569
|
'sample',
|
|
528
570
|
'assertion',
|
|
529
571
|
'eval_result',
|
|
530
572
|
'diagnostic',
|
|
531
|
-
], projectedNodeIds)
|
|
573
|
+
], projectedNodeIds).map((node) => {
|
|
574
|
+
if (!node.stableKey)
|
|
575
|
+
return node;
|
|
576
|
+
const sourceNode = [...nodesById.values()].find((candidate) => candidate.stableKey === node.stableKey);
|
|
577
|
+
if (!sourceNode)
|
|
578
|
+
return node;
|
|
579
|
+
const status = sourceNode.nodeKind === 'sample'
|
|
580
|
+
? sampleStatusByNodeId.get(sourceNode.id)
|
|
581
|
+
: sourceNode.nodeKind === 'assertion'
|
|
582
|
+
? assertionStatusByNodeId.get(sourceNode.id)
|
|
583
|
+
: undefined;
|
|
584
|
+
return status ? { ...node, status } : node;
|
|
585
|
+
});
|
|
532
586
|
return {
|
|
533
587
|
stage: {
|
|
534
588
|
sourceKind: 'eval',
|
|
@@ -544,6 +598,9 @@ function projectEvalStage(graph, path, entry) {
|
|
|
544
598
|
failedAssertionEdges,
|
|
545
599
|
diagnostics: diagnosticIds.size,
|
|
546
600
|
measurementNodes,
|
|
601
|
+
coverageEdges,
|
|
602
|
+
declaredCoverageStableKeys: [...declaredCoverageStableKeys].sort(),
|
|
603
|
+
declaredCoverageEdges: declaredCoverageEdges.sort((a, b) => a.sampleLabel.localeCompare(b.sampleLabel) || a.targetStableKey.localeCompare(b.targetStableKey)),
|
|
547
604
|
},
|
|
548
605
|
...(artifactHash ? { artifactHash } : {}),
|
|
549
606
|
...(sourceLocator ? { sourceLocator } : {}),
|
|
@@ -65,5 +65,6 @@ export const PROMPT_REGISTRY = [
|
|
|
65
65
|
{ promptId: 'sample-fixer', purpose: '坏用例修复', module: 'src/authoring/sample-fixer.ts', measurementInvariant: false },
|
|
66
66
|
{ promptId: 'skill-improve', purpose: 'skill 迭代改进(evolve)', module: 'src/authoring/evolver.ts', measurementInvariant: false },
|
|
67
67
|
{ promptId: 'doctor-fixer', purpose: 'doctor 健康项修复向导', module: 'src/doctor/fixer.ts', measurementInvariant: false },
|
|
68
|
-
{ promptId: 'skill-health', purpose: 'skill
|
|
68
|
+
{ promptId: 'skill-health', purpose: 'skill 健康度审计(仅 CLI doctor,不进 eval 评分门禁)', module: 'src/shared/llm-prompts/skill-health.ts', measurementInvariant: false },
|
|
69
|
+
{ promptId: 'skill-health-merge', purpose: '多采样 finding 同根因归并(doctor 多采样默认 llm 归并)', module: 'src/shared/llm-prompts/skill-health-merge.ts', measurementInvariant: false },
|
|
69
70
|
];
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 多采样 finding 归并 prompt(option C:LLM 合并 pass)。
|
|
3
|
+
*
|
|
4
|
+
* 健康度体检跑 N 次后,同一个根因问题常被不同次用**不同措辞**描述。纯字符串键
|
|
5
|
+
* 去重(consensus.ts findingKey)对"措辞/引用写法不同"会漏并 —— 把同一问题拆成
|
|
6
|
+
* 多条低支持度 finding。本 prompt 让 LLM 做一次跨采样聚类,把同根因 + 同修复方向
|
|
7
|
+
* 的 finding 归到一组,框架再据此算 support(k/n)。
|
|
8
|
+
*
|
|
9
|
+
* 关键契约:
|
|
10
|
+
* - 只在维度内部并(不跨 dim_id),保持维度归属稳定。
|
|
11
|
+
* - 每个输入 finding 必须且只属于一个 cluster;无法并的自成一组。
|
|
12
|
+
* - 宁可不并,也不要把两个不同问题错并(over-merge 会藏掉真问题,比漏并更糟)。
|
|
13
|
+
* - LLM 只负责"哪些 id 归一组 + 合并后文案",**支持度 k/n 由框架算**(不信 LLM 算术)。
|
|
14
|
+
*/
|
|
15
|
+
import type { HealthDimensionSpec, HealthFinding } from '../../doctor/health/dimension-spec.js';
|
|
16
|
+
/** 一条带稳定 id 的采样 finding(来自第 sampleIdx 次采样、dimId 维度)。 */
|
|
17
|
+
export interface TaggedFindingView {
|
|
18
|
+
id: string;
|
|
19
|
+
dimId: string;
|
|
20
|
+
finding: HealthFinding;
|
|
21
|
+
}
|
|
22
|
+
export declare function buildHealthMergePrompt(dims: HealthDimensionSpec[], tagged: TaggedFindingView[]): string;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 多采样 finding 归并 prompt(option C:LLM 合并 pass)。
|
|
3
|
+
*
|
|
4
|
+
* 健康度体检跑 N 次后,同一个根因问题常被不同次用**不同措辞**描述。纯字符串键
|
|
5
|
+
* 去重(consensus.ts findingKey)对"措辞/引用写法不同"会漏并 —— 把同一问题拆成
|
|
6
|
+
* 多条低支持度 finding。本 prompt 让 LLM 做一次跨采样聚类,把同根因 + 同修复方向
|
|
7
|
+
* 的 finding 归到一组,框架再据此算 support(k/n)。
|
|
8
|
+
*
|
|
9
|
+
* 关键契约:
|
|
10
|
+
* - 只在维度内部并(不跨 dim_id),保持维度归属稳定。
|
|
11
|
+
* - 每个输入 finding 必须且只属于一个 cluster;无法并的自成一组。
|
|
12
|
+
* - 宁可不并,也不要把两个不同问题错并(over-merge 会藏掉真问题,比漏并更糟)。
|
|
13
|
+
* - LLM 只负责"哪些 id 归一组 + 合并后文案",**支持度 k/n 由框架算**(不信 LLM 算术)。
|
|
14
|
+
*/
|
|
15
|
+
export function buildHealthMergePrompt(dims, tagged) {
|
|
16
|
+
// 按维度分组,只列出有 finding 的维度。
|
|
17
|
+
const byDim = new Map();
|
|
18
|
+
for (const t of tagged) {
|
|
19
|
+
if (!byDim.has(t.dimId))
|
|
20
|
+
byDim.set(t.dimId, []);
|
|
21
|
+
byDim.get(t.dimId).push(t);
|
|
22
|
+
}
|
|
23
|
+
const sections = [];
|
|
24
|
+
for (const dim of dims) {
|
|
25
|
+
const items = byDim.get(dim.id);
|
|
26
|
+
if (!items || items.length === 0)
|
|
27
|
+
continue;
|
|
28
|
+
const lines = items.map((t) => {
|
|
29
|
+
const f = t.finding;
|
|
30
|
+
const sug = f.suggestion ? ` sug="${oneLine(f.suggestion)}"` : '';
|
|
31
|
+
return `- [${t.id}] level=${f.level} desc="${oneLine(f.description ?? '')}"${sug}`;
|
|
32
|
+
});
|
|
33
|
+
sections.push(`## 维度 dim_id=\`${dim.id}\`\n${lines.join('\n')}`);
|
|
34
|
+
}
|
|
35
|
+
const allIds = tagged.map((t) => `\`${t.id}\``).join(', ');
|
|
36
|
+
return `你在归并一个 skill 健康度体检的**多次采样结果**。下面是同一个 skill 跑了多次体检、各次产出的 finding(问题),按维度分组。同一个根因问题在不同次可能用**不同措辞**描述、引用文件名写法也可能不同。
|
|
37
|
+
|
|
38
|
+
# 任务
|
|
39
|
+
|
|
40
|
+
把**同根因 + 同修复方向**的 finding 归并成一组(cluster)。
|
|
41
|
+
|
|
42
|
+
# 规则
|
|
43
|
+
|
|
44
|
+
- **只在同一维度(dim_id)内部归并**,绝不跨维度合并。
|
|
45
|
+
- "同根因"判定:指向同一处缺陷、修复动作本质相同。措辞不同、引用文件名写法不同(如 \`templates/01-x.tmpl.md\` vs \`templates/<NN>-<name>.tmpl.md\`)**不影响**判定。
|
|
46
|
+
- **每个输入 finding 必须且只能属于一个 cluster**。和谁都并不了的,自成一个单元素 cluster。
|
|
47
|
+
- **宁可不并,也不要把两个不同的问题错并成一组** —— 错并会藏掉真问题,比漏并更糟。
|
|
48
|
+
- 每个 cluster 给一句最准确的合并后 \`description\` 和一条合并后 \`suggestion\`;\`level\` 取该组最严重的(错误 > 警告)。
|
|
49
|
+
|
|
50
|
+
# 输入
|
|
51
|
+
|
|
52
|
+
${sections.join('\n\n')}
|
|
53
|
+
|
|
54
|
+
# 输出 JSON schema
|
|
55
|
+
|
|
56
|
+
输出**必须是单一合法 JSON 对象**。第一字符 \`{\`,最后 \`}\`,**不要**用 \`\`\`json\`\`\` 围栏,**不要**寒暄。
|
|
57
|
+
|
|
58
|
+
{
|
|
59
|
+
"clusters": [
|
|
60
|
+
{
|
|
61
|
+
"dim_id": "<该组所属维度 id>",
|
|
62
|
+
"finding_ids": ["<本组包含的输入 finding id>"],
|
|
63
|
+
"level": "错误 | 警告",
|
|
64
|
+
"description": "<合并后一句话讲清这个根因>",
|
|
65
|
+
"suggestion": "<合并后的统一修改建议>"
|
|
66
|
+
}
|
|
67
|
+
]
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
# 约束
|
|
71
|
+
|
|
72
|
+
- 所有输入 id(${allIds})必须出现且仅出现一次,分布在各 cluster 的 \`finding_ids\` 里。
|
|
73
|
+
- \`finding_ids\` 同组的所有 finding 必须是**同一个 dim_id**。
|
|
74
|
+
- \`level\` 用精确中文(错误 / 警告),不要英文或别名。`;
|
|
75
|
+
}
|
|
76
|
+
function oneLine(s) {
|
|
77
|
+
// 截断:防恶意/超长 SKILL.md 内容原样 ×N 灌进 merge prompt(无界膨胀 + 注入面)。
|
|
78
|
+
return s.replace(/\s+/g, ' ').replace(/"/g, "'").trim().slice(0, 300);
|
|
79
|
+
}
|
|
@@ -92,7 +92,7 @@ function renderBlock3CommonRules() {
|
|
|
92
92
|
- 缺少显式声明但从上下文可合理推断的信息
|
|
93
93
|
- \`建议\`:**不要输出建议级别的 finding**。如果问题不够格做警告,就不要报。
|
|
94
94
|
|
|
95
|
-
|
|
95
|
+
**完整性:把该维度下**所有**够格(错误 / 警告)的真实问题都列出来,不要因为"已经报了几条"就漏掉同样真实的问题。同时严守上面的 level 门槛 —— 不够格的不要为了凑数硬塞。优先级:错误 > 警告。**
|
|
96
96
|
|
|
97
97
|
# 维度 level 派生规则
|
|
98
98
|
|
|
@@ -29,7 +29,7 @@ export type ArtifactGraphNodeRole = 'entity' | 'observation' | 'aggregate';
|
|
|
29
29
|
export type ArtifactGraphNodeKind = 'skill' | 'skill_file' | 'frontmatter' | 'reference' | 'script' | 'tool' | 'env' | 'preflight' | 'hard_rule' | 'workflow' | 'workflow_node' | 'sample' | 'assertion' | 'variant' | 'doctor_rule_result' | 'eval_result' | 'judge_dimension' | 'diagnostic' | 'trace_session' | 'skill_invocation' | 'tool_call' | 'gap_signal';
|
|
30
30
|
export type ArtifactGraphStatus = 'ok' | 'warning' | 'failed' | 'skipped' | 'unknown' | 'not_measured';
|
|
31
31
|
export interface ArtifactGraphBinding {
|
|
32
|
-
bindingStrength: 'content-hash' | 'source-locator' | 'runtime-trace' | 'name-only' | 'aggregate';
|
|
32
|
+
bindingStrength: 'content-hash' | 'source-locator' | 'runtime-trace' | 'name-only' | 'aggregate' | 'explicit';
|
|
33
33
|
keys: Record<string, string>;
|
|
34
34
|
}
|
|
35
35
|
export interface ArtifactGraphAttrs {
|
package/dist/types/doctor.d.ts
CHANGED
|
@@ -74,11 +74,25 @@ export interface DoctorContext {
|
|
|
74
74
|
lang: 'zh' | 'en';
|
|
75
75
|
timeoutMs: number;
|
|
76
76
|
effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
77
|
-
/** 深度健康检查(LLM-judge,多维度)开关。CLI doctor 默认 true
|
|
78
|
-
*
|
|
79
|
-
* skill_health composer 才真正调 LLM。
|
|
80
|
-
* import './doctor/health/register.js'
|
|
77
|
+
/** 深度健康检查(LLM-judge,多维度)开关。CLI `omk doctor` 默认 true,并与
|
|
78
|
+
* 静态 rule 一起运行;`--static-only` 置 false。programmatic API 默认 false,
|
|
79
|
+
* eval preflight 走静态 rule 路径。true 时 skill_health composer 才真正调 LLM。
|
|
80
|
+
* composer 不在 BUILTIN_RULES,必须先 import './doctor/health/register.js'
|
|
81
|
+
* 让 registerRule 副作用生效。 */
|
|
81
82
|
runHealthCheck?: boolean;
|
|
83
|
+
/** 健康度体检的采样次数(self-consistency)。composer 跑 N 次 LLM(默认并行),把各次
|
|
84
|
+
* finding 取并集去重,每条标注支持度 k/N(N 次里出现 k 次)。默认 1 = 单次采样;注意
|
|
85
|
+
* 即便 N=1 也走统一归并路径,框架重算 dim level / overall(不再保留 LLM 自报值,
|
|
86
|
+
* 与历史单次路径在"LLM 自报 level 与 finding 矛盾"时会有差异)。N>1 压低采样方差。 */
|
|
87
|
+
healthSamples?: number;
|
|
88
|
+
/** 多采样 finding 归并策略。`string`(默认)= 反引号锚点 / 归一化文本字符串键去重,
|
|
89
|
+
* 便宜无额外 LLM 调用,但同根因不同措辞可能漏并。`llm` = 多跑一次 LLM 聚类
|
|
90
|
+
* (skill-health-merge prompt),跨措辞归并最准,代价是 +1 次 LLM 调用;失败回退
|
|
91
|
+
* string。仅 healthSamples>1 时生效。 */
|
|
92
|
+
healthMerge?: 'string' | 'llm';
|
|
93
|
+
/** 多采样并发数。默认 = healthSamples(全并行,样本相互独立)。设 1 = 串行。
|
|
94
|
+
* 并发只压墙钟时间,不改成本(调用次数不变),但会抬高瞬时并发(rate-limit 敏感时调小)。 */
|
|
95
|
+
healthConcurrency?: number;
|
|
82
96
|
}
|
|
83
97
|
export interface DoctorRule {
|
|
84
98
|
id: string;
|
|
@@ -86,8 +100,9 @@ export interface DoctorRule {
|
|
|
86
100
|
/** i18n key,terminal 渲染时用作 rule 标题。 */
|
|
87
101
|
labelKey: string;
|
|
88
102
|
/** true = 需要外部 I/O(网络 / LLM)的"在线"检查,跟 skill_health composer 同档:
|
|
89
|
-
*
|
|
90
|
-
* 缺省(undefined/false)= 纯静态低成本检查(内置 4 条)
|
|
103
|
+
* CLI `omk doctor` 默认会跑(endpoint 自定义维度置 true),`--static-only` 会跳过。
|
|
104
|
+
* 缺省(undefined/false)= 纯静态低成本检查(内置 4 条),由默认 doctor、
|
|
105
|
+
* `--static-only` 与 eval preflight 共同复用。 */
|
|
91
106
|
external?: boolean;
|
|
92
107
|
check(ctx: DoctorContext): Promise<DoctorRuleCheckOutcome>;
|
|
93
108
|
}
|
|
@@ -168,9 +183,18 @@ export interface DoctorRunOptions {
|
|
|
168
183
|
* 既可以是普通 DoctorRule,也可以是 ComposerRule(健康度体检走这条)。 */
|
|
169
184
|
rules?: DoctorRuleLike[];
|
|
170
185
|
/** 深度健康检查(7 维 LLM-judge)。透传给 DoctorContext.runHealthCheck。
|
|
171
|
-
* CLI doctor
|
|
172
|
-
*
|
|
186
|
+
* CLI `omk doctor` 默认 true;`--static-only` 置 false。programmatic API 默认 false
|
|
187
|
+
* (eval preflight 只跑静态 rule)。 */
|
|
173
188
|
runHealthCheck?: boolean;
|
|
189
|
+
/** 健康度体检采样次数(self-consistency)。透传给 DoctorContext.healthSamples。
|
|
190
|
+
* 默认 1(单次,行为与历史一致)。CLI 用 `--repeat` 暴露。 */
|
|
191
|
+
healthSamples?: number;
|
|
192
|
+
/** 多采样 finding 归并策略,透传给 DoctorContext.healthMerge。programmatic 默认 `string`;
|
|
193
|
+
* CLI(`omk doctor`)恒传 `llm`(不暴露开关)。 */
|
|
194
|
+
healthMerge?: 'string' | 'llm';
|
|
195
|
+
/** 多采样并发数,透传给 DoctorContext.healthConcurrency。默认 = healthSamples(全并行)。
|
|
196
|
+
* CLI 用 `--concurrency` 暴露。 */
|
|
197
|
+
healthConcurrency?: number;
|
|
174
198
|
effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
175
199
|
/** 批量体检进度回调(per-skill)。CLI 非 gate 模式注入、写 stderr;eval 内嵌
|
|
176
200
|
* 调用不传(eval 有自己的进度体系,不应冒出 doctor 进度)。 */
|
package/dist/types/eval.d.ts
CHANGED
|
@@ -27,6 +27,14 @@ export interface Assertion {
|
|
|
27
27
|
export type SampleProvenance = 'human' | 'llm-generated' | 'production-trace';
|
|
28
28
|
/** sample 难度等级。简单分桶,跟 IRT 风格 fine-grained difficulty 不同。 */
|
|
29
29
|
export type SampleDifficulty = 'easy' | 'medium' | 'hard';
|
|
30
|
+
/** Sample 可选声明的 skill 结构锚点。用于 Skill Map / 诊断,不参与 grading / judge / verdict。 */
|
|
31
|
+
export type SampleCoverageTargetKind = 'skill' | 'skill_file' | 'frontmatter' | 'reference' | 'script' | 'hard_rule' | 'workflow' | 'workflow_node';
|
|
32
|
+
export interface SampleCoverageTarget {
|
|
33
|
+
/** 目标类型。避免裸 kind,保持 Artifact.kind 语义唯一。 */
|
|
34
|
+
targetKind: SampleCoverageTargetKind;
|
|
35
|
+
/** 稳定引用:路径或 ID。reference/script 用 skill 根相对路径,workflow_node 用 workflowId.nodeId。 */
|
|
36
|
+
ref: string;
|
|
37
|
+
}
|
|
30
38
|
/** Sample 评测环境前置:声明性"已就绪"清单,LLM 看到后跳过环境探测,直接进入工作流。
|
|
31
39
|
* 类比 unit test 的 fixture / setup —— 评测是测 skill 工作流,不是测环境探测能力。 */
|
|
32
40
|
export interface SampleEnvironment {
|
|
@@ -112,6 +120,8 @@ export interface Sample {
|
|
|
112
120
|
* curated 用 `'human'`,production trace 抽样用 `'production-trace'`。
|
|
113
121
|
* 纯文档 / 诊断用。 */
|
|
114
122
|
provenance?: SampleProvenance;
|
|
123
|
+
/** 该 sample 可选声明的 skill 结构锚点。纯文档 / 图谱诊断用,不进入评分、评委或 verdict。 */
|
|
124
|
+
covers?: SampleCoverageTarget[];
|
|
115
125
|
/** 诱错样本(tripwire)标记。true = 此 sample 故意设计成 LLM 应该 fail 的诱导陷阱
|
|
116
126
|
* (如:用户用错误前提诱导 / 跳步骤 / 用错参数类型),用于测 skill 是否能让 LLM
|
|
117
127
|
* 识破并纠正。Diagnostic 看到 tripwire:true 时会建议"无需改 skill"(rootCause:
|
package/dist/types/report.d.ts
CHANGED
|
@@ -346,6 +346,7 @@ export interface ResultEntry {
|
|
|
346
346
|
* - assertions: 期望(运行后的 pass/fail 结果在 VariantResult.assertions.details 里)
|
|
347
347
|
* - mocks: 工具调用模拟返回(LLM 调到匹配的 tool/参数时拿到这段假返回,而不是真去调外部系统)
|
|
348
348
|
* - capability / construct / difficulty: 元数据
|
|
349
|
+
* - covers: sample 可选声明的 skill 结构锚点
|
|
349
350
|
* - context: 附加上下文(代码片段等)
|
|
350
351
|
*
|
|
351
352
|
* 旧 report 没有这个字段,renderer 据此 fallback(隐藏单测 tab 或提示"无设计快照")。
|
|
@@ -361,6 +362,7 @@ export interface SampleSnapshot {
|
|
|
361
362
|
difficulty?: import('./eval.js').SampleDifficulty;
|
|
362
363
|
construct?: string;
|
|
363
364
|
provenance?: import('./eval.js').SampleProvenance;
|
|
365
|
+
covers?: import('./eval.js').SampleCoverageTarget[];
|
|
364
366
|
/** Diagnostic 用 — tripwire sample 不该建议改 skill。 */
|
|
365
367
|
tripwire?: boolean;
|
|
366
368
|
}
|
|
@@ -476,7 +478,7 @@ export interface AnalysisResult {
|
|
|
476
478
|
coverage?: Record<string, KnowledgeCoverage>;
|
|
477
479
|
/** Per-variant knowledge gap reports. See docs/specs/knowledge-gap-signal-spec.md */
|
|
478
480
|
gapReports?: Record<string, GapReport>;
|
|
479
|
-
/** Sample design science aggregate. Built from
|
|
481
|
+
/** Sample design science aggregate. Built from measurement metadata
|
|
480
482
|
* (capability / difficulty / construct / provenance); persisted on report
|
|
481
483
|
* for studio to surface coverage gaps. See docs/specs/sample-design-spec.md. */
|
|
482
484
|
sampleQuality?: SampleQualityAggregate;
|
|
@@ -52,9 +52,20 @@ export interface SkillGraphStageSnapshot {
|
|
|
52
52
|
edgeCount: number;
|
|
53
53
|
}
|
|
54
54
|
export interface SkillGraphNodePreview {
|
|
55
|
+
stableKey?: string;
|
|
55
56
|
nodeKind: string;
|
|
56
57
|
label: string;
|
|
57
58
|
status?: string;
|
|
59
|
+
coverage?: 'declared' | 'undeclared';
|
|
60
|
+
coveredBySamples?: string[];
|
|
61
|
+
}
|
|
62
|
+
export interface SkillGraphCoverageEdgePreview {
|
|
63
|
+
sampleStableKey?: string;
|
|
64
|
+
sampleLabel: string;
|
|
65
|
+
sampleStatus?: string;
|
|
66
|
+
targetStableKey: string;
|
|
67
|
+
targetNodeKind: string;
|
|
68
|
+
targetLabel: string;
|
|
58
69
|
}
|
|
59
70
|
export interface SkillGraphSnapshot {
|
|
60
71
|
/** Studio 聚合 graph sidecar 时实际采用的绑定强度。 */
|
|
@@ -76,6 +87,9 @@ export interface SkillGraphSnapshot {
|
|
|
76
87
|
failedAssertionEdges: number;
|
|
77
88
|
diagnostics: number;
|
|
78
89
|
measurementNodes: SkillGraphNodePreview[];
|
|
90
|
+
coverageEdges: number;
|
|
91
|
+
declaredCoverageStableKeys: string[];
|
|
92
|
+
declaredCoverageEdges: SkillGraphCoverageEdgePreview[];
|
|
79
93
|
};
|
|
80
94
|
}
|
|
81
95
|
export interface SkillIndexEntry {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "oh-my-knowledge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.46.0",
|
|
4
4
|
"packageManager": "yarn@4.16.0",
|
|
5
5
|
"description": "Evaluation framework for LLM knowledge inputs — prompts, RAG corpora, skills, agent workflows. Fix the model, vary the artifact. Built-in statistical rigor: bootstrap CI, Krippendorff α, length-debias, saturation curves.",
|
|
6
6
|
"type": "module",
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
"docs:preview": "vitepress preview docs",
|
|
27
27
|
"typecheck": "tsc --noEmit",
|
|
28
28
|
"lint": "eslint 'src/**/*.ts' 'test/**/*.ts' --cache --cache-location node_modules/.cache/eslint/ --max-warnings 0",
|
|
29
|
+
"lint-staged": "lint-staged",
|
|
29
30
|
"test": "vitest run",
|
|
30
31
|
"ci": "run-s lint typecheck build build:docs:check test",
|
|
31
32
|
"prepublishOnly": "run-s clean build",
|
|
@@ -92,11 +93,11 @@
|
|
|
92
93
|
"license": "MIT",
|
|
93
94
|
"dependencies": {
|
|
94
95
|
"@anthropic-ai/claude-agent-sdk": "^0.3.143",
|
|
95
|
-
"@anthropic-ai/sdk": "^0.
|
|
96
|
+
"@anthropic-ai/sdk": "^0.105.0",
|
|
96
97
|
"@inquirer/prompts": "^8.4.3",
|
|
97
98
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
98
99
|
"@oclif/core": "^4",
|
|
99
|
-
"@openai/codex-sdk": "0.
|
|
100
|
+
"@openai/codex-sdk": "0.141.0",
|
|
100
101
|
"ajv": "^8.18.0",
|
|
101
102
|
"chart.js": "^4.5.1",
|
|
102
103
|
"js-yaml": "^4.1.1",
|
|
@@ -111,11 +112,11 @@
|
|
|
111
112
|
"@types/node": "^25.5.0",
|
|
112
113
|
"eslint": "^10.1.0",
|
|
113
114
|
"husky": "^9.1.7",
|
|
114
|
-
"lint-staged": "
|
|
115
|
+
"lint-staged": "17.0.7",
|
|
115
116
|
"npm-run-all2": "^9.0.1",
|
|
116
117
|
"typescript": "^6.0.2",
|
|
117
118
|
"typescript-eslint": "^8.58.0",
|
|
118
119
|
"vitepress": "^1.6.4",
|
|
119
|
-
"vitest": "
|
|
120
|
+
"vitest": "4.1.8"
|
|
120
121
|
}
|
|
121
122
|
}
|