oh-my-knowledge 0.44.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artifact-graph/doctor.js +1 -1
- package/dist/artifact-graph/eval.js +120 -0
- package/dist/doctor/index.js +23 -3
- package/dist/eval-core/evaluation-reporting.js +1 -0
- package/dist/inputs/load-samples.js +27 -0
- package/dist/renderer/skill-detail-renderer.js +77 -7
- package/dist/renderer/skill-list-renderer.js +1 -1
- package/dist/server/skill-index.js +21 -0
- package/dist/types/artifact-graph.d.ts +1 -1
- package/dist/types/eval.d.ts +10 -0
- package/dist/types/report.d.ts +3 -1
- package/dist/types/skill-index.d.ts +4 -0
- package/package.json +3 -3
|
@@ -506,7 +506,7 @@ export function renderDoctorEvidenceCard(graph, skill, lang) {
|
|
|
506
506
|
];
|
|
507
507
|
const hiddenStructure = renderStructureDetails(graph, lang);
|
|
508
508
|
return [
|
|
509
|
-
`## ${zh ? 'Skill
|
|
509
|
+
`## ${zh ? '知识图谱摘要' : 'Skill Map Summary'}${zh ? ':' : ': '}${skill.skillName}`,
|
|
510
510
|
'',
|
|
511
511
|
statusSentence,
|
|
512
512
|
'',
|
|
@@ -133,8 +133,73 @@ function sampleAttrs(snapshot) {
|
|
|
133
133
|
display.tripwire = true;
|
|
134
134
|
if (snapshot.assertions?.length)
|
|
135
135
|
display.assertionCount = snapshot.assertions.length;
|
|
136
|
+
if (snapshot.covers?.length)
|
|
137
|
+
display.declaredCoverageTargetCount = snapshot.covers.length;
|
|
136
138
|
return Object.keys(display).length > 0 ? { display } : undefined;
|
|
137
139
|
}
|
|
140
|
+
const COVERAGE_TARGET_NODE_KIND = {
|
|
141
|
+
skill: 'skill',
|
|
142
|
+
skill_file: 'skill_file',
|
|
143
|
+
frontmatter: 'frontmatter',
|
|
144
|
+
reference: 'reference',
|
|
145
|
+
script: 'script',
|
|
146
|
+
hard_rule: 'hard_rule',
|
|
147
|
+
workflow: 'workflow',
|
|
148
|
+
workflow_node: 'workflow_node',
|
|
149
|
+
};
|
|
150
|
+
function normalizeCoverageRef(target) {
|
|
151
|
+
const raw = target.ref.trim().replaceAll('\\', '/');
|
|
152
|
+
if (target.targetKind === 'reference' || target.targetKind === 'script' || target.targetKind === 'skill_file') {
|
|
153
|
+
return raw.replace(/^\/+/, '').replace(/^\.\//, '');
|
|
154
|
+
}
|
|
155
|
+
return raw;
|
|
156
|
+
}
|
|
157
|
+
function coverageTargetStableKey(target, artifactHash) {
|
|
158
|
+
const ref = normalizeCoverageRef(target);
|
|
159
|
+
switch (target.targetKind) {
|
|
160
|
+
case 'skill':
|
|
161
|
+
return `v1:skill:${artifactHash}`;
|
|
162
|
+
case 'skill_file':
|
|
163
|
+
return `v1:skill-file:${artifactHash}:${ref || 'SKILL.md'}`;
|
|
164
|
+
case 'frontmatter':
|
|
165
|
+
return `v1:frontmatter:${artifactHash}`;
|
|
166
|
+
case 'reference':
|
|
167
|
+
return `v1:reference:${artifactHash}:${ref}`;
|
|
168
|
+
case 'script':
|
|
169
|
+
return `v1:script:${artifactHash}:${ref}`;
|
|
170
|
+
case 'hard_rule':
|
|
171
|
+
return `v1:hard-rule:${artifactHash}:${ref}`;
|
|
172
|
+
case 'workflow':
|
|
173
|
+
return `v1:workflow:${artifactHash}:${ref}`;
|
|
174
|
+
case 'workflow_node':
|
|
175
|
+
return `v1:workflow-node:${artifactHash}:${ref}`;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
function coverageTargetLabel(target) {
|
|
179
|
+
const ref = normalizeCoverageRef(target);
|
|
180
|
+
switch (target.targetKind) {
|
|
181
|
+
case 'skill':
|
|
182
|
+
return ref && ref !== 'skill' ? ref : 'SKILL.md';
|
|
183
|
+
case 'skill_file':
|
|
184
|
+
return ref || 'SKILL.md';
|
|
185
|
+
case 'frontmatter':
|
|
186
|
+
return 'frontmatter';
|
|
187
|
+
default:
|
|
188
|
+
return ref;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
function coverageEvidence(report, sampleId, targetIndex, target) {
|
|
192
|
+
return [{
|
|
193
|
+
sourceKind: 'sample',
|
|
194
|
+
sourceId: sampleId,
|
|
195
|
+
selector: {
|
|
196
|
+
selectorKind: 'json-pointer',
|
|
197
|
+
value: `/sampleSnapshots/${jsonPointerToken(sampleId)}/covers/${targetIndex}`,
|
|
198
|
+
},
|
|
199
|
+
contentHash: report.meta.sampleHashes?.[sampleId],
|
|
200
|
+
label: `${sampleId} covers ${target.targetKind}:${normalizeCoverageRef(target)}`,
|
|
201
|
+
}];
|
|
202
|
+
}
|
|
138
203
|
export function evalGraphDirForReportOutput(reportOutputDir) {
|
|
139
204
|
return basename(reportOutputDir) === 'reports'
|
|
140
205
|
? join(dirname(reportOutputDir), 'graphs', 'eval')
|
|
@@ -147,6 +212,16 @@ export function buildEvalArtifactGraph(options) {
|
|
|
147
212
|
const edges = [];
|
|
148
213
|
const nodeIdsByStableKey = new Map();
|
|
149
214
|
const configs = variantConfigByName(report);
|
|
215
|
+
const skillArtifacts = report.meta.variants
|
|
216
|
+
.map((variant) => ({
|
|
217
|
+
variant,
|
|
218
|
+
artifactHash: report.meta.artifactHashes?.[variant],
|
|
219
|
+
config: configs.get(variant),
|
|
220
|
+
}))
|
|
221
|
+
.filter((item) => item.config?.artifactKind === 'skill'
|
|
222
|
+
&& typeof item.artifactHash === 'string'
|
|
223
|
+
&& item.artifactHash.length > 0
|
|
224
|
+
&& item.artifactHash !== 'no-skill');
|
|
150
225
|
const addNode = (stableKey, nodeKind, nodeRole, label, extra = {}) => {
|
|
151
226
|
const existing = nodeIdsByStableKey.get(stableKey);
|
|
152
227
|
if (existing)
|
|
@@ -175,6 +250,50 @@ export function buildEvalArtifactGraph(options) {
|
|
|
175
250
|
...extra,
|
|
176
251
|
});
|
|
177
252
|
};
|
|
253
|
+
const addCoverageEdges = (sampleId, sampleNodeId, snapshot) => {
|
|
254
|
+
if (!snapshot.covers?.length || skillArtifacts.length === 0)
|
|
255
|
+
return;
|
|
256
|
+
const seen = new Set();
|
|
257
|
+
snapshot.covers.forEach((target, targetIndex) => {
|
|
258
|
+
const targetRef = normalizeCoverageRef(target);
|
|
259
|
+
if (!targetRef && target.targetKind !== 'skill' && target.targetKind !== 'frontmatter')
|
|
260
|
+
return;
|
|
261
|
+
for (const artifact of skillArtifacts) {
|
|
262
|
+
const stableKey = coverageTargetStableKey(target, artifact.artifactHash);
|
|
263
|
+
const dedupeKey = `${sampleNodeId}|${stableKey}`;
|
|
264
|
+
if (seen.has(dedupeKey))
|
|
265
|
+
continue;
|
|
266
|
+
seen.add(dedupeKey);
|
|
267
|
+
const evidenceRefs = coverageEvidence(report, sampleId, targetIndex, target);
|
|
268
|
+
const targetNodeId = addNode(stableKey, COVERAGE_TARGET_NODE_KIND[target.targetKind], 'entity', coverageTargetLabel(target), {
|
|
269
|
+
binding: { bindingStrength: 'content-hash', keys: { artifactHash: artifact.artifactHash } },
|
|
270
|
+
attrs: {
|
|
271
|
+
display: {
|
|
272
|
+
targetKind: target.targetKind,
|
|
273
|
+
ref: targetRef,
|
|
274
|
+
variant: artifact.variant,
|
|
275
|
+
sourceLocator: artifact.config.locator,
|
|
276
|
+
},
|
|
277
|
+
},
|
|
278
|
+
evidenceRefs,
|
|
279
|
+
});
|
|
280
|
+
addEdge(sampleNodeId, targetNodeId, 'covers', {
|
|
281
|
+
confidence: 1,
|
|
282
|
+
binding: {
|
|
283
|
+
bindingStrength: 'explicit',
|
|
284
|
+
keys: {
|
|
285
|
+
sampleId,
|
|
286
|
+
targetKind: target.targetKind,
|
|
287
|
+
targetRef,
|
|
288
|
+
artifactHash: artifact.artifactHash,
|
|
289
|
+
},
|
|
290
|
+
},
|
|
291
|
+
attrs: { producer: { source: 'sample.covers' } },
|
|
292
|
+
evidenceRefs,
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
});
|
|
296
|
+
};
|
|
178
297
|
const variantNodeIds = new Map();
|
|
179
298
|
for (const variant of report.meta.variants) {
|
|
180
299
|
const artifactHash = report.meta.artifactHashes?.[variant];
|
|
@@ -241,6 +360,7 @@ export function buildEvalArtifactGraph(options) {
|
|
|
241
360
|
});
|
|
242
361
|
addEdge(sampleNodeId, assertionNodeId, 'contains');
|
|
243
362
|
});
|
|
363
|
+
addCoverageEdges(sampleId, sampleNodeId, snapshot);
|
|
244
364
|
}
|
|
245
365
|
for (const [resultIndex, result] of report.results.entries()) {
|
|
246
366
|
const sampleNodeId = addNode(sampleStableKey(report, result.sample_id), 'sample', 'entity', result.sample_id, {
|
package/dist/doctor/index.js
CHANGED
|
@@ -7,12 +7,14 @@
|
|
|
7
7
|
* fatal-fail 不中断后续 rule 执行(让用户一次看到全貌),但 report.outcome='failed',
|
|
8
8
|
* CLI 据此 exit 1 / abort eval。
|
|
9
9
|
*/
|
|
10
|
-
import { existsSync, statSync } from 'node:fs';
|
|
10
|
+
import { existsSync, readFileSync, statSync } from 'node:fs';
|
|
11
11
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
12
|
+
import { fileURLToPath } from 'node:url';
|
|
12
13
|
import { runFileSuffix } from '../eval-core/artifact-file-names.js';
|
|
13
14
|
import { discoverVariants, resolveArtifacts } from '../inputs/skill-loader.js';
|
|
14
15
|
import { DOCTOR_REPORT_SCHEMA_VERSION, isComposerRule } from '../types/doctor.js';
|
|
15
16
|
import { getRegisteredRules } from './rules.js';
|
|
17
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
16
18
|
// ---------------------------------------------------------------------------
|
|
17
19
|
// Target resolution
|
|
18
20
|
// ---------------------------------------------------------------------------
|
|
@@ -167,9 +169,27 @@ function inferSkillPath(artifact, baseDir) {
|
|
|
167
169
|
function nextReportId() {
|
|
168
170
|
return `doctor-${runFileSuffix()}`;
|
|
169
171
|
}
|
|
172
|
+
function findPackageJson(startDir) {
|
|
173
|
+
let dir = startDir;
|
|
174
|
+
for (let i = 0; i < 5; i += 1) {
|
|
175
|
+
const candidate = join(dir, 'package.json');
|
|
176
|
+
if (existsSync(candidate))
|
|
177
|
+
return candidate;
|
|
178
|
+
dir = dirname(dir);
|
|
179
|
+
}
|
|
180
|
+
return join(startDir, '..', 'package.json');
|
|
181
|
+
}
|
|
170
182
|
function readCliVersion() {
|
|
171
|
-
|
|
172
|
-
|
|
183
|
+
const envVersion = process.env.npm_package_version;
|
|
184
|
+
if (envVersion)
|
|
185
|
+
return envVersion;
|
|
186
|
+
try {
|
|
187
|
+
const pkg = JSON.parse(readFileSync(findPackageJson(__dirname), 'utf-8'));
|
|
188
|
+
if (typeof pkg.version === 'string' && pkg.version.length > 0)
|
|
189
|
+
return pkg.version;
|
|
190
|
+
}
|
|
191
|
+
catch { /* fall through */ }
|
|
192
|
+
return '0.0.0';
|
|
173
193
|
}
|
|
174
194
|
export async function runDoctor(opts) {
|
|
175
195
|
// 默认规则 = 内置 + registerRule() 注册的 custom。test 注入走 opts.rules。
|
|
@@ -273,6 +273,7 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
|
|
|
273
273
|
...(s.difficulty ? { difficulty: s.difficulty } : {}),
|
|
274
274
|
...(s.construct ? { construct: s.construct } : {}),
|
|
275
275
|
...(s.provenance ? { provenance: s.provenance } : {}),
|
|
276
|
+
...(s.covers && s.covers.length > 0 ? { covers: s.covers } : {}),
|
|
276
277
|
...(s.tripwire ? { tripwire: true } : {}),
|
|
277
278
|
}])),
|
|
278
279
|
};
|
|
@@ -108,6 +108,16 @@ export function validateSamples(samples) {
|
|
|
108
108
|
// Pure documentation/diagnostic fields; do NOT participate in grading/judge/verdict.
|
|
109
109
|
const VALID_DIFFICULTY = new Set(['easy', 'medium', 'hard']);
|
|
110
110
|
const VALID_PROVENANCE = new Set(['human', 'llm-generated', 'production-trace']);
|
|
111
|
+
const VALID_COVERAGE_TARGET_KIND = new Set([
|
|
112
|
+
'skill',
|
|
113
|
+
'skill_file',
|
|
114
|
+
'frontmatter',
|
|
115
|
+
'reference',
|
|
116
|
+
'script',
|
|
117
|
+
'hard_rule',
|
|
118
|
+
'workflow',
|
|
119
|
+
'workflow_node',
|
|
120
|
+
]);
|
|
111
121
|
for (const [i, sample] of samples.entries()) {
|
|
112
122
|
if (!sample.sample_id || typeof sample.sample_id !== 'string') {
|
|
113
123
|
throw new Error(`samples[${i}] missing or invalid required field: sample_id (must be a non-empty string)`);
|
|
@@ -135,6 +145,23 @@ export function validateSamples(samples) {
|
|
|
135
145
|
if (sample.construct !== undefined && typeof sample.construct !== 'string') {
|
|
136
146
|
throw new Error(`samples[${i}] (${sample.sample_id}) invalid construct: must be a string (got ${typeof sample.construct})`);
|
|
137
147
|
}
|
|
148
|
+
if (sample.covers !== undefined) {
|
|
149
|
+
if (!Array.isArray(sample.covers)) {
|
|
150
|
+
throw new Error(`samples[${i}] (${sample.sample_id}) invalid covers: must be an array of coverage target objects`);
|
|
151
|
+
}
|
|
152
|
+
for (const [j, target] of sample.covers.entries()) {
|
|
153
|
+
if (!target || typeof target !== 'object' || Array.isArray(target)) {
|
|
154
|
+
throw new Error(`samples[${i}] (${sample.sample_id}) covers[${j}] must be an object with targetKind/ref`);
|
|
155
|
+
}
|
|
156
|
+
const record = target;
|
|
157
|
+
if (typeof record.targetKind !== 'string' || !VALID_COVERAGE_TARGET_KIND.has(record.targetKind)) {
|
|
158
|
+
throw new Error(`samples[${i}] (${sample.sample_id}) covers[${j}] invalid targetKind: ${JSON.stringify(record.targetKind)}, expected one of [${[...VALID_COVERAGE_TARGET_KIND].join(', ')}]`);
|
|
159
|
+
}
|
|
160
|
+
if (typeof record.ref !== 'string' || !record.ref.trim()) {
|
|
161
|
+
throw new Error(`samples[${i}] (${sample.sample_id}) covers[${j}] invalid ref: must be a non-empty string`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
138
165
|
// tools_called / tools_not_called: values 必须非空。空 values 在 grader 里
|
|
139
166
|
// 永远 passed=true 但 weight=0,是无意义占位,污染断言计数(N/M 看上去比真实
|
|
140
167
|
// 通过率高)+ 让 generator 输出可观测的 garbage 沉淀到磁盘。直接拒掉,作者
|
|
@@ -433,6 +433,8 @@ details.si-pass-card[open] > summary > .si-pass-row1::before { content:'▾ ' }
|
|
|
433
433
|
.sm-node--warning { border-color:rgba(217,119,6,.38);background:rgba(217,119,6,.07) }
|
|
434
434
|
.sm-node--failed { border-color:rgba(220,38,38,.38);background:rgba(220,38,38,.06) }
|
|
435
435
|
.sm-node--pending,.sm-node--more { border-style:dashed;background:rgba(248,249,251,.94);color:#637083 }
|
|
436
|
+
.sm-node--coverage-declared { border-color:rgba(31,157,99,.42);box-shadow:0 10px 24px rgba(31,157,99,.08) }
|
|
437
|
+
.sm-node--coverage-undeclared { border-style:dashed;background:rgba(248,249,251,.94);color:#637083 }
|
|
436
438
|
.sm-node-title { font-size:12.2px;font-weight:700;color:#182033;line-height:1.28;word-break:break-word;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden }
|
|
437
439
|
.sm-node--skill .sm-node-title { font-size:15px;line-height:1.2 }
|
|
438
440
|
.sm-node-meta { font-size:10.5px;color:#637083;line-height:1.35;word-break:break-word }
|
|
@@ -1011,11 +1013,25 @@ const SKILL_MAP_INIT_SCRIPT = `
|
|
|
1011
1013
|
var isLeaf = el.getAttribute('data-sm-leaf') === '1';
|
|
1012
1014
|
var overflowGroup = el.getAttribute('data-sm-overflow-group');
|
|
1013
1015
|
var shouldHide = !layers[layer] || (isLeaf && !showLeaves) || !overflowGroupExpanded(overflowGroup);
|
|
1016
|
+
el.setAttribute('data-sm-filter-hidden', shouldHide ? '1' : '0');
|
|
1014
1017
|
el.hidden = shouldHide;
|
|
1015
1018
|
el.toggleAttribute('hidden', shouldHide);
|
|
1016
1019
|
el.setAttribute('aria-hidden', shouldHide ? 'true' : 'false');
|
|
1017
1020
|
el.style.display = shouldHide ? 'none' : '';
|
|
1018
1021
|
});
|
|
1022
|
+
map.querySelectorAll('[data-sm-edge-to]').forEach(function(edge){
|
|
1023
|
+
var toId = edge.getAttribute('data-sm-edge-to');
|
|
1024
|
+
var fromId = edge.getAttribute('data-sm-edge-from-node');
|
|
1025
|
+
var toNode = toId ? graph.querySelector('[data-sm-node-id="' + toId + '"]') : null;
|
|
1026
|
+
var fromNode = fromId ? graph.querySelector('[data-sm-node-id="' + fromId + '"]') : null;
|
|
1027
|
+
var filterHidden = edge.getAttribute('data-sm-filter-hidden') === '1';
|
|
1028
|
+
var endpointHidden = (toNode && toNode.hidden) || (fromNode && fromNode.hidden);
|
|
1029
|
+
var shouldHideEdge = filterHidden || endpointHidden;
|
|
1030
|
+
edge.hidden = shouldHideEdge;
|
|
1031
|
+
edge.toggleAttribute('hidden', shouldHideEdge);
|
|
1032
|
+
edge.setAttribute('aria-hidden', shouldHideEdge ? 'true' : 'false');
|
|
1033
|
+
edge.style.display = shouldHideEdge ? 'none' : '';
|
|
1034
|
+
});
|
|
1019
1035
|
}
|
|
1020
1036
|
|
|
1021
1037
|
function setMoreExpanded(toggle, expanded){
|
|
@@ -1653,12 +1669,27 @@ const SKILL_MAP_STATUS_CLASSES = new Set(['ok', 'warning', 'failed', 'skipped',
|
|
|
1653
1669
|
function graphNodeStatusClass(node) {
|
|
1654
1670
|
return node.status && SKILL_MAP_STATUS_CLASSES.has(node.status) ? ` sm-node--${node.status}` : '';
|
|
1655
1671
|
}
|
|
1672
|
+
function graphNodeCoverageClass(node) {
|
|
1673
|
+
if (node.coverage === 'declared')
|
|
1674
|
+
return ' sm-node--coverage-declared';
|
|
1675
|
+
if (node.coverage === 'undeclared')
|
|
1676
|
+
return ' sm-node--coverage-undeclared';
|
|
1677
|
+
return '';
|
|
1678
|
+
}
|
|
1679
|
+
function graphNodeCoverageLabel(node, lang) {
|
|
1680
|
+
if (node.coverage === 'declared')
|
|
1681
|
+
return lang === 'zh' ? '已由评测用例声明' : 'declared by eval samples';
|
|
1682
|
+
if (node.coverage === 'undeclared')
|
|
1683
|
+
return lang === 'zh' ? '尚未由评测用例声明' : 'not declared by eval samples';
|
|
1684
|
+
return '';
|
|
1685
|
+
}
|
|
1656
1686
|
function positionedNodeStyle(node) {
|
|
1657
1687
|
return `left:${node.x}px;top:${node.y}px`;
|
|
1658
1688
|
}
|
|
1659
1689
|
function renderSkillMapNode(item, lang) {
|
|
1660
1690
|
const title = graphNodeMapLabel(item.node);
|
|
1661
|
-
const
|
|
1691
|
+
const coverageLabel = graphNodeCoverageLabel(item.node, lang);
|
|
1692
|
+
const fullTitle = [graphNodeDisplayLabel(item.node, lang), coverageLabel].filter(Boolean).join(' · ');
|
|
1662
1693
|
const kindClass = item.node.nodeKind === 'more' ? ' sm-node--more' : '';
|
|
1663
1694
|
const leaf = item.layer === 'definition' || item.layer === 'measurement' ? ' data-sm-leaf="1"' : '';
|
|
1664
1695
|
const draggable = item.layer === 'definition' || item.layer === 'measurement' ? ' data-sm-draggable="1"' : '';
|
|
@@ -1666,7 +1697,7 @@ function renderSkillMapNode(item, lang) {
|
|
|
1666
1697
|
const moreToggle = item.moreToggle
|
|
1667
1698
|
? ` data-sm-more-toggle="${item.moreToggle.group}" data-sm-collapsed-label="${e(item.moreToggle.collapsedLabel)}" data-sm-expanded-label="${e(item.moreToggle.expandedLabel)}" role="button" tabindex="0" aria-expanded="false"`
|
|
1668
1699
|
: '';
|
|
1669
|
-
return `<div class="sm-node sm-node--${item.layer}${kindClass}${graphNodeStatusClass(item.node)}" data-sm-node-id="${e(item.id)}" data-sm-layer="${item.layer}" data-sm-x="${item.x}" data-sm-y="${item.y}" data-sm-origin-x="${item.x}" data-sm-origin-y="${item.y}"${leaf}${draggable}${overflow}${moreToggle} style="${positionedNodeStyle(item)}" title="${e(fullTitle)}">
|
|
1700
|
+
return `<div class="sm-node sm-node--${item.layer}${kindClass}${graphNodeStatusClass(item.node)}${graphNodeCoverageClass(item.node)}" data-sm-node-id="${e(item.id)}" data-sm-layer="${item.layer}" data-sm-x="${item.x}" data-sm-y="${item.y}" data-sm-origin-x="${item.x}" data-sm-origin-y="${item.y}"${leaf}${draggable}${overflow}${moreToggle} style="${positionedNodeStyle(item)}" title="${e(fullTitle)}">
|
|
1670
1701
|
<div class="sm-node-kind">${e(nodeKindLabel(item.node.nodeKind, lang))}</div>
|
|
1671
1702
|
<div class="sm-node-title">${e(title)}</div>
|
|
1672
1703
|
</div>`;
|
|
@@ -1739,6 +1770,36 @@ function orderedPreviewNodes(nodes, allowedKinds, priority) {
|
|
|
1739
1770
|
.sort((a, b) => (priority[a.node.nodeKind] ?? 99) - (priority[b.node.nodeKind] ?? 99) || a.index - b.index)
|
|
1740
1771
|
.map(({ node }) => node);
|
|
1741
1772
|
}
|
|
1773
|
+
function applyDefinitionCoverageDeclarations(nodes, declaredStableKeys) {
|
|
1774
|
+
if (!nodes)
|
|
1775
|
+
return undefined;
|
|
1776
|
+
if (!declaredStableKeys || declaredStableKeys.length === 0)
|
|
1777
|
+
return nodes;
|
|
1778
|
+
const declared = new Set(declaredStableKeys);
|
|
1779
|
+
return nodes.map((node) => {
|
|
1780
|
+
if (!node.stableKey || !DEFINITION_PREVIEW_KINDS.has(node.nodeKind))
|
|
1781
|
+
return node;
|
|
1782
|
+
return { ...node, coverage: declared.has(node.stableKey) ? 'declared' : 'undeclared' };
|
|
1783
|
+
});
|
|
1784
|
+
}
|
|
1785
|
+
function skillMapCoverageDeclarationSummary(doctor, evalGraph, lang) {
|
|
1786
|
+
const declaredKeys = new Set(evalGraph?.declaredCoverageStableKeys ?? []);
|
|
1787
|
+
if (!doctor || declaredKeys.size === 0)
|
|
1788
|
+
return null;
|
|
1789
|
+
const coverable = doctor.definitionNodes.filter((node) => node.stableKey && DEFINITION_PREVIEW_KINDS.has(node.nodeKind));
|
|
1790
|
+
if (coverable.length === 0)
|
|
1791
|
+
return null;
|
|
1792
|
+
const declaredCount = coverable.filter((node) => node.stableKey && declaredKeys.has(node.stableKey)).length;
|
|
1793
|
+
const samples = evalGraph?.samples ?? 0;
|
|
1794
|
+
return lang === 'zh'
|
|
1795
|
+
? `${samples} 条评测用例声明了 ${declaredCount}/${coverable.length} 个结构节点`
|
|
1796
|
+
: `${samples} eval samples declared ${declaredCount}/${coverable.length} structure nodes`;
|
|
1797
|
+
}
|
|
1798
|
+
function skillMapCoverageDeclarationHint(lang) {
|
|
1799
|
+
return lang === 'zh'
|
|
1800
|
+
? '这些关系来自 sample.covers,表示评测用例显式标注的主要触达节点;尚未声明只表示还没有被评测用例显式标注,不应直接视为测试缺口。'
|
|
1801
|
+
: 'These relationships come from sample.covers and show the main nodes explicitly annotated by eval samples; not declared only means not explicitly annotated yet, not necessarily a test gap.';
|
|
1802
|
+
}
|
|
1742
1803
|
function renderSkillEvidenceMarkdown(entry, lang) {
|
|
1743
1804
|
const zh = lang === 'zh';
|
|
1744
1805
|
const graph = entry.graph;
|
|
@@ -1758,7 +1819,8 @@ function renderSkillEvidenceMarkdown(entry, lang) {
|
|
|
1758
1819
|
!entry.eval ? `- ${zh ? '测量效果' : 'Measure impact'}${stepSeparator} \`omk eval --control baseline --treatment ${sourceArg}\`` : '',
|
|
1759
1820
|
!entry.observe ? `- ${zh ? '接入观察' : 'Add observation'}${stepSeparator} \`omk observe ingest <trace-dir>\`` : '',
|
|
1760
1821
|
].filter(Boolean);
|
|
1761
|
-
const
|
|
1822
|
+
const coverageSummary = skillMapCoverageDeclarationSummary(doctor, evalGraph, lang);
|
|
1823
|
+
const definitionPreview = previewNodes(applyDefinitionCoverageDeclarations(doctor?.definitionNodes, evalGraph?.declaredCoverageStableKeys), DEFINITION_PREVIEW_KINDS, 8);
|
|
1762
1824
|
const measurementPreview = previewNodes(evalGraph?.measurementNodes, MEASUREMENT_PREVIEW_KINDS, 8);
|
|
1763
1825
|
const previewMermaidLines = [
|
|
1764
1826
|
...definitionPreview.visible.flatMap((node, index) => [
|
|
@@ -1771,7 +1833,7 @@ function renderSkillEvidenceMarkdown(entry, lang) {
|
|
|
1771
1833
|
]),
|
|
1772
1834
|
];
|
|
1773
1835
|
return [
|
|
1774
|
-
`## Skill
|
|
1836
|
+
`## ${zh ? '知识图谱摘要' : 'Skill Map Summary'}${zh ? ':' : ': '}${entry.skillName}`,
|
|
1775
1837
|
'',
|
|
1776
1838
|
summary,
|
|
1777
1839
|
'',
|
|
@@ -1793,6 +1855,7 @@ function renderSkillEvidenceMarkdown(entry, lang) {
|
|
|
1793
1855
|
`| doctor | ${doctorStatus} | ${doctor ? `${doctor.nodeCount} nodes / ${doctor.edgeCount} edges` : (zh ? '无 graph' : 'no graph')} |`,
|
|
1794
1856
|
`| eval | ${evalStatus} | ${evalGraph ? `${zh ? 'variant 子图' : 'variant subgraph'}: ${evalGraph.nodeCount} nodes / ${evalGraph.edgeCount} edges` : (zh ? '无 graph' : 'no graph')} |`,
|
|
1795
1857
|
`| observe | ${observeStatus} | ${zh ? '未接 production graph' : 'production graph not connected'} |`,
|
|
1858
|
+
...(coverageSummary ? ['', zh ? '### 声明锚点' : '### Declared Anchors', '', coverageSummary, '', skillMapCoverageDeclarationHint(lang)] : []),
|
|
1796
1859
|
'',
|
|
1797
1860
|
zh ? '### 关键计数' : '### Key Counts',
|
|
1798
1861
|
'',
|
|
@@ -1820,7 +1883,9 @@ function renderSkillMapSection(entry, lang) {
|
|
|
1820
1883
|
const evalGraph = graph.eval;
|
|
1821
1884
|
const band = skillMapBand(entry);
|
|
1822
1885
|
const markdown = renderSkillEvidenceMarkdown(entry, lang);
|
|
1823
|
-
const
|
|
1886
|
+
const definitionNodesWithCoverageDeclarations = applyDefinitionCoverageDeclarations(doctor?.definitionNodes, evalGraph?.declaredCoverageStableKeys);
|
|
1887
|
+
const coverageSummary = skillMapCoverageDeclarationSummary(doctor, evalGraph, lang);
|
|
1888
|
+
const definitionCandidates = orderedPreviewNodes(definitionNodesWithCoverageDeclarations, DEFINITION_PREVIEW_KINDS, {
|
|
1824
1889
|
reference: 1,
|
|
1825
1890
|
script: 2,
|
|
1826
1891
|
workflow: 3,
|
|
@@ -1897,10 +1962,15 @@ function renderSkillMapSection(entry, lang) {
|
|
|
1897
1962
|
];
|
|
1898
1963
|
const mapNodes = [...definitionNodes, ...measurementNodes];
|
|
1899
1964
|
const mapDimensions = skillMapDimensions(mapNodes);
|
|
1965
|
+
const topMeta = [
|
|
1966
|
+
coverageSummary,
|
|
1967
|
+
!coverageSummary && doctor ? `${doctor.nodeCount} ${zh ? '结构节点' : 'definition nodes'}` : '',
|
|
1968
|
+
!coverageSummary && evalGraph ? `${evalGraph.samples} ${zh ? '用例' : 'samples'}` : '',
|
|
1969
|
+
].filter(Boolean).join(' · ');
|
|
1900
1970
|
return `<section id="section-skill-map" class="si-sect si-sect--${band} sm-sect">
|
|
1901
1971
|
<div class="si-sect-h">
|
|
1902
1972
|
<span class="si-sect-title">🗺 ${zh ? 'Skill Map' : 'Skill Map'}</span>
|
|
1903
|
-
<span class="si-sect-meta">${
|
|
1973
|
+
<span class="si-sect-meta">${e(topMeta || (zh ? '图谱数据未生成' : 'graph data missing'))}</span>
|
|
1904
1974
|
</div>
|
|
1905
1975
|
<div class="sm-body" data-sm-map data-sm-base-width="${mapDimensions.width}" data-sm-base-height="${mapDimensions.height}" data-sm-root-x="${SKILL_MAP_ROOT.x}" data-sm-root-y="${SKILL_MAP_ROOT.y}">
|
|
1906
1976
|
${renderSkillStageRail(entry, lang)}
|
|
@@ -1930,7 +2000,7 @@ function renderSkillMapSection(entry, lang) {
|
|
|
1930
2000
|
</div>
|
|
1931
2001
|
</div>
|
|
1932
2002
|
<details class="sm-card">
|
|
1933
|
-
<summary>${zh ? '
|
|
2003
|
+
<summary>${zh ? '复制图谱摘要' : 'Copy Skill Map Summary'}</summary>
|
|
1934
2004
|
<textarea readonly>${e(markdown)}</textarea>
|
|
1935
2005
|
</details>
|
|
1936
2006
|
</div>
|
|
@@ -134,7 +134,7 @@ function renderPanel(a, lang) {
|
|
|
134
134
|
// renderSkillModal 已移除:首页点行直接进 skill 详情页(doctor/eval/observe 已在该页合一),弹框预览冗余。
|
|
135
135
|
function renderRow(entry, insights, langQ, lang) {
|
|
136
136
|
const h = assessHealth(entry, insights, lang);
|
|
137
|
-
// 点行先进入 skill hub:Skill Map / 三阶段状态 /
|
|
137
|
+
// 点行先进入 skill hub:Skill Map / 三阶段状态 / 图谱摘要是用户第一视角。
|
|
138
138
|
const href = `/skills/${encodeURIComponent(entry.skillName)}${langQ}`;
|
|
139
139
|
const dT = entry.doctor ? entry.doctor.passCount + entry.doctor.warnCount + entry.doctor.failCount : 0;
|
|
140
140
|
const dP = dT > 0 ? Math.round(((entry.doctor.passCount + entry.doctor.warnCount * 0.5) / dT) * 100) : null;
|
|
@@ -412,6 +412,7 @@ function countGraphNodes(graph, nodeKind) {
|
|
|
412
412
|
}
|
|
413
413
|
function graphNodePreview(node) {
|
|
414
414
|
return {
|
|
415
|
+
stableKey: node.stableKey,
|
|
415
416
|
nodeKind: node.nodeKind,
|
|
416
417
|
label: node.label,
|
|
417
418
|
...(node.status ? { status: node.status } : {}),
|
|
@@ -499,6 +500,24 @@ function projectEvalStage(graph, path, entry) {
|
|
|
499
500
|
addEdge(edge);
|
|
500
501
|
}
|
|
501
502
|
}
|
|
503
|
+
const declaredCoverageStableKeys = new Set();
|
|
504
|
+
let coverageEdges = 0;
|
|
505
|
+
for (const edge of graph.edges) {
|
|
506
|
+
if (edge.edgeKind !== 'covers')
|
|
507
|
+
continue;
|
|
508
|
+
const from = nodesById.get(edge.fromNodeId);
|
|
509
|
+
const to = nodesById.get(edge.toNodeId);
|
|
510
|
+
if (from?.nodeKind !== 'sample' || !to)
|
|
511
|
+
continue;
|
|
512
|
+
if (!projectedNodeIds.has(from.id))
|
|
513
|
+
continue;
|
|
514
|
+
if (artifactHash && nodeArtifactHash(to) !== artifactHash)
|
|
515
|
+
continue;
|
|
516
|
+
addEdge(edge);
|
|
517
|
+
coverageEdges += 1;
|
|
518
|
+
if (to.stableKey)
|
|
519
|
+
declaredCoverageStableKeys.add(to.stableKey);
|
|
520
|
+
}
|
|
502
521
|
const sampleIds = new Set();
|
|
503
522
|
const assertionIds = new Set();
|
|
504
523
|
const diagnosticIds = new Set();
|
|
@@ -544,6 +563,8 @@ function projectEvalStage(graph, path, entry) {
|
|
|
544
563
|
failedAssertionEdges,
|
|
545
564
|
diagnostics: diagnosticIds.size,
|
|
546
565
|
measurementNodes,
|
|
566
|
+
coverageEdges,
|
|
567
|
+
declaredCoverageStableKeys: [...declaredCoverageStableKeys].sort(),
|
|
547
568
|
},
|
|
548
569
|
...(artifactHash ? { artifactHash } : {}),
|
|
549
570
|
...(sourceLocator ? { sourceLocator } : {}),
|
|
@@ -29,7 +29,7 @@ export type ArtifactGraphNodeRole = 'entity' | 'observation' | 'aggregate';
|
|
|
29
29
|
export type ArtifactGraphNodeKind = 'skill' | 'skill_file' | 'frontmatter' | 'reference' | 'script' | 'tool' | 'env' | 'preflight' | 'hard_rule' | 'workflow' | 'workflow_node' | 'sample' | 'assertion' | 'variant' | 'doctor_rule_result' | 'eval_result' | 'judge_dimension' | 'diagnostic' | 'trace_session' | 'skill_invocation' | 'tool_call' | 'gap_signal';
|
|
30
30
|
export type ArtifactGraphStatus = 'ok' | 'warning' | 'failed' | 'skipped' | 'unknown' | 'not_measured';
|
|
31
31
|
export interface ArtifactGraphBinding {
|
|
32
|
-
bindingStrength: 'content-hash' | 'source-locator' | 'runtime-trace' | 'name-only' | 'aggregate';
|
|
32
|
+
bindingStrength: 'content-hash' | 'source-locator' | 'runtime-trace' | 'name-only' | 'aggregate' | 'explicit';
|
|
33
33
|
keys: Record<string, string>;
|
|
34
34
|
}
|
|
35
35
|
export interface ArtifactGraphAttrs {
|
package/dist/types/eval.d.ts
CHANGED
|
@@ -27,6 +27,14 @@ export interface Assertion {
|
|
|
27
27
|
export type SampleProvenance = 'human' | 'llm-generated' | 'production-trace';
|
|
28
28
|
/** sample 难度等级。简单分桶,跟 IRT 风格 fine-grained difficulty 不同。 */
|
|
29
29
|
export type SampleDifficulty = 'easy' | 'medium' | 'hard';
|
|
30
|
+
/** Sample 可选声明的 skill 结构锚点。用于 Skill Map / 诊断,不参与 grading / judge / verdict。 */
|
|
31
|
+
export type SampleCoverageTargetKind = 'skill' | 'skill_file' | 'frontmatter' | 'reference' | 'script' | 'hard_rule' | 'workflow' | 'workflow_node';
|
|
32
|
+
export interface SampleCoverageTarget {
|
|
33
|
+
/** 目标类型。避免裸 kind,保持 Artifact.kind 语义唯一。 */
|
|
34
|
+
targetKind: SampleCoverageTargetKind;
|
|
35
|
+
/** 稳定引用:路径或 ID。reference/script 用 skill 根相对路径,workflow_node 用 workflowId.nodeId。 */
|
|
36
|
+
ref: string;
|
|
37
|
+
}
|
|
30
38
|
/** Sample 评测环境前置:声明性"已就绪"清单,LLM 看到后跳过环境探测,直接进入工作流。
|
|
31
39
|
* 类比 unit test 的 fixture / setup —— 评测是测 skill 工作流,不是测环境探测能力。 */
|
|
32
40
|
export interface SampleEnvironment {
|
|
@@ -112,6 +120,8 @@ export interface Sample {
|
|
|
112
120
|
* curated 用 `'human'`,production trace 抽样用 `'production-trace'`。
|
|
113
121
|
* 纯文档 / 诊断用。 */
|
|
114
122
|
provenance?: SampleProvenance;
|
|
123
|
+
/** 该 sample 可选声明的 skill 结构锚点。纯文档 / 图谱诊断用,不进入评分、评委或 verdict。 */
|
|
124
|
+
covers?: SampleCoverageTarget[];
|
|
115
125
|
/** 诱错样本(tripwire)标记。true = 此 sample 故意设计成 LLM 应该 fail 的诱导陷阱
|
|
116
126
|
* (如:用户用错误前提诱导 / 跳步骤 / 用错参数类型),用于测 skill 是否能让 LLM
|
|
117
127
|
* 识破并纠正。Diagnostic 看到 tripwire:true 时会建议"无需改 skill"(rootCause:
|
package/dist/types/report.d.ts
CHANGED
|
@@ -346,6 +346,7 @@ export interface ResultEntry {
|
|
|
346
346
|
* - assertions: 期望(运行后的 pass/fail 结果在 VariantResult.assertions.details 里)
|
|
347
347
|
* - mocks: 工具调用模拟返回(LLM 调到匹配的 tool/参数时拿到这段假返回,而不是真去调外部系统)
|
|
348
348
|
* - capability / construct / difficulty: 元数据
|
|
349
|
+
* - covers: sample 可选声明的 skill 结构锚点
|
|
349
350
|
* - context: 附加上下文(代码片段等)
|
|
350
351
|
*
|
|
351
352
|
* 旧 report 没有这个字段,renderer 据此 fallback(隐藏单测 tab 或提示"无设计快照")。
|
|
@@ -361,6 +362,7 @@ export interface SampleSnapshot {
|
|
|
361
362
|
difficulty?: import('./eval.js').SampleDifficulty;
|
|
362
363
|
construct?: string;
|
|
363
364
|
provenance?: import('./eval.js').SampleProvenance;
|
|
365
|
+
covers?: import('./eval.js').SampleCoverageTarget[];
|
|
364
366
|
/** Diagnostic 用 — tripwire sample 不该建议改 skill。 */
|
|
365
367
|
tripwire?: boolean;
|
|
366
368
|
}
|
|
@@ -476,7 +478,7 @@ export interface AnalysisResult {
|
|
|
476
478
|
coverage?: Record<string, KnowledgeCoverage>;
|
|
477
479
|
/** Per-variant knowledge gap reports. See docs/specs/knowledge-gap-signal-spec.md */
|
|
478
480
|
gapReports?: Record<string, GapReport>;
|
|
479
|
-
/** Sample design science aggregate. Built from
|
|
481
|
+
/** Sample design science aggregate. Built from measurement metadata
|
|
480
482
|
* (capability / difficulty / construct / provenance); persisted on report
|
|
481
483
|
* for studio to surface coverage gaps. See docs/specs/sample-design-spec.md. */
|
|
482
484
|
sampleQuality?: SampleQualityAggregate;
|
|
@@ -52,9 +52,11 @@ export interface SkillGraphStageSnapshot {
|
|
|
52
52
|
edgeCount: number;
|
|
53
53
|
}
|
|
54
54
|
export interface SkillGraphNodePreview {
|
|
55
|
+
stableKey?: string;
|
|
55
56
|
nodeKind: string;
|
|
56
57
|
label: string;
|
|
57
58
|
status?: string;
|
|
59
|
+
coverage?: 'declared' | 'undeclared';
|
|
58
60
|
}
|
|
59
61
|
export interface SkillGraphSnapshot {
|
|
60
62
|
/** Studio 聚合 graph sidecar 时实际采用的绑定强度。 */
|
|
@@ -76,6 +78,8 @@ export interface SkillGraphSnapshot {
|
|
|
76
78
|
failedAssertionEdges: number;
|
|
77
79
|
diagnostics: number;
|
|
78
80
|
measurementNodes: SkillGraphNodePreview[];
|
|
81
|
+
coverageEdges: number;
|
|
82
|
+
declaredCoverageStableKeys: string[];
|
|
79
83
|
};
|
|
80
84
|
}
|
|
81
85
|
export interface SkillIndexEntry {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "oh-my-knowledge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.45.0",
|
|
4
4
|
"packageManager": "yarn@4.16.0",
|
|
5
5
|
"description": "Evaluation framework for LLM knowledge inputs — prompts, RAG corpora, skills, agent workflows. Fix the model, vary the artifact. Built-in statistical rigor: bootstrap CI, Krippendorff α, length-debias, saturation curves.",
|
|
6
6
|
"type": "module",
|
|
@@ -92,11 +92,11 @@
|
|
|
92
92
|
"license": "MIT",
|
|
93
93
|
"dependencies": {
|
|
94
94
|
"@anthropic-ai/claude-agent-sdk": "^0.3.143",
|
|
95
|
-
"@anthropic-ai/sdk": "^0.
|
|
95
|
+
"@anthropic-ai/sdk": "^0.105.0",
|
|
96
96
|
"@inquirer/prompts": "^8.4.3",
|
|
97
97
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
98
98
|
"@oclif/core": "^4",
|
|
99
|
-
"@openai/codex-sdk": "0.
|
|
99
|
+
"@openai/codex-sdk": "0.141.0",
|
|
100
100
|
"ajv": "^8.18.0",
|
|
101
101
|
"chart.js": "^4.5.1",
|
|
102
102
|
"js-yaml": "^4.1.1",
|