oh-my-knowledge 0.48.0 → 0.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/README.md +50 -18
  2. package/README.zh.md +55 -23
  3. package/dist/analysis/coverage-analyzer.d.ts +1 -0
  4. package/dist/analysis/coverage-analyzer.js +125 -62
  5. package/dist/analysis/failure-clusterer.js +2 -1
  6. package/dist/analysis/gap-analyzer.d.ts +2 -2
  7. package/dist/analysis/gap-analyzer.js +13 -3
  8. package/dist/analysis/hedging-classifier.d.ts +2 -2
  9. package/dist/analysis/hedging-classifier.js +3 -4
  10. package/dist/analysis/report-diagnostics.js +9 -7
  11. package/dist/analysis/sample-diagnostics.js +6 -6
  12. package/dist/artifact-graph/doctor.js +15 -7
  13. package/dist/assets/agent-skills/omk/SKILL.md +27 -7
  14. package/dist/assets/agent-skills/omk/references/commands.md +18 -17
  15. package/dist/authoring/evolver.d.ts +10 -6
  16. package/dist/authoring/evolver.js +496 -83
  17. package/dist/authoring/generator.d.ts +3 -3
  18. package/dist/authoring/generator.js +5 -10
  19. package/dist/authoring/sample-fixer.d.ts +8 -6
  20. package/dist/authoring/sample-fixer.js +76 -5
  21. package/dist/cli/commands/doctor.js +31 -14
  22. package/dist/cli/commands/eval/index.d.ts +3 -0
  23. package/dist/cli/commands/eval/index.js +163 -17
  24. package/dist/cli/commands/evolve.d.ts +4 -4
  25. package/dist/cli/commands/evolve.js +27 -13
  26. package/dist/cli/commands/init.js +16 -3
  27. package/dist/cli/commands/observe/inbox.js +28 -21
  28. package/dist/cli/commands/observe/index.js +20 -11
  29. package/dist/cli/commands/observe/ingest.d.ts +3 -0
  30. package/dist/cli/commands/observe/ingest.js +30 -2
  31. package/dist/cli/commands/sample.d.ts +6 -3
  32. package/dist/cli/commands/sample.js +72 -68
  33. package/dist/cli/lib/codex-model-hint.d.ts +9 -0
  34. package/dist/cli/lib/codex-model-hint.js +45 -0
  35. package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
  36. package/dist/cli/lib/generation-failure-hint.js +61 -0
  37. package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
  38. package/dist/cli/lib/i18n-dict/common.js +4 -0
  39. package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
  40. package/dist/cli/lib/i18n-dict/gen.js +38 -6
  41. package/dist/cli/lib/i18n-dict/help.js +6 -6
  42. package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
  43. package/dist/cli/lib/i18n-dict/init.js +13 -9
  44. package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
  45. package/dist/cli/lib/i18n-dict/run.js +34 -2
  46. package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
  47. package/dist/cli/lib/llm-failure-classifier.js +8 -0
  48. package/dist/cli/lib/parse-run-config.d.ts +6 -5
  49. package/dist/cli/lib/parse-run-config.js +16 -9
  50. package/dist/cli/lib/runtime-defaults.d.ts +21 -0
  51. package/dist/cli/lib/runtime-defaults.js +79 -0
  52. package/dist/diagnosis/observe-mapper.js +14 -15
  53. package/dist/diagnosis/observe-producer.js +3 -1
  54. package/dist/diagnosis/studio-projection.js +14 -7
  55. package/dist/diagnosis/types.d.ts +2 -0
  56. package/dist/diagnosis/types.js +12 -0
  57. package/dist/doctor/endpoint-rule.js +2 -1
  58. package/dist/eval-core/artifact-file-names.js +18 -1
  59. package/dist/eval-core/artifact-index.d.ts +7 -11
  60. package/dist/eval-core/artifact-index.js +139 -80
  61. package/dist/eval-core/cache.d.ts +12 -3
  62. package/dist/eval-core/cache.js +89 -29
  63. package/dist/eval-core/comparability.js +10 -6
  64. package/dist/eval-core/evaluation-execution.d.ts +2 -1
  65. package/dist/eval-core/evaluation-execution.js +122 -37
  66. package/dist/eval-core/evaluation-job.d.ts +4 -1
  67. package/dist/eval-core/evaluation-job.js +4 -1
  68. package/dist/eval-core/evaluation-reporting.d.ts +15 -13
  69. package/dist/eval-core/evaluation-reporting.js +54 -52
  70. package/dist/eval-core/execution-strategy.d.ts +2 -0
  71. package/dist/eval-core/execution-strategy.js +11 -9
  72. package/dist/eval-core/fact-checker.js +15 -7
  73. package/dist/eval-core/holdout.js +3 -2
  74. package/dist/eval-core/judge-independence.d.ts +2 -2
  75. package/dist/eval-core/mock-hook.cjs +23 -6
  76. package/dist/eval-core/mocks-runtime.js +30 -8
  77. package/dist/eval-core/report-document.d.ts +12 -0
  78. package/dist/eval-core/report-document.js +1151 -0
  79. package/dist/eval-core/report-extensions.d.ts +4 -0
  80. package/dist/eval-core/report-extensions.js +500 -0
  81. package/dist/eval-core/report-file-migration.js +7 -2
  82. package/dist/eval-core/resume-compatibility.d.ts +31 -0
  83. package/dist/eval-core/resume-compatibility.js +141 -0
  84. package/dist/eval-core/sample-fingerprint.d.ts +12 -0
  85. package/dist/eval-core/sample-fingerprint.js +193 -0
  86. package/dist/eval-core/schema.js +86 -31
  87. package/dist/eval-core/verdict.d.ts +8 -4
  88. package/dist/eval-core/verdict.js +24 -10
  89. package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
  90. package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
  91. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
  92. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
  93. package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
  94. package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
  95. package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
  96. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
  97. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
  98. package/dist/eval-workflows/evaluation-pipeline.js +12 -7
  99. package/dist/eval-workflows/run-evaluation.d.ts +9 -7
  100. package/dist/eval-workflows/run-evaluation.js +79 -51
  101. package/dist/executors/anthropic-api.js +65 -9
  102. package/dist/executors/claude-cli.js +16 -79
  103. package/dist/executors/claude-protocol.d.ts +28 -0
  104. package/dist/executors/claude-protocol.js +180 -0
  105. package/dist/executors/claude-sdk-trace.js +56 -28
  106. package/dist/executors/claude-sdk.d.ts +1 -0
  107. package/dist/executors/claude-sdk.js +39 -93
  108. package/dist/executors/codex-cli-trace.js +166 -31
  109. package/dist/executors/codex-cli.d.ts +6 -8
  110. package/dist/executors/codex-cli.js +49 -151
  111. package/dist/executors/codex-protocol.d.ts +24 -0
  112. package/dist/executors/codex-protocol.js +234 -0
  113. package/dist/executors/codex-sdk.js +68 -120
  114. package/dist/executors/gemini.js +88 -13
  115. package/dist/executors/index.d.ts +2 -3
  116. package/dist/executors/index.js +5 -3
  117. package/dist/executors/openai-api.js +70 -9
  118. package/dist/executors/runtime-fingerprint.js +88 -11
  119. package/dist/executors/script-command.d.ts +8 -0
  120. package/dist/executors/script-command.js +87 -0
  121. package/dist/executors/script.js +202 -29
  122. package/dist/executors/shared.d.ts +35 -3
  123. package/dist/executors/shared.js +113 -15
  124. package/dist/grading/assertions.d.ts +1 -1
  125. package/dist/grading/assertions.js +19 -9
  126. package/dist/grading/diagnostic.d.ts +9 -2
  127. package/dist/grading/diagnostic.js +25 -2
  128. package/dist/grading/index.js +10 -4
  129. package/dist/grading/judge.js +19 -6
  130. package/dist/grading/layered-scores.d.ts +2 -3
  131. package/dist/grading/layered-scores.js +2 -3
  132. package/dist/inputs/load-samples.d.ts +1 -2
  133. package/dist/inputs/load-samples.js +23 -1
  134. package/dist/inputs/mcp-resolver.js +6 -3
  135. package/dist/inputs/sample-document.d.ts +11 -0
  136. package/dist/inputs/sample-document.js +96 -0
  137. package/dist/managed/evidence.d.ts +1 -0
  138. package/dist/managed/evidence.js +1 -1
  139. package/dist/managed/store.js +200 -91
  140. package/dist/observability/codex-trace-adapter.d.ts +5 -0
  141. package/dist/observability/codex-trace-adapter.js +850 -0
  142. package/dist/observability/experience.d.ts +32 -6
  143. package/dist/observability/experience.js +2695 -459
  144. package/dist/observability/feedback-matchers.js +16 -1
  145. package/dist/observability/inbox-view-model.d.ts +1 -1
  146. package/dist/observability/inbox-view-model.js +19 -14
  147. package/dist/observability/inbox.d.ts +7 -1
  148. package/dist/observability/inbox.js +632 -124
  149. package/dist/observability/problem-patterns.js +2 -0
  150. package/dist/observability/review-state.d.ts +6 -0
  151. package/dist/observability/review-state.js +235 -63
  152. package/dist/observability/skill-chain-advisories.js +1 -1
  153. package/dist/observability/skill-chain.js +17 -4
  154. package/dist/observability/skill-health-analyzer.d.ts +32 -7
  155. package/dist/observability/skill-health-analyzer.js +194 -121
  156. package/dist/observability/skill-health-report.d.ts +10 -0
  157. package/dist/observability/skill-health-report.js +620 -0
  158. package/dist/observability/soft-standards/constants.d.ts +0 -1
  159. package/dist/observability/soft-standards/constants.js +0 -1
  160. package/dist/observability/soft-standards/index.d.ts +1 -1
  161. package/dist/observability/soft-standards/index.js +1 -1
  162. package/dist/observability/soft-standards/llm-extractor.js +8 -10
  163. package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
  164. package/dist/observability/soft-standards/skill-standards-store.js +59 -18
  165. package/dist/observability/soft-standards/types.d.ts +2 -2
  166. package/dist/observability/trace-adapter.d.ts +12 -7
  167. package/dist/observability/trace-adapter.js +11 -9
  168. package/dist/observability/trace-attribution.d.ts +13 -5
  169. package/dist/observability/trace-attribution.js +315 -21
  170. package/dist/observability/trace-ingestion.d.ts +9 -0
  171. package/dist/observability/trace-ingestion.js +80 -0
  172. package/dist/observability/trace-ir.d.ts +113 -0
  173. package/dist/observability/trace-ir.js +87 -0
  174. package/dist/observability/trace-segmenter.d.ts +19 -6
  175. package/dist/observability/trace-segmenter.js +377 -196
  176. package/dist/observability/trace-session-index.d.ts +19 -0
  177. package/dist/observability/trace-session-index.js +68 -0
  178. package/dist/observability/trace-source.d.ts +12 -4
  179. package/dist/observability/trace-source.js +939 -215
  180. package/dist/renderer/html-renderer.js +37 -6
  181. package/dist/renderer/icons.js +3 -0
  182. package/dist/renderer/observation-inbox-renderer.js +208 -90
  183. package/dist/renderer/skill-detail-renderer.js +452 -109
  184. package/dist/renderer/skill-health-renderer.js +69 -12
  185. package/dist/renderer/summary.js +28 -7
  186. package/dist/renderer/table.js +21 -4
  187. package/dist/renderer/test-view.d.ts +1 -0
  188. package/dist/renderer/test-view.js +44 -9
  189. package/dist/server/indexed-report-store.js +14 -18
  190. package/dist/server/job-store.js +64 -26
  191. package/dist/server/report-server.js +190 -78
  192. package/dist/server/report-store.js +57 -80
  193. package/dist/server/skill-index.js +143 -49
  194. package/dist/server/skill-insights.js +44 -5
  195. package/dist/shared/artifact-graph.d.ts +3 -0
  196. package/dist/shared/artifact-graph.js +224 -0
  197. package/dist/shared/assertion-types.d.ts +8 -0
  198. package/dist/shared/assertion-types.js +46 -0
  199. package/dist/shared/atomic-json.d.ts +8 -0
  200. package/dist/shared/atomic-json.js +33 -0
  201. package/dist/shared/diagnosis-schema.d.ts +9 -0
  202. package/dist/shared/diagnosis-schema.js +181 -0
  203. package/dist/shared/doctor-report.d.ts +3 -0
  204. package/dist/shared/doctor-report.js +103 -0
  205. package/dist/shared/evaluation-job.d.ts +6 -0
  206. package/dist/shared/evaluation-job.js +217 -0
  207. package/dist/shared/executor-result.d.ts +17 -0
  208. package/dist/shared/executor-result.js +221 -0
  209. package/dist/shared/file-lock.d.ts +12 -0
  210. package/dist/shared/file-lock.js +129 -0
  211. package/dist/shared/json-value.d.ts +5 -0
  212. package/dist/shared/json-value.js +36 -0
  213. package/dist/shared/keyed-mutex.d.ts +7 -0
  214. package/dist/shared/keyed-mutex.js +24 -0
  215. package/dist/shared/record-count.d.ts +8 -0
  216. package/dist/shared/record-count.js +43 -0
  217. package/dist/shared/sample-contract.d.ts +3 -0
  218. package/dist/shared/sample-contract.js +332 -0
  219. package/dist/shared/timestamp.d.ts +6 -0
  220. package/dist/shared/timestamp.js +64 -0
  221. package/dist/shared/token-usage.d.ts +19 -0
  222. package/dist/shared/token-usage.js +50 -0
  223. package/dist/shared/tool-call-status.d.ts +8 -0
  224. package/dist/shared/tool-call-status.js +28 -0
  225. package/dist/shared/tool-identity.d.ts +21 -0
  226. package/dist/shared/tool-identity.js +84 -0
  227. package/dist/shared/tool-search.js +73 -16
  228. package/dist/shared/trace-projection.d.ts +5 -0
  229. package/dist/shared/trace-projection.js +20 -0
  230. package/dist/shared/trace-source-kind.d.ts +3 -0
  231. package/dist/shared/trace-source-kind.js +12 -0
  232. package/dist/types/diagnosis.d.ts +2 -0
  233. package/dist/types/eval.d.ts +4 -0
  234. package/dist/types/executor.d.ts +32 -5
  235. package/dist/types/index.d.ts +1 -0
  236. package/dist/types/index.js +1 -0
  237. package/dist/types/judge.d.ts +2 -0
  238. package/dist/types/observability.d.ts +116 -9
  239. package/dist/types/report.d.ts +58 -6
  240. package/dist/types/skill-index.d.ts +7 -0
  241. package/dist/types/trace.d.ts +2 -0
  242. package/dist/types/trace.js +1 -0
  243. package/package.json +9 -5
@@ -2,8 +2,11 @@
2
2
  * Knowledge coverage analyzer.
3
3
  * Computes coverage rates by comparing consumed knowledge against the full index.
4
4
  */
5
- import { basename, join, resolve } from 'node:path';
6
- import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
5
+ import { basename, isAbsolute, join, relative, resolve, } from 'node:path';
6
+ import { existsSync, readdirSync, readFileSync, realpathSync, statSync, } from 'node:fs';
7
+ import { ownRecordValue, setOwnRecordValue, } from '../shared/record-count.js';
8
+ import { toolCallQuery } from '../shared/tool-search.js';
9
+ import { isToolCallSuccess } from '../shared/tool-call-status.js';
7
10
  function classifyEntry(path) {
8
11
  if (/principle/i.test(path))
9
12
  return 'principle';
@@ -20,16 +23,24 @@ function classifyEntry(path) {
20
23
  export function extractReferencedPaths(artifactContent) {
21
24
  const paths = new Set();
22
25
  const pathPatterns = [
23
- /\.claude\/[a-zA-Z0-9_/.-]+\.md/g,
24
- /\.claude\/[a-zA-Z0-9_/.-]+\.sh/g,
26
+ /\.(?:agents|claude|codex|gemini)\/[a-zA-Z0-9_/.-]+\.(?:md|sh)/g,
25
27
  /repos\/[a-zA-Z0-9_/-]+/g,
28
+ /(?<![a-zA-Z0-9:/])((?:\.{1,2}\/)?(?:[a-zA-Z0-9_.-]+\/)+[a-zA-Z0-9_.-]+\.(?:md|sh))\b/g,
26
29
  ];
27
30
  for (const pattern of pathPatterns) {
28
31
  for (const match of artifactContent.matchAll(pattern)) {
29
- paths.add(match[0]);
32
+ paths.add(match[1] ?? match[0]);
30
33
  }
31
34
  }
32
- for (const match of artifactContent.matchAll(/\b([a-zA-Z0-9_-]+\.md)\b/g)) {
35
+ for (const match of artifactContent.matchAll(/\]\(([^)\s]+)\)/g)) {
36
+ const target = match[1].split('#', 1)[0];
37
+ if (target
38
+ && !/^[a-z][a-z0-9+.-]*:/i.test(target)
39
+ && /\.(?:md|sh)$/i.test(target)) {
40
+ paths.add(target);
41
+ }
42
+ }
43
+ for (const match of artifactContent.matchAll(/(?<![/\\])\b([a-zA-Z0-9_-]+\.md)\b/g)) {
33
44
  const name = match[1];
34
45
  if (/^(README|CHANGELOG|LICENSE|package)\.md$/i.test(name))
35
46
  continue;
@@ -37,27 +48,50 @@ export function extractReferencedPaths(artifactContent) {
37
48
  }
38
49
  return [...paths];
39
50
  }
40
- function scanKnowledgeDir(dir, prefix = '') {
51
+ function scanKnowledgeDir(dir, prefix = '', visitedDirectories = new Set()) {
41
52
  if (!existsSync(dir))
42
53
  return [];
54
+ let realDir;
55
+ try {
56
+ realDir = realpathSync(dir);
57
+ if (!statSync(realDir).isDirectory())
58
+ return [];
59
+ }
60
+ catch {
61
+ return [];
62
+ }
63
+ if (visitedDirectories.has(realDir))
64
+ return [];
65
+ visitedDirectories.add(realDir);
43
66
  const entries = [];
44
- for (const name of readdirSync(dir)) {
67
+ for (const name of readdirSync(dir).sort()) {
45
68
  if (name.startsWith('.'))
46
69
  continue;
47
70
  const fullPath = join(dir, name);
48
71
  const relativePath = prefix ? `${prefix}/${name}` : name;
49
- const stat = statSync(fullPath);
72
+ let stat;
73
+ try {
74
+ stat = statSync(fullPath);
75
+ }
76
+ catch {
77
+ continue;
78
+ }
50
79
  if (stat.isDirectory()) {
51
- entries.push(...scanKnowledgeDir(fullPath, relativePath));
80
+ entries.push(...scanKnowledgeDir(fullPath, relativePath, visitedDirectories));
52
81
  }
53
82
  else if (/\.(md|sh)$/.test(name)) {
54
83
  let lineCount;
84
+ let realPath;
55
85
  try {
86
+ realPath = realpathSync(fullPath);
56
87
  lineCount = readFileSync(fullPath, 'utf-8').split('\n').length;
57
88
  }
58
- catch { }
89
+ catch {
90
+ continue;
91
+ }
59
92
  entries.push({
60
93
  path: relativePath,
94
+ realPath,
61
95
  type: classifyEntry(relativePath),
62
96
  lineCount,
63
97
  });
@@ -67,22 +101,41 @@ function scanKnowledgeDir(dir, prefix = '') {
67
101
  }
68
102
  export function buildKnowledgeIndex(cwd) {
69
103
  const knowledgeDirs = [
104
+ { dir: join(cwd, '.agents', 'skills'), prefix: '.agents/skills' },
70
105
  { dir: join(cwd, '.claude', 'knowledge'), prefix: '.claude/knowledge' },
71
106
  { dir: join(cwd, '.claude', 'skills'), prefix: '.claude/skills' },
107
+ { dir: join(cwd, '.codex', 'skills'), prefix: '.codex/skills' },
108
+ { dir: join(cwd, '.gemini', 'skills'), prefix: '.gemini/skills' },
72
109
  ];
73
- const entries = [];
110
+ const entriesByRealPath = new Map();
74
111
  for (const { dir, prefix } of knowledgeDirs) {
75
- entries.push(...scanKnowledgeDir(dir, prefix));
112
+ for (const entry of scanKnowledgeDir(dir, prefix)) {
113
+ const existing = entriesByRealPath.get(entry.realPath);
114
+ if (existing) {
115
+ existing.aliases = [...new Set([...(existing.aliases ?? []), entry.path])];
116
+ continue;
117
+ }
118
+ const { realPath, ...knowledgeEntry } = entry;
119
+ void realPath;
120
+ entriesByRealPath.set(entry.realPath, knowledgeEntry);
121
+ }
76
122
  }
77
- const claudeMd = join(cwd, 'CLAUDE.md');
78
- if (existsSync(claudeMd)) {
123
+ for (const instructionFile of ['AGENTS.md', 'CLAUDE.md', 'GEMINI.md']) {
124
+ const instructionPath = join(cwd, instructionFile);
125
+ if (!existsSync(instructionPath))
126
+ continue;
79
127
  let lineCount;
80
128
  try {
81
- lineCount = readFileSync(claudeMd, 'utf-8').split('\n').length;
129
+ lineCount = readFileSync(instructionPath, 'utf-8').split('\n').length;
82
130
  }
83
131
  catch { }
84
- entries.push({ path: 'CLAUDE.md', type: 'principle', lineCount });
132
+ entriesByRealPath.set(realpathSync(instructionPath), {
133
+ path: instructionFile,
134
+ type: 'principle',
135
+ lineCount,
136
+ });
85
137
  }
138
+ const entries = [...entriesByRealPath.values()];
86
139
  const totalLines = entries.reduce((sum, entry) => sum + (entry.lineCount || 0), 0);
87
140
  return {
88
141
  entries,
@@ -92,48 +145,35 @@ export function buildKnowledgeIndex(cwd) {
92
145
  }
93
146
  export function buildFullKnowledgeIndex(artifactContent, cwd) {
94
147
  const entriesMap = new Map();
95
- if (artifactContent) {
96
- if (cwd) {
97
- const knowledgeDir = join(cwd, '.claude', 'knowledge');
98
- for (const entry of scanKnowledgeDir(knowledgeDir, '.claude/knowledge')) {
99
- entriesMap.set(entry.path, entry);
100
- }
101
- const claudeMd = join(cwd, 'CLAUDE.md');
102
- if (existsSync(claudeMd)) {
103
- let lineCount;
104
- try {
105
- lineCount = readFileSync(claudeMd, 'utf-8').split('\n').length;
106
- }
107
- catch { }
108
- entriesMap.set('CLAUDE.md', { path: 'CLAUDE.md', type: 'principle', lineCount });
109
- }
148
+ if (cwd) {
149
+ const dirIndex = buildKnowledgeIndex(cwd);
150
+ for (const entry of dirIndex.entries) {
151
+ entriesMap.set(entry.path, entry);
110
152
  }
153
+ }
154
+ if (artifactContent) {
111
155
  const refPaths = extractReferencedPaths(artifactContent);
112
156
  for (const path of refPaths) {
113
157
  if (!entriesMap.has(path)) {
114
- const existing = [...entriesMap.values()].find((entry) => entry.path.endsWith('/' + path) || entry.path === path);
158
+ const existing = [...entriesMap.values()].find((entry) => [entry.path, ...(entry.aliases ?? [])].some((candidate) => candidate.endsWith('/' + path) || candidate === path));
115
159
  if (!existing) {
116
160
  let lineCount;
117
161
  if (cwd) {
118
- const fullPath = resolve(cwd, path);
162
+ const fullPath = resolveInside(cwd, path);
119
163
  try {
120
- if (existsSync(fullPath) && statSync(fullPath).isFile()) {
164
+ if (fullPath && existsSync(fullPath) && statSync(fullPath).isFile()) {
121
165
  lineCount = readFileSync(fullPath, 'utf-8').split('\n').length;
122
166
  }
123
167
  }
124
168
  catch { }
125
169
  }
126
- entriesMap.set(path, { path, type: classifyEntry(path), lineCount });
170
+ if (!cwd || resolveInside(cwd, path)) {
171
+ entriesMap.set(path, { path, type: classifyEntry(path), lineCount });
172
+ }
127
173
  }
128
174
  }
129
175
  }
130
176
  }
131
- else if (cwd) {
132
- const dirIndex = buildKnowledgeIndex(cwd);
133
- for (const entry of dirIndex.entries) {
134
- entriesMap.set(entry.path, entry);
135
- }
136
- }
137
177
  const entries = [...entriesMap.values()];
138
178
  const totalLines = entries.reduce((sum, entry) => sum + (entry.lineCount || 0), 0);
139
179
  return {
@@ -142,18 +182,28 @@ export function buildFullKnowledgeIndex(artifactContent, cwd) {
142
182
  totalLines,
143
183
  };
144
184
  }
185
+ function resolveInside(root, path) {
186
+ const absoluteRoot = resolve(root);
187
+ const candidate = resolve(absoluteRoot, path);
188
+ const relativePath = relative(absoluteRoot, candidate);
189
+ return relativePath === ''
190
+ || (!relativePath.startsWith('..') && !isAbsolute(relativePath))
191
+ ? candidate
192
+ : null;
193
+ }
145
194
  export function extractKnowledgeConsumption(toolCalls) {
146
195
  const filesRead = [];
147
196
  const grepPatterns = [];
148
197
  const bashGrepPatterns = [];
149
198
  for (const toolCall of toolCalls) {
150
- const input = toolCall.input;
151
- if (!input)
152
- continue;
199
+ const input = toolCall.input && typeof toolCall.input === 'object'
200
+ ? toolCall.input
201
+ : {};
202
+ const legacyInput = typeof toolCall.input === 'string' ? toolCall.input : undefined;
153
203
  switch (toolCall.tool) {
154
204
  case 'Read': {
155
- const filePath = input.file_path;
156
- if (filePath)
205
+ const filePath = typeof input.file_path === 'string' ? input.file_path : legacyInput;
206
+ if (filePath && isToolCallSuccess(toolCall))
157
207
  filesRead.push(filePath);
158
208
  break;
159
209
  }
@@ -165,9 +215,12 @@ export function extractKnowledgeConsumption(toolCalls) {
165
215
  break;
166
216
  }
167
217
  case 'Bash': {
168
- const command = input.command;
218
+ const command = typeof input.command === 'string' ? input.command : legacyInput;
169
219
  if (!command)
170
220
  break;
221
+ const readPath = toolCallQuery(toolCall).path;
222
+ if (readPath && isToolCallSuccess(toolCall))
223
+ filesRead.push(readPath);
171
224
  const grepMatch = command.match(/(?:grep|rg)\s+(?:-[a-zA-Z]+\s+)*["']?([^"'\s|]+)["']?\s+([^\s|>]+)/);
172
225
  if (grepMatch) {
173
226
  bashGrepPatterns.push({ pattern: grepMatch[1], path: grepMatch[2] });
@@ -179,17 +232,27 @@ export function extractKnowledgeConsumption(toolCalls) {
179
232
  return { filesRead: [...new Set(filesRead)], grepPatterns, bashGrepPatterns };
180
233
  }
181
234
  export function normalizeKnowledgePath(filePath, cwd) {
182
- if (cwd && filePath.startsWith(cwd)) {
183
- const relative = filePath.slice(cwd.endsWith('/') ? cwd.length : cwd.length + 1);
184
- return relative;
235
+ if (cwd) {
236
+ const relativePath = relative(resolve(cwd), resolve(cwd, filePath));
237
+ if (relativePath === ''
238
+ || (!relativePath.startsWith('..') && !isAbsolute(relativePath))) {
239
+ return relativePath.replaceAll('\\', '/');
240
+ }
185
241
  }
186
- const patterns = ['.claude/knowledge/', '.claude/skills/'];
242
+ const normalized = filePath.replaceAll('\\', '/');
243
+ const patterns = [
244
+ '.agents/skills/',
245
+ '.claude/knowledge/',
246
+ '.claude/skills/',
247
+ '.codex/skills/',
248
+ '.gemini/skills/',
249
+ ];
187
250
  for (const pattern of patterns) {
188
- const index = filePath.indexOf(pattern);
251
+ const index = normalized.indexOf(pattern);
189
252
  if (index !== -1)
190
- return filePath.slice(index);
253
+ return normalized.slice(index);
191
254
  }
192
- return basename(filePath);
255
+ return basename(normalized);
193
256
  }
194
257
  /**
195
258
  * Compute knowledge coverage for a single variant across all samples.
@@ -207,13 +270,13 @@ export function computeCoverage(results, variant, index, cwd) {
207
270
  const consumedNormalized = new Set(consumption.filesRead.map((f) => normalizeKnowledgePath(f, cwd)));
208
271
  // Match index entries against consumed files
209
272
  const entries = index.entries.map((entry) => {
210
- // Check if this entry was accessed — match by exact path or by filename
211
- const accessed = consumedNormalized.has(entry.path) ||
212
- [...consumedNormalized].some((consumed) => consumed.endsWith('/' + entry.path) || consumed.endsWith(entry.path));
273
+ const paths = [entry.path, ...(entry.aliases ?? [])];
274
+ const accessed = paths.some((path) => consumedNormalized.has(path)
275
+ || [...consumedNormalized].some((consumed) => consumed.endsWith('/' + path)));
213
276
  const accessCount = accessed
214
277
  ? consumption.filesRead.filter((f) => {
215
278
  const norm = normalizeKnowledgePath(f, cwd);
216
- return norm === entry.path || norm.endsWith('/' + entry.path) || norm.endsWith(entry.path);
279
+ return paths.some((path) => norm === path || norm.endsWith('/' + path));
217
280
  }).length
218
281
  : 0;
219
282
  return {
@@ -249,13 +312,13 @@ export function computeCoverage(results, variant, index, cwd) {
249
312
  export function computeReportCoverage(report, artifactContents, cwds) {
250
313
  const coverageMap = {};
251
314
  for (const variant of report.meta.variants) {
252
- const content = artifactContents[variant] || null;
253
- const cwd = cwds[variant] || null;
315
+ const content = ownRecordValue(artifactContents, variant) || null;
316
+ const cwd = ownRecordValue(cwds, variant) || null;
254
317
  // Build knowledge index for this variant
255
318
  const index = buildFullKnowledgeIndex(content, cwd);
256
319
  if (index.totalFiles === 0)
257
320
  continue;
258
- coverageMap[variant] = computeCoverage(report.results, variant, index, cwd);
321
+ setOwnRecordValue(coverageMap, variant, computeCoverage(report.results, variant, index, cwd));
259
322
  }
260
323
  return coverageMap;
261
324
  }
@@ -38,6 +38,7 @@
38
38
  * a categorization task, not a quality scoring task, so the length-debias
39
39
  * directive is inapplicable.
40
40
  */
41
+ import { isToolCallFailure } from '../shared/tool-call-status.js';
41
42
  const DEFAULT_MAX_CLUSTERS = 5;
42
43
  const DEFAULT_FAILURE_THRESHOLD = 3;
43
44
  const DEFAULT_MAX_FED = 50;
@@ -159,7 +160,7 @@ function describeFailure(entry, variant, r) {
159
160
  if (r.llmReason)
160
161
  parts.push(`judge_reason="${truncate(r.llmReason, 200)}"`);
161
162
  // Tool failures (agent traces).
162
- const toolFailures = (r.toolCalls ?? []).filter((tc) => !tc.success).slice(0, 3);
163
+ const toolFailures = (r.toolCalls ?? []).filter(isToolCallFailure).slice(0, 3);
163
164
  if (toolFailures.length > 0) {
164
165
  parts.push(`failed_tools=[${toolFailures.map((tc) => `${tc.tool}: ${truncate(String(tc.output ?? ''), 80)}`).join('; ')}]`);
165
166
  }
@@ -105,7 +105,7 @@ export declare function computeReportGapRates(results: ResultEntry[], variants:
105
105
  * 失败降级在 classifier 层:单批失败 → 该 batch 全部 isUncertainty=true (保守保留)。
106
106
  * 调用方拿到的 report 永远是合法的,不会因 classifier 失败而崩。
107
107
  */
108
- export declare function applyHedgingClassifier(report: GapReport, executor: ExecutorFn, opts?: ClassifyOptions): Promise<{
108
+ export declare function applyHedgingClassifier(report: GapReport, executor: ExecutorFn, opts: ClassifyOptions): Promise<{
109
109
  report: GapReport;
110
110
  costUSD: number;
111
111
  truncated: boolean;
@@ -114,7 +114,7 @@ export declare function applyHedgingClassifier(report: GapReport, executor: Exec
114
114
  * 批量为一组 variant 的 gap reports 跑 classifier。
115
115
  * 串行(不并行)避免 rate limit + 让 cache hit 在第一批后被后续 batch 复用。
116
116
  */
117
- export declare function applyHedgingClassifierToReports(reports: Record<string, GapReport>, executor: ExecutorFn, opts?: ClassifyOptions): Promise<{
117
+ export declare function applyHedgingClassifierToReports(reports: Record<string, GapReport>, executor: ExecutorFn, opts: ClassifyOptions): Promise<{
118
118
  reports: Record<string, GapReport>;
119
119
  costUSD: number;
120
120
  }>;
@@ -17,6 +17,8 @@
17
17
  */
18
18
  import { classifyHedgingCandidates } from './hedging-classifier.js';
19
19
  import { isFailedSearchToolCall, toolCallQuery } from '../shared/tool-search.js';
20
+ import { setOwnRecordValue } from '../shared/record-count.js';
21
+ import { toolCallStatus } from '../shared/tool-call-status.js';
20
22
  // ---------- Pattern tables ----------
21
23
  const EXPLICIT_MARKER_PATTERNS = [
22
24
  /【推断】/g,
@@ -98,7 +100,15 @@ export function extractFailedSearchSignals(toolCalls) {
98
100
  sampleId: '',
99
101
  type: 'failed_search',
100
102
  context,
101
- evidence: { tool: tc.tool, pattern, path, success: tc.success },
103
+ evidence: {
104
+ tool: tc.tool,
105
+ pattern,
106
+ path,
107
+ status: toolCallStatus(tc),
108
+ statusSource: tc.statusSource ?? 'unknown',
109
+ // Retained for readers of pre-four-state reports.
110
+ success: tc.success,
111
+ },
102
112
  weight: SIGNAL_WEIGHTS.failed_search,
103
113
  });
104
114
  }
@@ -366,7 +376,7 @@ export function computeGapReport(results, variant) {
366
376
  export function computeReportGapRates(results, variants) {
367
377
  const out = {};
368
378
  for (const v of variants) {
369
- out[v] = computeGapReport(results, v);
379
+ setOwnRecordValue(out, v, computeGapReport(results, v));
370
380
  }
371
381
  return out;
372
382
  }
@@ -463,7 +473,7 @@ export async function applyHedgingClassifierToReports(reports, executor, opts) {
463
473
  let totalCost = 0;
464
474
  for (const [variant, report] of Object.entries(reports)) {
465
475
  const result = await applyHedgingClassifier(report, executor, opts);
466
- out[variant] = result.report;
476
+ setOwnRecordValue(out, variant, result.report);
467
477
  totalCost += result.costUSD;
468
478
  }
469
479
  return { reports: out, costUSD: totalCost };
@@ -16,8 +16,8 @@ export interface HedgingCandidate {
16
16
  context: string;
17
17
  }
18
18
  export interface ClassifyOptions {
19
+ model: string;
19
20
  maxCandidates?: number;
20
- model?: string;
21
21
  batchSize?: number;
22
22
  }
23
23
  export interface ClassifyResult {
@@ -30,4 +30,4 @@ export declare function clearHedgingCache(): void;
30
30
  * 对一批 candidate 做分类判定。返回与 input 等长的 verdicts。
31
31
  * 失败降级:单批 LLM 调用 / 解析失败 → 该 batch 全部 isUncertainty=true (保守保留)。
32
32
  */
33
- export declare function classifyHedgingCandidates(candidates: HedgingCandidate[], executor: ExecutorFn, opts?: ClassifyOptions): Promise<ClassifyResult>;
33
+ export declare function classifyHedgingCandidates(candidates: HedgingCandidate[], executor: ExecutorFn, opts: ClassifyOptions): Promise<ClassifyResult>;
@@ -12,7 +12,6 @@
12
12
  import { createHash } from 'node:crypto';
13
13
  const DEFAULT_MAX_CANDIDATES = 50;
14
14
  const DEFAULT_BATCH_SIZE = 10;
15
- const DEFAULT_MODEL = 'claude-haiku-4-5';
16
15
  // in-memory cache,进程内复用。key = sha256(sentence)。
17
16
  const verdictCache = new Map();
18
17
  export function clearHedgingCache() {
@@ -75,9 +74,9 @@ function fallbackVerdict(reason) {
75
74
  * 失败降级:单批 LLM 调用 / 解析失败 → 该 batch 全部 isUncertainty=true (保守保留)。
76
75
  */
77
76
  export async function classifyHedgingCandidates(candidates, executor, opts) {
78
- const max = opts?.maxCandidates ?? DEFAULT_MAX_CANDIDATES;
79
- const batchSize = opts?.batchSize ?? DEFAULT_BATCH_SIZE;
80
- const model = opts?.model ?? DEFAULT_MODEL;
77
+ const max = opts.maxCandidates ?? DEFAULT_MAX_CANDIDATES;
78
+ const batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
79
+ const model = opts.model;
81
80
  const truncated = candidates.length > max;
82
81
  const work = candidates.slice(0, max);
83
82
  const verdicts = new Array(candidates.length);
@@ -3,6 +3,8 @@
3
3
  */
4
4
  import { normalizeCapability } from './sample-diagnostics.js';
5
5
  import { analyzeJudgeIndependence } from '../eval-core/judge-independence.js';
6
+ import { isToolCallFailure } from '../shared/tool-call-status.js';
7
+ import { incrementRecordCount, setOwnRecordValue } from '../shared/record-count.js';
6
8
  /**
7
9
  * Analyze an evaluation report and produce structured insights.
8
10
  */
@@ -111,7 +113,7 @@ export function buildSampleQualityAggregate(samples) {
111
113
  if (seen.has(cap))
112
114
  continue; // 同 sample 内同 capability 重复声明只计 1
113
115
  seen.add(cap);
114
- capabilityCoverage[cap] = (capabilityCoverage[cap] || 0) + 1;
116
+ incrementRecordCount(capabilityCoverage, cap);
115
117
  }
116
118
  }
117
119
  // difficulty
@@ -125,18 +127,18 @@ export function buildSampleQualityAggregate(samples) {
125
127
  // construct (free-form)
126
128
  if (sample.construct) {
127
129
  withConstruct++;
128
- constructDistribution[sample.construct] = (constructDistribution[sample.construct] || 0) + 1;
130
+ incrementRecordCount(constructDistribution, sample.construct);
129
131
  }
130
132
  else {
131
- constructDistribution.unspecified = (constructDistribution.unspecified || 0) + 1;
133
+ incrementRecordCount(constructDistribution, 'unspecified');
132
134
  }
133
135
  // provenance
134
136
  if (sample.provenance) {
135
137
  withProvenance++;
136
- provenanceBreakdown[sample.provenance] = (provenanceBreakdown[sample.provenance] || 0) + 1;
138
+ incrementRecordCount(provenanceBreakdown, sample.provenance);
137
139
  }
138
140
  else {
139
- provenanceBreakdown.unspecified = (provenanceBreakdown.unspecified || 0) + 1;
141
+ incrementRecordCount(provenanceBreakdown, 'unspecified');
140
142
  }
141
143
  // rubric length(only counted if present, NaN-safe)
142
144
  if (sample.rubric) {
@@ -192,7 +194,7 @@ export function buildRepresentativeness(aggregate) {
192
194
  const declaredConstruct = {};
193
195
  for (const [k, v] of Object.entries(aggregate.constructDistribution)) {
194
196
  if (k !== 'unspecified')
195
- declaredConstruct[k] = v;
197
+ setOwnRecordValue(declaredConstruct, k, v);
196
198
  }
197
199
  const consTotal = Object.values(declaredConstruct).reduce((a, b) => a + b, 0);
198
200
  const [dominantConstruct, constructConcentration] = dominantShare(declaredConstruct, consTotal);
@@ -715,7 +717,7 @@ function detectToolPermissionIssues(results, variants, insights) {
715
717
  for (const variant of variants) {
716
718
  const calls = result.variants?.[variant]?.toolCalls || [];
717
719
  for (const call of calls) {
718
- if (call.success)
720
+ if (!isToolCallFailure(call))
719
721
  continue;
720
722
  const output = String(call.output || '');
721
723
  if (/EACCES|permission denied/i.test(output)) {
@@ -24,6 +24,7 @@
24
24
  * sorted by severity then by sample_id so the report is stable across runs.
25
25
  */
26
26
  import { rougeN } from '../grading/assertions.js';
27
+ import { ownRecordValue, setOwnRecordValue, } from '../shared/record-count.js';
27
28
  const DEFAULTS = {
28
29
  duplicateRouge: 0.7,
29
30
  ambiguousStddev: 1.0,
@@ -193,10 +194,9 @@ export function diagnoseSamples(report, options = {}) {
193
194
  if (seenCaps.has(cap))
194
195
  continue;
195
196
  seenCaps.add(cap);
196
- if (!capabilityCount[cap])
197
- capabilityCount[cap] = { count: 0, sampleIds: [] };
198
- capabilityCount[cap].count++;
199
- capabilityCount[cap].sampleIds.push(sample.sample_id);
197
+ const info = ownRecordValue(capabilityCount, cap) ?? setOwnRecordValue(capabilityCount, cap, { count: 0, sampleIds: [] });
198
+ info.count++;
199
+ info.sampleIds.push(sample.sample_id);
200
200
  }
201
201
  }
202
202
  for (const [cap, info] of Object.entries(capabilityCount)) {
@@ -247,9 +247,9 @@ function emptyReport(samples) {
247
247
  function scoresMap(entry, variants) {
248
248
  const out = {};
249
249
  for (const v of variants) {
250
- const r = entry.variants?.[v];
250
+ const r = entry.variants ? ownRecordValue(entry.variants, v) : undefined;
251
251
  if (r && typeof r.compositeScore === 'number') {
252
- out[v] = Number(r.compositeScore.toFixed(2));
252
+ setOwnRecordValue(out, v, Number(r.compositeScore.toFixed(2)));
253
253
  }
254
254
  }
255
255
  return out;
@@ -3,12 +3,14 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSy
3
3
  import { basename, dirname, join, relative, resolve, sep } from 'node:path';
4
4
  import { cardFileName, graphFileName, safeArtifactFileStem } from '../eval-core/artifact-file-names.js';
5
5
  import { hashArtifactSource } from '../inputs/content-hash.js';
6
+ import { parseArtifactGraphDocument } from '../shared/artifact-graph.js';
7
+ import { writeJsonFileAtomic } from '../shared/atomic-json.js';
6
8
  import { extractMarkdownStepWorkflows, extractSkillHardRules, extractSkillWorkflows, parseSkillFrontmatter, } from '../shared/hard-rules.js';
7
9
  function shortHash(input) {
8
10
  return createHash('sha256').update(input).digest('hex').slice(0, 12);
9
11
  }
10
- function safeFileName(id) {
11
- return safeArtifactFileStem(id);
12
+ function isCanonicalFileStem(id) {
13
+ return id.length > 0 && safeArtifactFileStem(id) === id;
12
14
  }
13
15
  export function doctorGraphDirForDoctorOutput(doctorOutputDir) {
14
16
  return basename(doctorOutputDir) === 'doctors'
@@ -547,20 +549,26 @@ export function renderDoctorEvidenceCard(graph, skill, lang) {
547
549
  export function persistDoctorGraphSidecars(options) {
548
550
  const dir = doctorGraphDirForDoctorOutput(options.outputDir);
549
551
  mkdirSync(dir, { recursive: true });
550
- const graph = buildDoctorArtifactGraph(options);
551
- const fileStem = safeFileName(options.fileStem);
552
+ if (!isCanonicalFileStem(options.fileStem)) {
553
+ throw new Error('invalid doctor graph file stem');
554
+ }
555
+ const graph = parseArtifactGraphDocument(buildDoctorArtifactGraph(options));
556
+ if (!graph)
557
+ throw new Error('invalid doctor artifact graph');
558
+ const fileStem = options.fileStem;
552
559
  const graphPath = join(dir, graphFileName(fileStem));
553
560
  const evidenceCardPath = join(dir, cardFileName(fileStem));
554
- writeFileSync(graphPath, JSON.stringify(graph, null, 2), 'utf8');
561
+ writeJsonFileAtomic(graphPath, graph);
555
562
  writeFileSync(evidenceCardPath, renderDoctorEvidenceCard(graph, options.skill, options.lang), 'utf8');
556
563
  return { graphPath, evidenceCardPath };
557
564
  }
558
565
  export function removeDoctorGraphSidecars(doctorOutputDir, fileStem) {
566
+ if (!isCanonicalFileStem(fileStem))
567
+ return;
559
568
  const dir = doctorGraphDirForDoctorOutput(doctorOutputDir);
560
- const safeStem = safeFileName(fileStem);
561
569
  for (const ext of ['graph.json', 'card.md']) {
562
570
  try {
563
- unlinkSync(join(dir, `${safeStem}.${ext}`));
571
+ unlinkSync(join(dir, `${fileStem}.${ext}`));
564
572
  }
565
573
  catch {
566
574
  // best-effort cleanup only