oh-my-knowledge 0.47.0 → 0.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. package/README.md +50 -18
  2. package/README.zh.md +55 -23
  3. package/dist/analysis/coverage-analyzer.d.ts +1 -0
  4. package/dist/analysis/coverage-analyzer.js +125 -62
  5. package/dist/analysis/failure-clusterer.js +2 -1
  6. package/dist/analysis/gap-analyzer.d.ts +2 -2
  7. package/dist/analysis/gap-analyzer.js +13 -3
  8. package/dist/analysis/hedging-classifier.d.ts +2 -2
  9. package/dist/analysis/hedging-classifier.js +3 -4
  10. package/dist/analysis/report-diagnostics.js +9 -7
  11. package/dist/analysis/sample-diagnostics.js +6 -6
  12. package/dist/artifact-graph/doctor.js +15 -7
  13. package/dist/assets/agent-skills/omk/SKILL.md +27 -7
  14. package/dist/assets/agent-skills/omk/references/commands.md +19 -17
  15. package/dist/authoring/evolver.d.ts +10 -6
  16. package/dist/authoring/evolver.js +496 -83
  17. package/dist/authoring/generator.d.ts +3 -3
  18. package/dist/authoring/generator.js +5 -10
  19. package/dist/authoring/sample-fixer.d.ts +8 -6
  20. package/dist/authoring/sample-fixer.js +76 -5
  21. package/dist/cli/commands/doctor.js +31 -14
  22. package/dist/cli/commands/eval/index.d.ts +3 -0
  23. package/dist/cli/commands/eval/index.js +163 -17
  24. package/dist/cli/commands/evolve.d.ts +4 -4
  25. package/dist/cli/commands/evolve.js +27 -13
  26. package/dist/cli/commands/init.js +16 -3
  27. package/dist/cli/commands/observe/inbox.js +44 -21
  28. package/dist/cli/commands/observe/index.js +20 -11
  29. package/dist/cli/commands/observe/ingest.d.ts +3 -0
  30. package/dist/cli/commands/observe/ingest.js +35 -4
  31. package/dist/cli/commands/sample.d.ts +9 -3
  32. package/dist/cli/commands/sample.js +91 -74
  33. package/dist/cli/lib/cmd-flags.d.ts +1 -0
  34. package/dist/cli/lib/codex-model-hint.d.ts +9 -0
  35. package/dist/cli/lib/codex-model-hint.js +45 -0
  36. package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
  37. package/dist/cli/lib/generation-failure-hint.js +61 -0
  38. package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
  39. package/dist/cli/lib/i18n-dict/common.js +4 -0
  40. package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
  41. package/dist/cli/lib/i18n-dict/gen.js +38 -6
  42. package/dist/cli/lib/i18n-dict/help.js +6 -6
  43. package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
  44. package/dist/cli/lib/i18n-dict/init.js +13 -9
  45. package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
  46. package/dist/cli/lib/i18n-dict/run.js +34 -2
  47. package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
  48. package/dist/cli/lib/llm-failure-classifier.js +8 -0
  49. package/dist/cli/lib/parse-run-config.d.ts +6 -5
  50. package/dist/cli/lib/parse-run-config.js +16 -9
  51. package/dist/cli/lib/runtime-defaults.d.ts +21 -0
  52. package/dist/cli/lib/runtime-defaults.js +79 -0
  53. package/dist/diagnosis/observe-mapper.js +14 -15
  54. package/dist/diagnosis/observe-producer.js +3 -1
  55. package/dist/diagnosis/studio-projection.js +14 -7
  56. package/dist/diagnosis/types.d.ts +2 -0
  57. package/dist/diagnosis/types.js +12 -0
  58. package/dist/doctor/endpoint-rule.js +2 -1
  59. package/dist/eval-core/artifact-file-names.js +18 -1
  60. package/dist/eval-core/artifact-index.d.ts +7 -11
  61. package/dist/eval-core/artifact-index.js +139 -80
  62. package/dist/eval-core/cache.d.ts +12 -3
  63. package/dist/eval-core/cache.js +89 -29
  64. package/dist/eval-core/comparability.js +10 -6
  65. package/dist/eval-core/evaluation-execution.d.ts +2 -1
  66. package/dist/eval-core/evaluation-execution.js +122 -37
  67. package/dist/eval-core/evaluation-job.d.ts +4 -1
  68. package/dist/eval-core/evaluation-job.js +4 -1
  69. package/dist/eval-core/evaluation-reporting.d.ts +15 -13
  70. package/dist/eval-core/evaluation-reporting.js +54 -52
  71. package/dist/eval-core/execution-strategy.d.ts +2 -0
  72. package/dist/eval-core/execution-strategy.js +11 -9
  73. package/dist/eval-core/fact-checker.js +15 -7
  74. package/dist/eval-core/holdout.js +3 -2
  75. package/dist/eval-core/judge-independence.d.ts +2 -2
  76. package/dist/eval-core/mock-hook.cjs +23 -6
  77. package/dist/eval-core/mocks-runtime.js +30 -8
  78. package/dist/eval-core/report-document.d.ts +12 -0
  79. package/dist/eval-core/report-document.js +1151 -0
  80. package/dist/eval-core/report-extensions.d.ts +4 -0
  81. package/dist/eval-core/report-extensions.js +500 -0
  82. package/dist/eval-core/report-file-migration.js +7 -2
  83. package/dist/eval-core/resume-compatibility.d.ts +31 -0
  84. package/dist/eval-core/resume-compatibility.js +141 -0
  85. package/dist/eval-core/sample-fingerprint.d.ts +12 -0
  86. package/dist/eval-core/sample-fingerprint.js +193 -0
  87. package/dist/eval-core/schema.js +86 -31
  88. package/dist/eval-core/verdict.d.ts +8 -4
  89. package/dist/eval-core/verdict.js +24 -10
  90. package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
  91. package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
  92. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
  93. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
  94. package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
  95. package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
  96. package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
  97. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
  98. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
  99. package/dist/eval-workflows/evaluation-pipeline.js +12 -7
  100. package/dist/eval-workflows/run-evaluation.d.ts +9 -7
  101. package/dist/eval-workflows/run-evaluation.js +79 -51
  102. package/dist/executors/anthropic-api.js +65 -9
  103. package/dist/executors/claude-cli.js +16 -79
  104. package/dist/executors/claude-protocol.d.ts +28 -0
  105. package/dist/executors/claude-protocol.js +180 -0
  106. package/dist/executors/claude-sdk-trace.js +56 -28
  107. package/dist/executors/claude-sdk.d.ts +1 -0
  108. package/dist/executors/claude-sdk.js +39 -93
  109. package/dist/executors/codex-cli-trace.js +166 -31
  110. package/dist/executors/codex-cli.d.ts +6 -8
  111. package/dist/executors/codex-cli.js +49 -151
  112. package/dist/executors/codex-protocol.d.ts +24 -0
  113. package/dist/executors/codex-protocol.js +234 -0
  114. package/dist/executors/codex-sdk.js +68 -120
  115. package/dist/executors/gemini.js +88 -13
  116. package/dist/executors/index.d.ts +2 -3
  117. package/dist/executors/index.js +5 -3
  118. package/dist/executors/openai-api.js +70 -9
  119. package/dist/executors/runtime-fingerprint.js +88 -11
  120. package/dist/executors/script-command.d.ts +8 -0
  121. package/dist/executors/script-command.js +87 -0
  122. package/dist/executors/script.js +202 -29
  123. package/dist/executors/shared.d.ts +35 -3
  124. package/dist/executors/shared.js +113 -15
  125. package/dist/grading/assertions.d.ts +1 -1
  126. package/dist/grading/assertions.js +19 -9
  127. package/dist/grading/diagnostic.d.ts +9 -2
  128. package/dist/grading/diagnostic.js +25 -2
  129. package/dist/grading/index.js +10 -4
  130. package/dist/grading/judge.js +19 -6
  131. package/dist/grading/layered-scores.d.ts +2 -3
  132. package/dist/grading/layered-scores.js +2 -3
  133. package/dist/inputs/load-samples.d.ts +1 -2
  134. package/dist/inputs/load-samples.js +23 -1
  135. package/dist/inputs/mcp-resolver.js +6 -3
  136. package/dist/inputs/sample-document.d.ts +11 -0
  137. package/dist/inputs/sample-document.js +96 -0
  138. package/dist/managed/evidence.d.ts +1 -0
  139. package/dist/managed/evidence.js +1 -1
  140. package/dist/managed/store.js +200 -91
  141. package/dist/observability/codex-trace-adapter.d.ts +5 -0
  142. package/dist/observability/codex-trace-adapter.js +850 -0
  143. package/dist/observability/experience.d.ts +32 -6
  144. package/dist/observability/experience.js +2695 -459
  145. package/dist/observability/feedback-matchers.js +16 -1
  146. package/dist/observability/inbox-view-model.d.ts +2 -1
  147. package/dist/observability/inbox-view-model.js +20 -14
  148. package/dist/observability/inbox.d.ts +7 -1
  149. package/dist/observability/inbox.js +632 -124
  150. package/dist/observability/problem-patterns.js +2 -0
  151. package/dist/observability/review-state.d.ts +6 -0
  152. package/dist/observability/review-state.js +235 -63
  153. package/dist/observability/skill-chain-advisories.js +1 -1
  154. package/dist/observability/skill-chain.js +17 -4
  155. package/dist/observability/skill-health-analyzer.d.ts +32 -7
  156. package/dist/observability/skill-health-analyzer.js +194 -121
  157. package/dist/observability/skill-health-report.d.ts +10 -0
  158. package/dist/observability/skill-health-report.js +620 -0
  159. package/dist/observability/soft-standards/constants.d.ts +0 -1
  160. package/dist/observability/soft-standards/constants.js +0 -1
  161. package/dist/observability/soft-standards/index.d.ts +1 -1
  162. package/dist/observability/soft-standards/index.js +1 -1
  163. package/dist/observability/soft-standards/llm-extractor.js +8 -10
  164. package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
  165. package/dist/observability/soft-standards/skill-standards-store.js +59 -18
  166. package/dist/observability/soft-standards/types.d.ts +2 -2
  167. package/dist/observability/trace-adapter.d.ts +12 -7
  168. package/dist/observability/trace-adapter.js +11 -9
  169. package/dist/observability/trace-attribution.d.ts +13 -5
  170. package/dist/observability/trace-attribution.js +315 -21
  171. package/dist/observability/trace-ingestion.d.ts +9 -0
  172. package/dist/observability/trace-ingestion.js +80 -0
  173. package/dist/observability/trace-ir.d.ts +113 -0
  174. package/dist/observability/trace-ir.js +87 -0
  175. package/dist/observability/trace-segmenter.d.ts +19 -6
  176. package/dist/observability/trace-segmenter.js +377 -196
  177. package/dist/observability/trace-session-index.d.ts +19 -0
  178. package/dist/observability/trace-session-index.js +68 -0
  179. package/dist/observability/trace-source.d.ts +12 -4
  180. package/dist/observability/trace-source.js +939 -215
  181. package/dist/renderer/html-renderer.js +37 -6
  182. package/dist/renderer/icons.js +3 -0
  183. package/dist/renderer/observation-inbox-renderer.js +227 -91
  184. package/dist/renderer/skill-detail-renderer.js +452 -109
  185. package/dist/renderer/skill-health-renderer.js +69 -12
  186. package/dist/renderer/summary.js +28 -7
  187. package/dist/renderer/table.js +21 -4
  188. package/dist/renderer/test-view.d.ts +1 -0
  189. package/dist/renderer/test-view.js +44 -9
  190. package/dist/server/indexed-report-store.js +14 -18
  191. package/dist/server/job-store.js +64 -26
  192. package/dist/server/report-server.js +190 -78
  193. package/dist/server/report-store.js +57 -80
  194. package/dist/server/skill-index.js +143 -49
  195. package/dist/server/skill-insights.js +44 -5
  196. package/dist/shared/artifact-graph.d.ts +3 -0
  197. package/dist/shared/artifact-graph.js +224 -0
  198. package/dist/shared/assertion-types.d.ts +8 -0
  199. package/dist/shared/assertion-types.js +46 -0
  200. package/dist/shared/atomic-json.d.ts +8 -0
  201. package/dist/shared/atomic-json.js +33 -0
  202. package/dist/shared/diagnosis-schema.d.ts +9 -0
  203. package/dist/shared/diagnosis-schema.js +181 -0
  204. package/dist/shared/doctor-report.d.ts +3 -0
  205. package/dist/shared/doctor-report.js +103 -0
  206. package/dist/shared/evaluation-job.d.ts +6 -0
  207. package/dist/shared/evaluation-job.js +217 -0
  208. package/dist/shared/executor-result.d.ts +17 -0
  209. package/dist/shared/executor-result.js +221 -0
  210. package/dist/shared/file-lock.d.ts +12 -0
  211. package/dist/shared/file-lock.js +129 -0
  212. package/dist/shared/json-value.d.ts +5 -0
  213. package/dist/shared/json-value.js +36 -0
  214. package/dist/shared/keyed-mutex.d.ts +7 -0
  215. package/dist/shared/keyed-mutex.js +24 -0
  216. package/dist/shared/record-count.d.ts +8 -0
  217. package/dist/shared/record-count.js +43 -0
  218. package/dist/shared/sample-contract.d.ts +3 -0
  219. package/dist/shared/sample-contract.js +332 -0
  220. package/dist/shared/shell-quote.d.ts +2 -0
  221. package/dist/shared/shell-quote.js +7 -0
  222. package/dist/shared/timestamp.d.ts +6 -0
  223. package/dist/shared/timestamp.js +64 -0
  224. package/dist/shared/token-usage.d.ts +19 -0
  225. package/dist/shared/token-usage.js +50 -0
  226. package/dist/shared/tool-call-status.d.ts +8 -0
  227. package/dist/shared/tool-call-status.js +28 -0
  228. package/dist/shared/tool-identity.d.ts +21 -0
  229. package/dist/shared/tool-identity.js +84 -0
  230. package/dist/shared/tool-search.js +73 -16
  231. package/dist/shared/trace-projection.d.ts +5 -0
  232. package/dist/shared/trace-projection.js +20 -0
  233. package/dist/shared/trace-source-kind.d.ts +3 -0
  234. package/dist/shared/trace-source-kind.js +12 -0
  235. package/dist/types/diagnosis.d.ts +2 -0
  236. package/dist/types/eval.d.ts +4 -0
  237. package/dist/types/executor.d.ts +32 -5
  238. package/dist/types/index.d.ts +1 -0
  239. package/dist/types/index.js +1 -0
  240. package/dist/types/judge.d.ts +2 -0
  241. package/dist/types/observability.d.ts +116 -9
  242. package/dist/types/report.d.ts +58 -6
  243. package/dist/types/skill-index.d.ts +7 -0
  244. package/dist/types/trace.d.ts +2 -0
  245. package/dist/types/trace.js +1 -0
  246. package/package.json +9 -5
@@ -8,7 +8,7 @@ import { renderSkillHealthReport } from '../renderer/skill-health-renderer.js';
8
8
  import { renderDoctorDetail } from '../renderer/doctor-detail-renderer.js';
9
9
  import { assessHealth, renderSkillDetail } from '../renderer/skill-detail-renderer.js';
10
10
  import { renderObservationInboxPage } from '../renderer/observation-inbox-renderer.js';
11
- import { DEFAULT_LANG, t, layout } from '../renderer/layout.js';
11
+ import { DEFAULT_LANG, e, t, layout } from '../renderer/layout.js';
12
12
  import { loadAllManagedRecords, resolveManagedDir, managedDir as projectManagedDir, listManagedRows, buildVersionScores } from '../managed/index.js';
13
13
  import { renderManagedList, renderManagedHistory } from '../renderer/managed-history-renderer.js';
14
14
  import { DEFAULT_JOBS_DIR } from '../eval-core/default-dirs.js';
@@ -21,12 +21,13 @@ import { buildSkillIndex } from './skill-index.js';
21
21
  import { createFileJobStore } from './job-store.js';
22
22
  import { createFileStore, queryJob, queryJobList, queryRun, queryRunList, queryTrend } from './report-store.js';
23
23
  import { createIndexedReportStore } from './indexed-report-store.js';
24
- import { confidenceOf } from '../observability/skill-health-analyzer.js';
24
+ import { parseDoctorReport } from '../shared/doctor-report.js';
25
+ import { confidenceOf, toolStabilityOf, } from '../observability/skill-health-analyzer.js';
26
+ import { parseSkillHealthReport } from '../observability/skill-health-report.js';
25
27
  import { DEFAULT_OBSERVATIONS_DIR, findObservationInboxItem, formatObservationShow, queryObservationInbox } from '../observability/inbox.js';
26
28
  import { buildObservationInboxViewModel } from '../observability/inbox-view-model.js';
27
29
  import { activeStudioDiagnostics } from '../diagnosis/studio-projection.js';
28
- import { deleteObservationReviewState, loadObservationReviewState, updateObservationReviewState } from '../observability/review-state.js';
29
- import { updateSkillDerivedStandardStatus } from '../observability/soft-standards/index.js';
30
+ import { deleteObservationReviewState, loadObservationReviewState, ObservationReviewStateValidationError, updateObservationReviewState, } from '../observability/review-state.js';
30
31
  const DEFAULT_PORT = 7799;
31
32
  const PORT_HINT = `OMK_REPORT_PORT=${DEFAULT_PORT} omk eval ...`;
32
33
  let cachedChartJsBytes;
@@ -58,8 +59,8 @@ function listAnalyses(dir, includeCards = false) {
58
59
  if (!id)
59
60
  continue;
60
61
  try {
61
- const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
62
- if (!data.meta || !data.overall)
62
+ const data = parseSkillHealthReport(JSON.parse(readFileSync(join(dir, file), 'utf-8')));
63
+ if (!data)
63
64
  continue;
64
65
  items.push({
65
66
  id,
@@ -82,11 +83,25 @@ function listAnalyses(dir, includeCards = false) {
82
83
  for (const card of listLiveObserveCards()) {
83
84
  if (seen.has(card.id))
84
85
  continue;
86
+ let report = null;
87
+ try {
88
+ report = parseSkillHealthReport(JSON.parse(readFileSync(card.path, 'utf-8')));
89
+ }
90
+ catch {
91
+ // Scratch cards only discover the canonical report; corrupt targets stay invisible.
92
+ }
93
+ if (!report)
94
+ continue;
85
95
  items.push({
86
- id: card.id, generatedAt: card.meta.generatedAt, sessionCount: card.meta.sessionCount,
87
- segmentCount: card.meta.segmentCount, skillCount: Object.keys(card.bySkill).length,
88
- healthBand: card.overall.healthBand, confidence: card.overall.confidence ?? confidenceOf(card.meta.segmentCount),
96
+ id: card.id,
97
+ generatedAt: report.meta.generatedAt,
98
+ sessionCount: report.meta.sessionCount,
99
+ segmentCount: report.meta.segmentCount,
100
+ skillCount: Object.keys(report.bySkill).length,
101
+ healthBand: report.overall.healthBand,
102
+ confidence: report.overall.confidence ?? confidenceOf(report.meta.segmentCount),
89
103
  });
104
+ seen.add(card.id);
90
105
  }
91
106
  }
92
107
  // 最新在前
@@ -98,7 +113,9 @@ function loadAnalysis(dir, id, includeCards = false) {
98
113
  const path = reportFilePath(dir, id);
99
114
  if (existsSync(path)) {
100
115
  try {
101
- return JSON.parse(readFileSync(path, 'utf-8'));
116
+ const report = parseSkillHealthReport(JSON.parse(readFileSync(path, 'utf-8')));
117
+ if (report)
118
+ return report;
102
119
  }
103
120
  catch { /* fall through to card */ }
104
121
  }
@@ -109,7 +126,7 @@ function loadAnalysis(dir, id, includeCards = false) {
109
126
  const card = listObserveCards().find((c) => c.id === id);
110
127
  if (card && existsSync(card.path)) {
111
128
  try {
112
- return JSON.parse(readFileSync(card.path, 'utf-8'));
129
+ return parseSkillHealthReport(JSON.parse(readFileSync(card.path, 'utf-8')));
113
130
  }
114
131
  catch { /* corrupt 真身 */ }
115
132
  }
@@ -126,8 +143,8 @@ function loadDoctorReport(dir, id, skillName, includeCards = false) {
126
143
  if (!isReportFileName(file))
127
144
  continue;
128
145
  try {
129
- const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
130
- if (data?.kind !== 'doctor' || data.id !== id)
146
+ const data = parseDoctorReport(JSON.parse(readFileSync(join(dir, file), 'utf-8')));
147
+ if (!data || data.id !== id)
131
148
  continue;
132
149
  if (!skillName || data.skills?.some((s) => s.skillName === skillName))
133
150
  return data;
@@ -150,8 +167,8 @@ function loadDoctorReport(dir, id, skillName, includeCards = false) {
150
167
  if (!existsSync(card.path))
151
168
  continue;
152
169
  try {
153
- const data = JSON.parse(readFileSync(card.path, 'utf-8'));
154
- if (data?.kind === 'doctor' && data.id === id) {
170
+ const data = parseDoctorReport(JSON.parse(readFileSync(card.path, 'utf-8')));
171
+ if (data && data.id === id) {
155
172
  if (!skillName || data.skills?.some((s) => s.skillName === skillName))
156
173
  return data;
157
174
  fallback ??= data;
@@ -178,26 +195,59 @@ function querySkillTrend(dir, skillName, includeCards = false) {
178
195
  const u = h.usage;
179
196
  const billable = (u?.inputTokens ?? 0) + (u?.outputTokens ?? 0);
180
197
  const cached = (u?.cacheReadTokens ?? 0) + (u?.cacheCreationTokens ?? 0);
198
+ const toolCallCount = h.toolCallCount ?? 0;
199
+ const toolResolvedCount = h.toolResolvedCount ?? toolCallCount;
200
+ const toolCancelledCount = h.toolCancelledCount ?? 0;
201
+ const toolComparableCount = Math.max(0, toolResolvedCount - toolCancelledCount);
181
202
  points.push({
182
203
  analysisId: it.id,
183
204
  generatedAt: report.meta.generatedAt,
184
205
  gapRate: h.gap?.gapRate ?? 0,
185
206
  weightedGapRate: h.gap?.weightedGapRate ?? 0,
186
- failureRate: h.toolFailureRate ?? 0,
207
+ failureRate: measuredToolFailureRate(h),
208
+ toolCallCount,
209
+ toolResolvedCount,
210
+ toolComparableCount,
211
+ toolCancelledCount,
212
+ toolOutcomeCoverage: toolCallCount > 0
213
+ ? Number((toolResolvedCount / toolCallCount).toFixed(4))
214
+ : null,
187
215
  coverageRate: h.coverage?.fileCoverageRate ?? null,
188
216
  billableTokens: billable,
189
217
  cachedTokens: cached,
190
218
  totalTokens: u?.totalTokens ?? 0,
191
219
  avgTokensPerSegment: u?.avgTokensPerSegment ?? 0,
220
+ tokenCoverage: u?.tokenCoverage ?? 0,
192
221
  durationMs: u?.durationMs ?? 0,
193
222
  segmentCount: h.segmentCount ?? 0,
194
- stability: h.stability ?? 'stable',
223
+ stability: observedToolStability(h),
195
224
  });
196
225
  }
197
226
  // 最旧在前,便于折线图从左到右展示时间序列
198
227
  points.sort((a, b) => a.generatedAt.localeCompare(b.generatedAt));
199
228
  return { skillName, points };
200
229
  }
230
+ function measuredToolFailureRate(health) {
231
+ if (health.stability === 'unknown')
232
+ return null;
233
+ if (health.toolResolvedCount !== undefined || health.toolCancelledCount !== undefined) {
234
+ const comparable = Math.max(0, (health.toolResolvedCount ?? health.toolCallCount) - (health.toolCancelledCount ?? 0));
235
+ return comparable > 0 ? health.toolFailureRate : null;
236
+ }
237
+ // Legacy reports predate toolResolvedCount but treated every recorded call as
238
+ // resolved. Preserve those measured rates when a non-zero denominator exists.
239
+ return health.toolCallCount > 0 ? health.toolFailureRate : null;
240
+ }
241
+ function observedToolStability(health) {
242
+ if (health.stability === 'unknown')
243
+ return 'unknown';
244
+ const failureRate = measuredToolFailureRate(health);
245
+ if (failureRate == null)
246
+ return 'unknown';
247
+ const resolvedToolCalls = health.toolResolvedCount ?? health.toolCallCount;
248
+ const comparableToolCalls = Math.max(0, resolvedToolCalls - (health.toolCancelledCount ?? 0));
249
+ return toolStabilityOf(failureRate, comparableToolCalls, health.toolCallCount);
250
+ }
201
251
  /**
202
252
  * 比较两份 skill health report. `from` 通常是较早的,`to` 是较晚的;
203
253
  * 对于每个 skill, 显示前后值和 delta. 缺失一侧时 presence 标记。
@@ -213,15 +263,19 @@ function querySkillDiff(dir, fromId, toId, includeCards = false) {
213
263
  const f = from.bySkill[skill];
214
264
  const t = to.bySkill[skill];
215
265
  if (f && t) {
266
+ const fromFailure = measuredToolFailureRate(f);
267
+ const toFailure = measuredToolFailureRate(t);
216
268
  rows.push({
217
269
  skillName: skill,
218
270
  presence: 'both',
219
271
  fromGap: f.gap.weightedGapRate,
220
272
  toGap: t.gap.weightedGapRate,
221
273
  deltaGap: t.gap.weightedGapRate - f.gap.weightedGapRate,
222
- fromFailure: f.toolFailureRate,
223
- toFailure: t.toolFailureRate,
224
- deltaFailure: t.toolFailureRate - f.toolFailureRate,
274
+ fromFailure,
275
+ toFailure,
276
+ deltaFailure: fromFailure != null && toFailure != null
277
+ ? toFailure - fromFailure
278
+ : undefined,
225
279
  fromCoverage: f.coverage?.fileCoverageRate ?? null,
226
280
  toCoverage: t.coverage?.fileCoverageRate ?? null,
227
281
  deltaCoverage: (f.coverage?.fileCoverageRate != null && t.coverage?.fileCoverageRate != null)
@@ -233,10 +287,10 @@ function querySkillDiff(dir, fromId, toId, includeCards = false) {
233
287
  });
234
288
  }
235
289
  else if (f) {
236
- rows.push({ skillName: skill, presence: 'only-from', fromGap: f.gap.weightedGapRate, fromFailure: f.toolFailureRate, fromCoverage: f.coverage?.fileCoverageRate ?? null, fromSegments: f.segmentCount });
290
+ rows.push({ skillName: skill, presence: 'only-from', fromGap: f.gap.weightedGapRate, fromFailure: measuredToolFailureRate(f), fromCoverage: f.coverage?.fileCoverageRate ?? null, fromSegments: f.segmentCount });
237
291
  }
238
292
  else if (t) {
239
- rows.push({ skillName: skill, presence: 'only-to', toGap: t.gap.weightedGapRate, toFailure: t.toolFailureRate, toCoverage: t.coverage?.fileCoverageRate ?? null, toSegments: t.segmentCount });
293
+ rows.push({ skillName: skill, presence: 'only-to', toGap: t.gap.weightedGapRate, toFailure: measuredToolFailureRate(t), toCoverage: t.coverage?.fileCoverageRate ?? null, toSegments: t.segmentCount });
240
294
  }
241
295
  }
242
296
  // 按 deltaGap 绝对值倒序 (变化大的在前,缺失的放最后)
@@ -260,26 +314,78 @@ function fmtPct(v) {
260
314
  return '—';
261
315
  return `${Math.round(v * 100)}%`;
262
316
  }
263
- function readJsonBody(req, maxBytes = 1024 * 1024) {
317
+ class RequestBodyError extends Error {
318
+ statusCode;
319
+ name = 'RequestBodyError';
320
+ constructor(message, statusCode) {
321
+ super(message);
322
+ this.statusCode = statusCode;
323
+ }
324
+ }
325
+ function assertTrustedMutationRequest(req) {
326
+ const fetchSite = req.headers['sec-fetch-site'];
327
+ if (fetchSite === 'cross-site') {
328
+ throw new RequestBodyError('cross-origin mutation is not allowed', 403);
329
+ }
330
+ const origin = req.headers.origin;
331
+ if (!origin)
332
+ return;
333
+ if (Array.isArray(origin) || !req.headers.host) {
334
+ throw new RequestBodyError('cross-origin mutation is not allowed', 403);
335
+ }
336
+ try {
337
+ if (new URL(origin).host !== req.headers.host) {
338
+ throw new RequestBodyError('cross-origin mutation is not allowed', 403);
339
+ }
340
+ }
341
+ catch (error) {
342
+ if (error instanceof RequestBodyError)
343
+ throw error;
344
+ throw new RequestBodyError('cross-origin mutation is not allowed', 403);
345
+ }
346
+ }
347
+ function assertJsonContentType(req) {
348
+ const contentType = req.headers['content-type'];
349
+ const mediaType = typeof contentType === 'string'
350
+ ? contentType.split(';', 1)[0].trim().toLowerCase()
351
+ : '';
352
+ if (mediaType === 'application/json' || mediaType.endsWith('+json'))
353
+ return;
354
+ req.resume();
355
+ throw new RequestBodyError('content-type must be application/json', 415);
356
+ }
357
+ function readJsonObjectBody(req, maxBytes = 1024 * 1024) {
358
+ assertJsonContentType(req);
264
359
  return new Promise((resolve, reject) => {
265
360
  let size = 0;
361
+ let tooLarge = false;
266
362
  const chunks = [];
267
363
  req.on('data', (chunk) => {
268
364
  size += chunk.length;
269
365
  if (size > maxBytes) {
270
- reject(new Error('request body too large'));
271
- req.destroy();
366
+ tooLarge = true;
367
+ chunks.length = 0;
272
368
  return;
273
369
  }
274
- chunks.push(chunk);
370
+ if (!tooLarge)
371
+ chunks.push(chunk);
275
372
  });
276
373
  req.on('end', () => {
374
+ if (tooLarge) {
375
+ reject(new RequestBodyError('request body too large', 413));
376
+ return;
377
+ }
277
378
  try {
278
379
  const raw = Buffer.concat(chunks).toString('utf-8').trim();
279
- resolve(raw ? JSON.parse(raw) : {});
380
+ const parsed = raw ? JSON.parse(raw) : {};
381
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
382
+ reject(new RequestBodyError('json body must be an object', 400));
383
+ return;
384
+ }
385
+ resolve(parsed);
280
386
  }
281
387
  catch {
282
- reject(new Error('invalid json body'));
388
+ reject(new RequestBodyError('invalid json body', 400));
283
389
  }
284
390
  });
285
391
  req.on('error', reject);
@@ -313,8 +419,9 @@ function buildSkillContext(entry, currentDim, currentReportId, insights, lang) {
313
419
  const evBand = ev && ev.compositeScore != null
314
420
  ? (ev.compositeScore >= 4 ? 'green' : ev.compositeScore >= 3 ? 'yellow' : 'red') : 'gray';
315
421
  const ob = entry.observe;
316
- const obScore = ob ? Math.round((1 - ob.gapRate) * 100) : null;
317
- const obBand = ob ? ob.healthBand : 'gray';
422
+ const obTrustworthy = ob != null && ob.confidence !== 'underpowered';
423
+ const obScore = obTrustworthy ? Math.round((1 - ob.gapRate) * 100) : null;
424
+ const obBand = obTrustworthy ? ob.healthBand : 'gray';
318
425
  // 历史:eval + doctor 历次 run,各自新→旧。点行跳到那次报告。
319
426
  // evalHistory 是「每轮一条」,但一份 evolve 报告(多轮)是同一个 reportId —— 按 reportId 去重,
320
427
  // 一份报告只算一条(取末轮 treatment 分数,与报告头部一致),否则同一报告的多轮会都标「当前」。
@@ -372,7 +479,7 @@ function renderSkillDiffPage(diff, lang = DEFAULT_LANG) {
372
479
  : r.presence === 'only-to' ? `<span style="color:var(--accent);font-size:10px;padding:1px 6px;background:var(--info-bg);border-radius:3px" data-i18n="diffTagNew">${t('diffTagNew', lang)}</span>`
373
480
  : '';
374
481
  return `<tr>
375
- <td style="padding:8px 10px;font-family:ui-monospace,monospace">${r.skillName} ${tag}</td>
482
+ <td style="padding:8px 10px;font-family:ui-monospace,monospace">${e(r.skillName)} ${tag}</td>
376
483
  <td style="padding:8px 10px;text-align:right">${r.fromSegments ?? '—'} → ${r.toSegments ?? '—'} ${r.presence === 'both' ? `(${fmtDelta(r.deltaSegments, false)})` : ''}</td>
377
484
  <td style="padding:8px 10px;text-align:right">${fmtPct(r.fromGap)} → ${fmtPct(r.toGap)} ${r.presence === 'both' ? fmtDelta(r.deltaGap) : ''}</td>
378
485
  <td style="padding:8px 10px;text-align:right">${fmtPct(r.fromFailure)} → ${fmtPct(r.toFailure)} ${r.presence === 'both' ? fmtDelta(r.deltaFailure) : ''}</td>
@@ -388,7 +495,7 @@ function renderSkillDiffPage(diff, lang = DEFAULT_LANG) {
388
495
  </nav>
389
496
  <h1 data-i18n="skillDiffHeading" style="font-size:20px;margin:8px 0">${t('skillDiffHeading', lang)}</h1>
390
497
  <div style="color:var(--text-muted);font-size:13px;margin-bottom:20px">
391
- <span data-i18n="diffNavFrom">${t('diffNavFrom', lang)}</span> <code>${fromId}</code> (${fromAt.slice(0, 19).replace('T', ' ')}) → <span data-i18n="diffNavTo">${t('diffNavTo', lang)}</span> <code>${toId}</code> (${toAt.slice(0, 19).replace('T', ' ')})<br/>
498
+ <span data-i18n="diffNavFrom">${t('diffNavFrom', lang)}</span> <code>${e(fromId)}</code> (${e(fromAt.slice(0, 19).replace('T', ' '))}) → <span data-i18n="diffNavTo">${t('diffNavTo', lang)}</span> <code>${e(toId)}</code> (${e(toAt.slice(0, 19).replace('T', ' '))})<br/>
392
499
  <span data-i18n="diffSortHint">${t('diffSortHint', lang)}</span>
393
500
  </div>
394
501
  <table style="border-collapse:collapse;width:100%;font-size:13px">
@@ -411,7 +518,7 @@ function renderSkillTrendPage(trend, lang = DEFAULT_LANG) {
411
518
  const emptyBody = `
412
519
  <main style="max-width:900px;margin:0 auto;padding:24px">
413
520
  <nav style="margin-bottom:12px"><a href="/observe-health${langQ}" data-i18n="backToAnalyses" style="color:var(--accent);text-decoration:none">${t('backToAnalyses', lang)}</a></nav>
414
- <h1 style="font-size:20px;margin:8px 0 4px"><span data-i18n="skillTrendHeading">${t('skillTrendHeading', lang)}</span> · ${skillName}</h1>
521
+ <h1 style="font-size:20px;margin:8px 0 4px"><span data-i18n="skillTrendHeading">${t('skillTrendHeading', lang)}</span> · ${e(skillName)}</h1>
415
522
  <p style="color:var(--text-muted)" data-i18n="noTrendData">${t('noTrendData', lang)}</p>
416
523
  </main>`;
417
524
  return layout(`${t('skillTrendHeading', lang)} · ${skillName}`, emptyBody, lang);
@@ -423,10 +530,14 @@ function renderSkillTrendPage(trend, lang = DEFAULT_LANG) {
423
530
  const pathOf = (key) => {
424
531
  const usable = points.map((p, i) => ({ x: toX(i), y: p[key] ?? null }));
425
532
  let d = '';
533
+ let drawing = false;
426
534
  for (const pt of usable) {
427
- if (pt.y == null)
535
+ if (pt.y == null) {
536
+ drawing = false;
428
537
  continue;
429
- d += d ? ` L ${pt.x} ${toY(pt.y)}` : `M ${pt.x} ${toY(pt.y)}`;
538
+ }
539
+ d += drawing ? ` L ${pt.x} ${toY(pt.y)}` : `M ${pt.x} ${toY(pt.y)}`;
540
+ drawing = true;
430
541
  }
431
542
  return d;
432
543
  };
@@ -446,21 +557,28 @@ function renderSkillTrendPage(trend, lang = DEFAULT_LANG) {
446
557
  <span style="color:#a78bfa">● <span data-i18n="trendLegendFailure">${t('trendLegendFailure', lang)}</span></span> ·
447
558
  <span style="color:#4ade80">● <span data-i18n="trendLegendCoverage">${t('trendLegendCoverage', lang)}</span></span>
448
559
  </div>`;
449
- const rows = points.map((p) => `<tr>
450
- <td style="padding:6px 10px;font-family:ui-monospace,monospace;font-size:12px"><a href="/observe-health/${encodeURIComponent(p.analysisId)}${langQ}" style="color:var(--accent);text-decoration:none">${p.generatedAt.slice(0, 19).replace('T', ' ')}</a></td>
560
+ const rows = points.map((p) => {
561
+ const failureCell = p.failureRate == null
562
+ ? '—'
563
+ : `${Math.round(p.failureRate * 100)}%${p.toolCallCount > 0
564
+ ? `<br><span style="color:var(--text-muted);font-size:11px">${p.toolComparableCount}/${p.toolCallCount} ${lang === 'zh' ? '结果可比较' : 'comparable'}${p.toolCancelledCount > 0 ? ` · ${p.toolCancelledCount} ${lang === 'zh' ? '取消' : 'cancelled'}` : ''}</span>`
565
+ : ''}`;
566
+ return `<tr>
567
+ <td style="padding:6px 10px;font-family:ui-monospace,monospace;font-size:12px"><a href="/observe-health/${encodeURIComponent(p.analysisId)}${langQ}" style="color:var(--accent);text-decoration:none">${e(p.generatedAt.slice(0, 19).replace('T', ' '))}</a></td>
451
568
  <td style="padding:6px 10px;text-align:right">${p.segmentCount}</td>
452
569
  <td style="padding:6px 10px;text-align:right;color:#f87171">${Math.round(p.gapRate * 100)}%</td>
453
570
  <td style="padding:6px 10px;text-align:right;color:#fbbf24">${Math.round(p.weightedGapRate * 100)}%</td>
454
- <td style="padding:6px 10px;text-align:right;color:#a78bfa">${Math.round(p.failureRate * 100)}%</td>
571
+ <td style="padding:6px 10px;text-align:right;color:#a78bfa">${failureCell}</td>
455
572
  <td style="padding:6px 10px;text-align:right;color:#4ade80">${p.coverageRate == null ? '—' : Math.round(p.coverageRate * 100) + '%'}</td>
456
573
  <td style="padding:6px 10px;text-align:right;font-family:ui-monospace,monospace;font-size:12px" title="input+output only; cache 分开计">${(p.billableTokens / 1000).toFixed(1)}k</td>
457
574
  <td style="padding:6px 10px;text-align:right;font-family:ui-monospace,monospace;font-size:12px">${(p.durationMs / 1000).toFixed(1)}s</td>
458
- </tr>`).join('');
459
- const subtitle = `${points.length} <span data-i18n="trendNPoints">${t('trendNPoints', lang)}</span> · <span data-i18n="trendEarliest">${t('trendEarliest', lang)}</span> ${points[0].generatedAt.slice(0, 10)} · <span data-i18n="trendLatest">${t('trendLatest', lang)}</span> ${points[points.length - 1].generatedAt.slice(0, 10)}`;
575
+ </tr>`;
576
+ }).join('');
577
+ const subtitle = `${points.length} <span data-i18n="trendNPoints">${t('trendNPoints', lang)}</span> · <span data-i18n="trendEarliest">${t('trendEarliest', lang)}</span> ${e(points[0].generatedAt.slice(0, 10))} · <span data-i18n="trendLatest">${t('trendLatest', lang)}</span> ${e(points[points.length - 1].generatedAt.slice(0, 10))}`;
460
578
  const body = `
461
579
  <main style="max-width:900px;margin:0 auto;padding:24px">
462
580
  <nav style="margin-bottom:8px"><a href="/observe-health${langQ}" data-i18n="backToAnalyses" style="color:var(--accent);text-decoration:none">${t('backToAnalyses', lang)}</a></nav>
463
- <h1 style="font-size:20px;margin:8px 0 4px"><span data-i18n="skillTrendHeading">${t('skillTrendHeading', lang)}</span> · ${skillName}</h1>
581
+ <h1 style="font-size:20px;margin:8px 0 4px"><span data-i18n="skillTrendHeading">${t('skillTrendHeading', lang)}</span> · ${e(skillName)}</h1>
464
582
  <div style="color:var(--text-muted);font-size:13px;margin-bottom:16px">${subtitle}</div>
465
583
  ${svg}
466
584
  ${legend}
@@ -619,6 +737,14 @@ export function createReportServer({ port, host: hostOption, reportsDir, analyse
619
737
  try {
620
738
  const parsed = new URL(req.url || '/', 'http://127.0.0.1');
621
739
  const path = parsed.pathname;
740
+ try {
741
+ decodeURIComponent(path);
742
+ }
743
+ catch {
744
+ res.writeHead(404, { 'Content-Type': 'text/plain; charset=utf-8' });
745
+ res.end('Not Found');
746
+ return;
747
+ }
622
748
  const langParam = parsed.searchParams.get('lang');
623
749
  const lang = langParam === 'en' ? 'en' : langParam === 'zh' ? 'zh' : DEFAULT_LANG;
624
750
  // observe-health / doctors 目录每请求解析一次(项目优先→全局兜底),下面各 handler 用这两个 local 字符串。
@@ -643,6 +769,7 @@ export function createReportServer({ port, host: hostOption, reportsDir, analyse
643
769
  return;
644
770
  }
645
771
  if (path === '/api/shutdown' && req.method === 'POST') {
772
+ assertTrustedMutationRequest(req);
646
773
  res.writeHead(200, { 'Content-Type': 'application/json' });
647
774
  res.end(JSON.stringify({ ok: true }));
648
775
  // Graceful shutdown after response sent
@@ -794,39 +921,36 @@ export function createReportServer({ port, host: hostOption, reportsDir, analyse
794
921
  return;
795
922
  }
796
923
  if (req.method === 'POST') {
797
- const body = await readJsonBody(req);
924
+ assertTrustedMutationRequest(req);
925
+ const body = await readJsonObjectBody(req);
798
926
  const targetType = body.targetType;
799
- const targetId = String(body.targetId ?? '');
927
+ const targetId = body.targetId;
800
928
  const verdict = body.verdict;
801
929
  const now = new Date().toISOString();
802
- if (targetType === 'soft_standard') {
803
- const parsedTarget = parseSoftStandardReviewTarget(targetId);
804
- const status = skillDerivedStandardStatusFromVerdict(verdict);
805
- if (parsedTarget && status) {
806
- updateSkillDerivedStandardStatus(observationsDir, parsedTarget.skillName, parsedTarget.standardId, status, now);
807
- }
808
- }
809
930
  const state = updateObservationReviewState(observationsDir, {
810
931
  targetType,
811
932
  targetId,
812
933
  verdict,
813
- note: typeof body.note === 'string' ? body.note : undefined,
814
- reason: typeof body.reason === 'string' ? body.reason : undefined,
934
+ note: body.note,
935
+ reason: body.reason,
815
936
  metricKey: body.metricKey,
816
937
  metricScope: body.metricScope,
817
- metricScopeId: typeof body.metricScopeId === 'string' ? body.metricScopeId : undefined,
818
- sourceTrace: typeof body.sourceTrace === 'string' ? body.sourceTrace : undefined,
819
- sessionId: typeof body.sessionId === 'string' ? body.sessionId : undefined,
820
- messageIndex: typeof body.messageIndex === 'number' ? body.messageIndex : undefined,
821
- messageUuid: typeof body.messageUuid === 'string' ? body.messageUuid : undefined,
822
- toolUseId: typeof body.toolUseId === 'string' ? body.toolUseId : undefined,
823
- snippet: typeof body.snippet === 'string' ? body.snippet : undefined,
938
+ metricScopeId: body.metricScopeId,
939
+ traceId: body.traceId,
940
+ sourceTrace: body.sourceTrace,
941
+ sessionId: body.sessionId,
942
+ messageIndex: body.messageIndex,
943
+ messageUuid: body.messageUuid,
944
+ callInstanceId: body.callInstanceId,
945
+ toolUseId: body.toolUseId,
946
+ snippet: body.snippet,
824
947
  }, now);
825
948
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
826
949
  res.end(JSON.stringify(state));
827
950
  return;
828
951
  }
829
952
  if (req.method === 'DELETE') {
953
+ assertTrustedMutationRequest(req);
830
954
  const targetType = parsed.searchParams.get('targetType');
831
955
  const targetId = parsed.searchParams.get('targetId') ?? '';
832
956
  const state = deleteObservationReviewState(observationsDir, targetType, targetId);
@@ -967,6 +1091,7 @@ export function createReportServer({ port, host: hostOption, reportsDir, analyse
967
1091
  if (reportApiMatch) {
968
1092
  const id = decodeURIComponent(reportApiMatch[1]);
969
1093
  if (req.method === 'DELETE') {
1094
+ assertTrustedMutationRequest(req);
970
1095
  const removed = await reportStore.remove(id);
971
1096
  if (!removed) {
972
1097
  res.writeHead(404, { 'Content-Type': 'application/json' });
@@ -1107,7 +1232,12 @@ export function createReportServer({ port, host: hostOption, reportsDir, analyse
1107
1232
  res.destroy(err instanceof Error ? err : new Error(getErrorMessage(err)));
1108
1233
  return;
1109
1234
  }
1110
- res.writeHead(500, { 'Content-Type': 'application/json' });
1235
+ const statusCode = err instanceof RequestBodyError
1236
+ ? err.statusCode
1237
+ : err instanceof ObservationReviewStateValidationError
1238
+ ? 400
1239
+ : 500;
1240
+ res.writeHead(statusCode, { 'Content-Type': 'application/json' });
1111
1241
  res.end(JSON.stringify({ error: getErrorMessage(err) }));
1112
1242
  }
1113
1243
  }
@@ -1187,21 +1317,3 @@ export function createReportServer({ port, host: hostOption, reportsDir, analyse
1187
1317
  }
1188
1318
  return { start, stop, getUrl };
1189
1319
  }
1190
- function parseSoftStandardReviewTarget(targetId) {
1191
- const separator = targetId.indexOf(':');
1192
- if (separator <= 0 || separator === targetId.length - 1)
1193
- return null;
1194
- return {
1195
- skillName: targetId.slice(0, separator),
1196
- standardId: targetId.slice(separator + 1),
1197
- };
1198
- }
1199
- function skillDerivedStandardStatusFromVerdict(verdict) {
1200
- if (verdict === 'real_issue')
1201
- return 'author_confirmed';
1202
- if (verdict === 'not_issue')
1203
- return 'rejected';
1204
- if (verdict === 'needs_more_context' || verdict === 'reviewed')
1205
- return 'pending_review';
1206
- return null;
1207
- }