oh-my-knowledge 0.31.0 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1096) hide show
  1. package/README.md +31 -842
  2. package/README.zh.md +43 -854
  3. package/dist/analysis/coverage-analyzer.d.ts.map +1 -0
  4. package/dist/analysis/coverage-analyzer.js.map +1 -0
  5. package/dist/analysis/failure-clusterer.d.ts.map +1 -0
  6. package/dist/analysis/failure-clusterer.js.map +1 -0
  7. package/dist/analysis/gap-analyzer.d.ts +121 -0
  8. package/dist/analysis/gap-analyzer.d.ts.map +1 -0
  9. package/dist/analysis/gap-analyzer.js +471 -0
  10. package/dist/analysis/gap-analyzer.js.map +1 -0
  11. package/dist/analysis/hedging-classifier.d.ts.map +1 -0
  12. package/dist/analysis/hedging-classifier.js.map +1 -0
  13. package/dist/analysis/report-diagnostics.d.ts +34 -0
  14. package/dist/analysis/report-diagnostics.d.ts.map +1 -0
  15. package/dist/analysis/report-diagnostics.js +753 -0
  16. package/dist/analysis/report-diagnostics.js.map +1 -0
  17. package/dist/analysis/sample-diagnostics.d.ts.map +1 -0
  18. package/dist/analysis/sample-diagnostics.js.map +1 -0
  19. package/dist/analysis/saturation.d.ts.map +1 -0
  20. package/dist/analysis/saturation.js.map +1 -0
  21. package/dist/authoring/evolver.d.ts.map +1 -0
  22. package/dist/authoring/evolver.js.map +1 -0
  23. package/dist/authoring/generator.d.ts.map +1 -0
  24. package/dist/authoring/generator.js +720 -0
  25. package/dist/authoring/generator.js.map +1 -0
  26. package/dist/authoring/sample-fixer.d.ts.map +1 -0
  27. package/dist/authoring/sample-fixer.js.map +1 -0
  28. package/dist/cli/commands/doctor.d.ts +26 -0
  29. package/dist/cli/commands/doctor.d.ts.map +1 -0
  30. package/dist/cli/commands/doctor.js +302 -0
  31. package/dist/cli/commands/doctor.js.map +1 -0
  32. package/dist/cli/commands/eval/gold/compare.d.ts.map +1 -0
  33. package/dist/cli/commands/eval/gold/compare.js.map +1 -0
  34. package/dist/cli/commands/eval/gold/index.d.ts.map +1 -0
  35. package/dist/cli/commands/eval/gold/index.js.map +1 -0
  36. package/dist/cli/commands/eval/gold/init.d.ts.map +1 -0
  37. package/dist/cli/commands/eval/gold/init.js.map +1 -0
  38. package/dist/cli/commands/eval/gold/validate.d.ts.map +1 -0
  39. package/dist/cli/commands/eval/gold/validate.js.map +1 -0
  40. package/dist/cli/commands/eval/index.d.ts.map +1 -0
  41. package/dist/cli/commands/eval/index.js +465 -0
  42. package/dist/cli/commands/eval/index.js.map +1 -0
  43. package/dist/cli/commands/evolve.d.ts.map +1 -0
  44. package/dist/cli/commands/evolve.js +283 -0
  45. package/dist/cli/commands/evolve.js.map +1 -0
  46. package/dist/cli/commands/init.d.ts.map +1 -0
  47. package/dist/cli/commands/init.js.map +1 -0
  48. package/dist/cli/commands/observe/inbox.d.ts.map +1 -0
  49. package/dist/cli/commands/observe/inbox.js +260 -0
  50. package/dist/cli/commands/observe/inbox.js.map +1 -0
  51. package/dist/cli/commands/observe/index.d.ts.map +1 -0
  52. package/dist/cli/commands/observe/index.js.map +1 -0
  53. package/dist/cli/commands/observe/ingest.d.ts.map +1 -0
  54. package/dist/cli/commands/observe/ingest.js +71 -0
  55. package/dist/cli/commands/observe/ingest.js.map +1 -0
  56. package/dist/cli/commands/observe/show.d.ts.map +1 -0
  57. package/dist/cli/commands/observe/show.js.map +1 -0
  58. package/dist/cli/commands/sample.d.ts.map +1 -0
  59. package/dist/cli/commands/sample.js +487 -0
  60. package/dist/cli/commands/sample.js.map +1 -0
  61. package/dist/cli/commands/studio.d.ts.map +1 -0
  62. package/dist/cli/commands/studio.js +158 -0
  63. package/dist/cli/commands/studio.js.map +1 -0
  64. package/dist/cli/index.d.ts.map +1 -0
  65. package/dist/cli/index.js.map +1 -0
  66. package/dist/cli/lib/cli-exit.d.ts.map +1 -0
  67. package/dist/cli/lib/cli-exit.js.map +1 -0
  68. package/dist/cli/lib/cmd-flags.d.ts +224 -0
  69. package/dist/cli/lib/cmd-flags.d.ts.map +1 -0
  70. package/dist/cli/lib/cmd-flags.js.map +1 -0
  71. package/dist/cli/lib/i18n-dict/common.d.ts +4 -0
  72. package/dist/cli/lib/i18n-dict/common.d.ts.map +1 -0
  73. package/dist/cli/lib/i18n-dict/common.js +67 -0
  74. package/dist/cli/lib/i18n-dict/common.js.map +1 -0
  75. package/dist/cli/lib/i18n-dict/evolve.d.ts +4 -0
  76. package/dist/cli/lib/i18n-dict/evolve.d.ts.map +1 -0
  77. package/dist/cli/lib/i18n-dict/evolve.js +43 -0
  78. package/dist/cli/lib/i18n-dict/evolve.js.map +1 -0
  79. package/dist/cli/lib/i18n-dict/gen.d.ts +4 -0
  80. package/dist/cli/lib/i18n-dict/gen.d.ts.map +1 -0
  81. package/dist/cli/lib/i18n-dict/gen.js +63 -0
  82. package/dist/cli/lib/i18n-dict/gen.js.map +1 -0
  83. package/dist/cli/lib/i18n-dict/help.d.ts +4 -0
  84. package/dist/cli/lib/i18n-dict/help.d.ts.map +1 -0
  85. package/dist/cli/lib/i18n-dict/help.js +281 -0
  86. package/dist/cli/lib/i18n-dict/help.js.map +1 -0
  87. package/dist/cli/lib/i18n-dict/init.d.ts +4 -0
  88. package/dist/cli/lib/i18n-dict/init.d.ts.map +1 -0
  89. package/dist/cli/lib/i18n-dict/init.js +27 -0
  90. package/dist/cli/lib/i18n-dict/init.js.map +1 -0
  91. package/dist/cli/lib/i18n-dict/run.d.ts +4 -0
  92. package/dist/cli/lib/i18n-dict/run.d.ts.map +1 -0
  93. package/dist/cli/lib/i18n-dict/run.js +143 -0
  94. package/dist/cli/lib/i18n-dict/run.js.map +1 -0
  95. package/dist/cli/lib/i18n-dict/types.d.ts +5 -0
  96. package/dist/cli/lib/i18n-dict/types.d.ts.map +1 -0
  97. package/dist/cli/lib/i18n-dict/types.js +2 -0
  98. package/dist/cli/lib/i18n-dict/types.js.map +1 -0
  99. package/dist/cli/lib/i18n-dict.d.ts +63 -0
  100. package/dist/cli/lib/i18n-dict.d.ts.map +1 -0
  101. package/dist/cli/lib/i18n-dict.js +67 -0
  102. package/dist/cli/lib/i18n-dict.js.map +1 -0
  103. package/dist/cli/lib/i18n.d.ts.map +1 -0
  104. package/dist/cli/lib/i18n.js.map +1 -0
  105. package/dist/cli/lib/parse-run-config/judge-models.d.ts +24 -0
  106. package/dist/cli/lib/parse-run-config/judge-models.d.ts.map +1 -0
  107. package/dist/cli/lib/parse-run-config/judge-models.js +55 -0
  108. package/dist/cli/lib/parse-run-config/judge-models.js.map +1 -0
  109. package/dist/cli/lib/parse-run-config/samples-discovery.d.ts +19 -0
  110. package/dist/cli/lib/parse-run-config/samples-discovery.d.ts.map +1 -0
  111. package/dist/cli/lib/parse-run-config/samples-discovery.js +53 -0
  112. package/dist/cli/lib/parse-run-config/samples-discovery.js.map +1 -0
  113. package/dist/cli/lib/parse-run-config/variant-resolution.d.ts +18 -0
  114. package/dist/cli/lib/parse-run-config/variant-resolution.d.ts.map +1 -0
  115. package/dist/cli/lib/parse-run-config/variant-resolution.js +59 -0
  116. package/dist/cli/lib/parse-run-config/variant-resolution.js.map +1 -0
  117. package/dist/cli/lib/parse-run-config.d.ts +93 -0
  118. package/dist/cli/lib/parse-run-config.d.ts.map +1 -0
  119. package/dist/cli/lib/parse-run-config.js +157 -0
  120. package/dist/cli/lib/parse-run-config.js.map +1 -0
  121. package/dist/cli/lib/progress.d.ts.map +1 -0
  122. package/dist/cli/lib/progress.js.map +1 -0
  123. package/dist/cli/lib/resolve-skill-input.d.ts +8 -0
  124. package/dist/cli/lib/resolve-skill-input.d.ts.map +1 -0
  125. package/dist/cli/lib/resolve-skill-input.js +38 -0
  126. package/dist/cli/lib/resolve-skill-input.js.map +1 -0
  127. package/dist/cli/lib/run-tally.d.ts.map +1 -0
  128. package/dist/cli/lib/run-tally.js.map +1 -0
  129. package/dist/cli/lib/shared.d.ts.map +1 -0
  130. package/dist/cli/lib/shared.js.map +1 -0
  131. package/dist/cli/lib/update-check.d.ts +9 -0
  132. package/dist/cli/lib/update-check.d.ts.map +1 -0
  133. package/dist/cli/lib/update-check.js +113 -0
  134. package/dist/cli/lib/update-check.js.map +1 -0
  135. package/dist/cli/oclif/base-command.d.ts.map +1 -0
  136. package/dist/cli/oclif/base-command.js.map +1 -0
  137. package/dist/cli/oclif/help.d.ts.map +1 -0
  138. package/dist/cli/oclif/help.js.map +1 -0
  139. package/dist/cli/oclif/i18n.d.ts.map +1 -0
  140. package/dist/cli/oclif/i18n.js.map +1 -0
  141. package/dist/cli/oclif/parsers.d.ts.map +1 -0
  142. package/dist/cli/oclif/parsers.js.map +1 -0
  143. package/dist/cli/oclif/projection.d.ts.map +1 -0
  144. package/dist/cli/oclif/projection.js.map +1 -0
  145. package/dist/cli/oclif/run.d.ts.map +1 -0
  146. package/dist/cli/oclif/run.js +11 -0
  147. package/dist/cli/oclif/run.js.map +1 -0
  148. package/dist/diagnosis/observe-mapper.d.ts.map +1 -0
  149. package/dist/diagnosis/observe-mapper.js.map +1 -0
  150. package/dist/diagnosis/observe-producer.d.ts.map +1 -0
  151. package/dist/diagnosis/observe-producer.js +199 -0
  152. package/dist/diagnosis/observe-producer.js.map +1 -0
  153. package/dist/diagnosis/studio-projection.d.ts +7 -0
  154. package/dist/diagnosis/studio-projection.d.ts.map +1 -0
  155. package/dist/diagnosis/studio-projection.js.map +1 -0
  156. package/dist/diagnosis/types.d.ts +4 -0
  157. package/dist/diagnosis/types.d.ts.map +1 -0
  158. package/dist/diagnosis/types.js +20 -0
  159. package/dist/diagnosis/types.js.map +1 -0
  160. package/dist/doctor/fixer.d.ts.map +1 -0
  161. package/dist/doctor/fixer.js.map +1 -0
  162. package/dist/doctor/health/builtin-dimensions.d.ts.map +1 -0
  163. package/dist/doctor/health/builtin-dimensions.js.map +1 -0
  164. package/dist/doctor/health/composer.d.ts.map +1 -0
  165. package/dist/doctor/health/composer.js +296 -0
  166. package/dist/doctor/health/composer.js.map +1 -0
  167. package/dist/doctor/health/dimension-registry.d.ts.map +1 -0
  168. package/dist/doctor/health/dimension-registry.js.map +1 -0
  169. package/dist/doctor/health/dimension-spec.d.ts +47 -0
  170. package/dist/doctor/health/dimension-spec.d.ts.map +1 -0
  171. package/dist/doctor/health/dimension-spec.js.map +1 -0
  172. package/dist/doctor/health/parser.d.ts.map +1 -0
  173. package/dist/doctor/health/parser.js.map +1 -0
  174. package/dist/doctor/health/prompt-builder.d.ts.map +1 -0
  175. package/dist/doctor/health/prompt-builder.js.map +1 -0
  176. package/dist/doctor/health/register.d.ts.map +1 -0
  177. package/dist/doctor/health/register.js.map +1 -0
  178. package/dist/doctor/index.d.ts.map +1 -0
  179. package/dist/doctor/index.js.map +1 -0
  180. package/dist/doctor/messages.d.ts +21 -0
  181. package/dist/doctor/messages.d.ts.map +1 -0
  182. package/dist/doctor/messages.js +217 -0
  183. package/dist/doctor/messages.js.map +1 -0
  184. package/dist/doctor/preflight.d.ts.map +1 -0
  185. package/dist/doctor/preflight.js.map +1 -0
  186. package/dist/doctor/renderer.d.ts +17 -0
  187. package/dist/doctor/renderer.d.ts.map +1 -0
  188. package/dist/doctor/renderer.js +122 -0
  189. package/dist/doctor/renderer.js.map +1 -0
  190. package/dist/doctor/rules.d.ts.map +1 -0
  191. package/dist/doctor/rules.js +289 -0
  192. package/dist/doctor/rules.js.map +1 -0
  193. package/dist/eval-core/bootstrap.d.ts.map +1 -0
  194. package/dist/eval-core/bootstrap.js.map +1 -0
  195. package/dist/eval-core/cache.d.ts.map +1 -0
  196. package/dist/eval-core/cache.js.map +1 -0
  197. package/dist/eval-core/comparability.d.ts.map +1 -0
  198. package/dist/eval-core/comparability.js.map +1 -0
  199. package/dist/eval-core/dependency-checker.d.ts +37 -0
  200. package/dist/eval-core/dependency-checker.d.ts.map +1 -0
  201. package/dist/eval-core/dependency-checker.js.map +1 -0
  202. package/dist/eval-core/evaluation-execution.d.ts.map +1 -0
  203. package/dist/eval-core/evaluation-execution.js.map +1 -0
  204. package/dist/eval-core/evaluation-job.d.ts.map +1 -0
  205. package/dist/eval-core/evaluation-job.js.map +1 -0
  206. package/dist/eval-core/evaluation-reporting.d.ts.map +1 -0
  207. package/dist/eval-core/evaluation-reporting.js.map +1 -0
  208. package/dist/eval-core/execution-strategy.d.ts.map +1 -0
  209. package/dist/eval-core/execution-strategy.js +165 -0
  210. package/dist/eval-core/execution-strategy.js.map +1 -0
  211. package/dist/eval-core/fact-checker.d.ts.map +1 -0
  212. package/dist/eval-core/fact-checker.js.map +1 -0
  213. package/dist/eval-core/layer-gates.d.ts.map +1 -0
  214. package/dist/eval-core/layer-gates.js.map +1 -0
  215. package/dist/eval-core/mocks-runtime.d.ts.map +1 -0
  216. package/dist/eval-core/mocks-runtime.js +559 -0
  217. package/dist/eval-core/mocks-runtime.js.map +1 -0
  218. package/dist/eval-core/schema.d.ts.map +1 -0
  219. package/dist/eval-core/schema.js.map +1 -0
  220. package/dist/eval-core/statistics.d.ts.map +1 -0
  221. package/dist/eval-core/statistics.js.map +1 -0
  222. package/dist/eval-core/task-planner.d.ts.map +1 -0
  223. package/dist/eval-core/task-planner.js.map +1 -0
  224. package/dist/eval-core/verdict.d.ts.map +1 -0
  225. package/dist/eval-core/verdict.js.map +1 -0
  226. package/dist/eval-workflows/batch-evaluation-workflow.d.ts.map +1 -0
  227. package/dist/eval-workflows/batch-evaluation-workflow.js.map +1 -0
  228. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +22 -0
  229. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts.map +1 -0
  230. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +67 -0
  231. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js.map +1 -0
  232. package/dist/eval-workflows/evaluation-pipeline/report-finalize.d.ts +32 -0
  233. package/dist/eval-workflows/evaluation-pipeline/report-finalize.d.ts.map +1 -0
  234. package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +57 -0
  235. package/dist/eval-workflows/evaluation-pipeline/report-finalize.js.map +1 -0
  236. package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +60 -0
  237. package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts.map +1 -0
  238. package/dist/eval-workflows/evaluation-pipeline/run-state.js +88 -0
  239. package/dist/eval-workflows/evaluation-pipeline/run-state.js.map +1 -0
  240. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +21 -0
  241. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts.map +1 -0
  242. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +46 -0
  243. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js.map +1 -0
  244. package/dist/eval-workflows/evaluation-pipeline.d.ts +97 -0
  245. package/dist/eval-workflows/evaluation-pipeline.d.ts.map +1 -0
  246. package/dist/eval-workflows/evaluation-pipeline.js +179 -0
  247. package/dist/eval-workflows/evaluation-pipeline.js.map +1 -0
  248. package/dist/eval-workflows/evaluation-preparation.d.ts.map +1 -0
  249. package/dist/eval-workflows/evaluation-preparation.js.map +1 -0
  250. package/dist/eval-workflows/messages.d.ts +4 -0
  251. package/dist/eval-workflows/messages.d.ts.map +1 -0
  252. package/dist/eval-workflows/messages.js +30 -0
  253. package/dist/eval-workflows/messages.js.map +1 -0
  254. package/dist/eval-workflows/run-evaluation.d.ts.map +1 -0
  255. package/dist/eval-workflows/run-evaluation.js +528 -0
  256. package/dist/eval-workflows/run-evaluation.js.map +1 -0
  257. package/dist/executors/anthropic-api.d.ts.map +1 -0
  258. package/dist/executors/anthropic-api.js.map +1 -0
  259. package/dist/executors/claude-cli.d.ts.map +1 -0
  260. package/dist/executors/claude-cli.js.map +1 -0
  261. package/dist/executors/claude-sdk-trace.d.ts.map +1 -0
  262. package/dist/executors/claude-sdk-trace.js.map +1 -0
  263. package/dist/executors/claude-sdk.d.ts.map +1 -0
  264. package/dist/executors/claude-sdk.js.map +1 -0
  265. package/dist/executors/codex-cli-trace.d.ts.map +1 -0
  266. package/dist/executors/codex-cli-trace.js.map +1 -0
  267. package/dist/executors/codex-cli.d.ts.map +1 -0
  268. package/dist/executors/codex-cli.js.map +1 -0
  269. package/dist/executors/codex-sdk.d.ts.map +1 -0
  270. package/dist/executors/codex-sdk.js.map +1 -0
  271. package/dist/executors/gemini.d.ts.map +1 -0
  272. package/dist/executors/gemini.js.map +1 -0
  273. package/dist/executors/index.d.ts.map +1 -0
  274. package/dist/executors/index.js.map +1 -0
  275. package/dist/executors/openai-api.d.ts.map +1 -0
  276. package/dist/executors/openai-api.js.map +1 -0
  277. package/dist/executors/runtime-fingerprint.d.ts.map +1 -0
  278. package/dist/executors/runtime-fingerprint.js.map +1 -0
  279. package/dist/executors/script.d.ts.map +1 -0
  280. package/dist/executors/script.js.map +1 -0
  281. package/dist/executors/shared.d.ts +194 -0
  282. package/dist/executors/shared.d.ts.map +1 -0
  283. package/dist/executors/shared.js +253 -0
  284. package/dist/executors/shared.js.map +1 -0
  285. package/dist/grading/assertions.d.ts.map +1 -0
  286. package/dist/grading/assertions.js.map +1 -0
  287. package/dist/grading/debias-validate.d.ts.map +1 -0
  288. package/dist/grading/debias-validate.js.map +1 -0
  289. package/dist/grading/diagnostic.d.ts.map +1 -0
  290. package/dist/grading/diagnostic.js.map +1 -0
  291. package/dist/grading/gold-cli.d.ts.map +1 -0
  292. package/dist/grading/gold-cli.js.map +1 -0
  293. package/dist/grading/gold-dataset.d.ts.map +1 -0
  294. package/dist/grading/gold-dataset.js.map +1 -0
  295. package/dist/grading/human-gold.d.ts.map +1 -0
  296. package/dist/grading/human-gold.js.map +1 -0
  297. package/dist/grading/index.d.ts.map +1 -0
  298. package/dist/grading/index.js.map +1 -0
  299. package/dist/grading/judge.d.ts.map +1 -0
  300. package/dist/grading/judge.js.map +1 -0
  301. package/dist/grading/layered-scores.d.ts.map +1 -0
  302. package/dist/grading/layered-scores.js.map +1 -0
  303. package/dist/inputs/eval-config.d.ts.map +1 -0
  304. package/dist/inputs/eval-config.js.map +1 -0
  305. package/dist/inputs/load-samples.d.ts.map +1 -0
  306. package/dist/inputs/load-samples.js.map +1 -0
  307. package/dist/inputs/mcp-resolver.d.ts.map +1 -0
  308. package/dist/inputs/mcp-resolver.js.map +1 -0
  309. package/dist/inputs/skill-loader.d.ts.map +1 -0
  310. package/dist/inputs/skill-loader.js +261 -0
  311. package/dist/inputs/skill-loader.js.map +1 -0
  312. package/dist/inputs/url-fetcher.d.ts.map +1 -0
  313. package/dist/inputs/url-fetcher.js.map +1 -0
  314. package/dist/observability/experience-frontmatter.d.ts +37 -0
  315. package/dist/observability/experience-frontmatter.d.ts.map +1 -0
  316. package/dist/observability/experience-frontmatter.js +100 -0
  317. package/dist/observability/experience-frontmatter.js.map +1 -0
  318. package/dist/observability/experience.d.ts +23 -0
  319. package/dist/observability/experience.d.ts.map +1 -0
  320. package/dist/observability/experience.js +3438 -0
  321. package/dist/observability/experience.js.map +1 -0
  322. package/dist/observability/feedback-matchers.d.ts +37 -0
  323. package/dist/observability/feedback-matchers.d.ts.map +1 -0
  324. package/dist/observability/feedback-matchers.js +232 -0
  325. package/dist/observability/feedback-matchers.js.map +1 -0
  326. package/dist/observability/feedback-projection.d.ts +21 -0
  327. package/dist/observability/feedback-projection.d.ts.map +1 -0
  328. package/dist/observability/feedback-projection.js +20 -0
  329. package/dist/observability/feedback-projection.js.map +1 -0
  330. package/dist/observability/inbox-view-model.d.ts +43 -0
  331. package/dist/observability/inbox-view-model.d.ts.map +1 -0
  332. package/dist/observability/inbox-view-model.js +180 -0
  333. package/dist/observability/inbox-view-model.js.map +1 -0
  334. package/dist/observability/inbox.d.ts +28 -0
  335. package/dist/observability/inbox.d.ts.map +1 -0
  336. package/dist/observability/inbox.js +765 -0
  337. package/dist/observability/inbox.js.map +1 -0
  338. package/dist/observability/problem-patterns.d.ts +12 -0
  339. package/dist/observability/problem-patterns.d.ts.map +1 -0
  340. package/dist/observability/problem-patterns.js +244 -0
  341. package/dist/observability/problem-patterns.js.map +1 -0
  342. package/dist/observability/prompts/llm-enhanced-review.prompt.md +224 -0
  343. package/dist/observability/resolved-review.d.ts +37 -0
  344. package/dist/observability/resolved-review.d.ts.map +1 -0
  345. package/dist/observability/resolved-review.js +552 -0
  346. package/dist/observability/resolved-review.js.map +1 -0
  347. package/dist/observability/review-state.d.ts +20 -0
  348. package/dist/observability/review-state.d.ts.map +1 -0
  349. package/dist/observability/review-state.js.map +1 -0
  350. package/dist/observability/skill-chain-advisories.d.ts +16 -0
  351. package/dist/observability/skill-chain-advisories.d.ts.map +1 -0
  352. package/dist/observability/skill-chain-advisories.js.map +1 -0
  353. package/dist/observability/skill-chain.d.ts +7 -0
  354. package/dist/observability/skill-chain.d.ts.map +1 -0
  355. package/dist/observability/skill-chain.js +474 -0
  356. package/dist/observability/skill-chain.js.map +1 -0
  357. package/dist/observability/skill-health-analyzer.d.ts.map +1 -0
  358. package/dist/observability/skill-health-analyzer.js.map +1 -0
  359. package/dist/observability/soft-standards/constants.d.ts +5 -0
  360. package/dist/observability/soft-standards/constants.d.ts.map +1 -0
  361. package/dist/observability/soft-standards/constants.js +10 -0
  362. package/dist/observability/soft-standards/constants.js.map +1 -0
  363. package/dist/observability/soft-standards/index.d.ts +16 -0
  364. package/dist/observability/soft-standards/index.d.ts.map +1 -0
  365. package/dist/observability/soft-standards/index.js +15 -0
  366. package/dist/observability/soft-standards/index.js.map +1 -0
  367. package/dist/observability/soft-standards/llm-extractor.d.ts +6 -0
  368. package/dist/observability/soft-standards/llm-extractor.d.ts.map +1 -0
  369. package/dist/observability/soft-standards/llm-extractor.js +692 -0
  370. package/dist/observability/soft-standards/llm-extractor.js.map +1 -0
  371. package/dist/observability/soft-standards/runtime-evaluator.d.ts +12 -0
  372. package/dist/observability/soft-standards/runtime-evaluator.d.ts.map +1 -0
  373. package/dist/observability/soft-standards/runtime-evaluator.js +385 -0
  374. package/dist/observability/soft-standards/runtime-evaluator.js.map +1 -0
  375. package/dist/observability/soft-standards/skill-standards-store.d.ts +11 -0
  376. package/dist/observability/soft-standards/skill-standards-store.d.ts.map +1 -0
  377. package/dist/observability/soft-standards/skill-standards-store.js +158 -0
  378. package/dist/observability/soft-standards/skill-standards-store.js.map +1 -0
  379. package/dist/observability/soft-standards/types.d.ts +212 -0
  380. package/dist/observability/soft-standards/types.d.ts.map +1 -0
  381. package/dist/observability/soft-standards/types.js +2 -0
  382. package/dist/observability/soft-standards/types.js.map +1 -0
  383. package/dist/observability/text-signals.d.ts +14 -0
  384. package/dist/observability/text-signals.d.ts.map +1 -0
  385. package/dist/observability/text-signals.js +116 -0
  386. package/dist/observability/text-signals.js.map +1 -0
  387. package/dist/observability/trace-adapter.d.ts.map +1 -0
  388. package/dist/observability/trace-adapter.js.map +1 -0
  389. package/dist/observability/trace-attribution.d.ts +56 -0
  390. package/dist/observability/trace-attribution.d.ts.map +1 -0
  391. package/dist/observability/trace-attribution.js +200 -0
  392. package/dist/observability/trace-attribution.js.map +1 -0
  393. package/dist/observability/trace-segmenter.d.ts +52 -0
  394. package/dist/observability/trace-segmenter.d.ts.map +1 -0
  395. package/dist/observability/trace-segmenter.js +322 -0
  396. package/dist/observability/trace-segmenter.js.map +1 -0
  397. package/dist/observability/trace-source.d.ts +92 -0
  398. package/dist/observability/trace-source.d.ts.map +1 -0
  399. package/dist/observability/trace-source.js +502 -0
  400. package/dist/observability/trace-source.js.map +1 -0
  401. package/dist/renderer/html-renderer.d.ts.map +1 -0
  402. package/dist/renderer/html-renderer.js.map +1 -0
  403. package/dist/renderer/layout.d.ts.map +1 -0
  404. package/dist/renderer/layout.js.map +1 -0
  405. package/dist/renderer/observation-inbox/helpers.d.ts +14 -0
  406. package/dist/renderer/observation-inbox/helpers.d.ts.map +1 -0
  407. package/dist/renderer/observation-inbox/helpers.js +75 -0
  408. package/dist/renderer/observation-inbox/helpers.js.map +1 -0
  409. package/dist/renderer/observation-inbox/styles.d.ts +2 -0
  410. package/dist/renderer/observation-inbox/styles.d.ts.map +1 -0
  411. package/dist/renderer/observation-inbox/styles.js +5184 -0
  412. package/dist/renderer/observation-inbox/styles.js.map +1 -0
  413. package/dist/renderer/observation-inbox-renderer.d.ts +5 -0
  414. package/dist/renderer/observation-inbox-renderer.d.ts.map +1 -0
  415. package/dist/renderer/observation-inbox-renderer.js +7182 -0
  416. package/dist/renderer/observation-inbox-renderer.js.map +1 -0
  417. package/dist/renderer/skill-detail-renderer.d.ts +13 -0
  418. package/dist/renderer/skill-detail-renderer.d.ts.map +1 -0
  419. package/dist/renderer/skill-detail-renderer.js.map +1 -0
  420. package/dist/renderer/skill-health-renderer.d.ts.map +1 -0
  421. package/dist/renderer/skill-health-renderer.js.map +1 -0
  422. package/dist/renderer/skill-list-renderer.d.ts +3 -0
  423. package/dist/renderer/skill-list-renderer.d.ts.map +1 -0
  424. package/dist/renderer/skill-list-renderer.js.map +1 -0
  425. package/dist/renderer/summary.d.ts +68 -0
  426. package/dist/renderer/summary.d.ts.map +1 -0
  427. package/dist/renderer/summary.js +1896 -0
  428. package/dist/renderer/summary.js.map +1 -0
  429. package/dist/renderer/table.d.ts.map +1 -0
  430. package/dist/renderer/table.js.map +1 -0
  431. package/dist/renderer/test-view.d.ts.map +1 -0
  432. package/dist/renderer/test-view.js.map +1 -0
  433. package/dist/renderer/trends.d.ts.map +1 -0
  434. package/dist/renderer/trends.js.map +1 -0
  435. package/dist/server/job-store.d.ts.map +1 -0
  436. package/dist/server/job-store.js.map +1 -0
  437. package/dist/server/report-server.d.ts.map +1 -0
  438. package/dist/server/report-server.js +892 -0
  439. package/dist/server/report-server.js.map +1 -0
  440. package/dist/server/report-store.d.ts.map +1 -0
  441. package/dist/server/report-store.js.map +1 -0
  442. package/dist/server/skill-index.d.ts +14 -0
  443. package/dist/server/skill-index.d.ts.map +1 -0
  444. package/dist/server/skill-index.js +381 -0
  445. package/dist/server/skill-index.js.map +1 -0
  446. package/dist/server/skill-insights.d.ts +32 -0
  447. package/dist/server/skill-insights.d.ts.map +1 -0
  448. package/dist/server/skill-insights.js.map +1 -0
  449. package/dist/shared/hard-rules.d.ts.map +1 -0
  450. package/dist/shared/hard-rules.js.map +1 -0
  451. package/dist/shared/llm-prompts/index.d.ts +14 -0
  452. package/dist/shared/llm-prompts/index.d.ts.map +1 -0
  453. package/dist/shared/llm-prompts/index.js +22 -0
  454. package/dist/shared/llm-prompts/index.js.map +1 -0
  455. package/dist/shared/llm-prompts/skill-health.d.ts.map +1 -0
  456. package/dist/shared/llm-prompts/skill-health.js.map +1 -0
  457. package/dist/shared/time.d.ts.map +1 -0
  458. package/dist/shared/time.js.map +1 -0
  459. package/dist/shared/tool-search.d.ts.map +1 -0
  460. package/dist/shared/tool-search.js.map +1 -0
  461. package/dist/types/dependencies.d.ts +23 -0
  462. package/dist/types/dependencies.d.ts.map +1 -0
  463. package/dist/types/dependencies.js +2 -0
  464. package/dist/types/dependencies.js.map +1 -0
  465. package/dist/types/diagnosis.d.ts +88 -0
  466. package/dist/types/diagnosis.d.ts.map +1 -0
  467. package/dist/types/diagnosis.js +2 -0
  468. package/dist/types/diagnosis.js.map +1 -0
  469. package/dist/types/doctor.d.ts +157 -0
  470. package/dist/types/doctor.d.ts.map +1 -0
  471. package/dist/types/doctor.js.map +1 -0
  472. package/dist/types/eval.d.ts +377 -0
  473. package/dist/types/eval.d.ts.map +1 -0
  474. package/dist/types/eval.js.map +1 -0
  475. package/dist/types/executor.d.ts.map +1 -0
  476. package/dist/types/executor.js.map +1 -0
  477. package/dist/types/index.d.ts +12 -0
  478. package/dist/types/index.d.ts.map +1 -0
  479. package/dist/types/index.js +12 -0
  480. package/dist/types/index.js.map +1 -0
  481. package/dist/types/judge.d.ts.map +1 -0
  482. package/dist/types/judge.js.map +1 -0
  483. package/dist/types/observability.d.ts +865 -0
  484. package/dist/types/observability.d.ts.map +1 -0
  485. package/dist/types/observability.js +14 -0
  486. package/dist/types/observability.js.map +1 -0
  487. package/dist/types/report.d.ts +560 -0
  488. package/dist/types/report.d.ts.map +1 -0
  489. package/dist/types/report.js.map +1 -0
  490. package/dist/types/shared.d.ts.map +1 -0
  491. package/dist/types/shared.js.map +1 -0
  492. package/dist/types/skill-index.d.ts +142 -0
  493. package/dist/types/skill-index.d.ts.map +1 -0
  494. package/dist/types/skill-index.js +2 -0
  495. package/dist/types/skill-index.js.map +1 -0
  496. package/dist/types/storage.d.ts.map +1 -0
  497. package/dist/types/storage.js.map +1 -0
  498. package/dist/util/safe-slice.d.ts +23 -0
  499. package/dist/util/safe-slice.d.ts.map +1 -0
  500. package/dist/util/safe-slice.js +33 -0
  501. package/dist/util/safe-slice.js.map +1 -0
  502. package/package.json +20 -13
  503. package/dist/src/analysis/coverage-analyzer.d.ts.map +0 -1
  504. package/dist/src/analysis/coverage-analyzer.js.map +0 -1
  505. package/dist/src/analysis/failure-clusterer.d.ts.map +0 -1
  506. package/dist/src/analysis/failure-clusterer.js.map +0 -1
  507. package/dist/src/analysis/gap-analyzer.d.ts +0 -121
  508. package/dist/src/analysis/gap-analyzer.d.ts.map +0 -1
  509. package/dist/src/analysis/gap-analyzer.js +0 -471
  510. package/dist/src/analysis/gap-analyzer.js.map +0 -1
  511. package/dist/src/analysis/hedging-classifier.d.ts.map +0 -1
  512. package/dist/src/analysis/hedging-classifier.js.map +0 -1
  513. package/dist/src/analysis/report-diagnostics.d.ts +0 -34
  514. package/dist/src/analysis/report-diagnostics.d.ts.map +0 -1
  515. package/dist/src/analysis/report-diagnostics.js +0 -753
  516. package/dist/src/analysis/report-diagnostics.js.map +0 -1
  517. package/dist/src/analysis/sample-diagnostics.d.ts.map +0 -1
  518. package/dist/src/analysis/sample-diagnostics.js.map +0 -1
  519. package/dist/src/analysis/saturation.d.ts.map +0 -1
  520. package/dist/src/analysis/saturation.js.map +0 -1
  521. package/dist/src/authoring/evolver.d.ts.map +0 -1
  522. package/dist/src/authoring/evolver.js.map +0 -1
  523. package/dist/src/authoring/generator.d.ts.map +0 -1
  524. package/dist/src/authoring/generator.js +0 -720
  525. package/dist/src/authoring/generator.js.map +0 -1
  526. package/dist/src/authoring/sample-fixer.d.ts.map +0 -1
  527. package/dist/src/authoring/sample-fixer.js.map +0 -1
  528. package/dist/src/cli/commands/doctor.d.ts +0 -26
  529. package/dist/src/cli/commands/doctor.d.ts.map +0 -1
  530. package/dist/src/cli/commands/doctor.js +0 -283
  531. package/dist/src/cli/commands/doctor.js.map +0 -1
  532. package/dist/src/cli/commands/eval/gold/compare.d.ts.map +0 -1
  533. package/dist/src/cli/commands/eval/gold/compare.js.map +0 -1
  534. package/dist/src/cli/commands/eval/gold/index.d.ts.map +0 -1
  535. package/dist/src/cli/commands/eval/gold/index.js.map +0 -1
  536. package/dist/src/cli/commands/eval/gold/init.d.ts.map +0 -1
  537. package/dist/src/cli/commands/eval/gold/init.js.map +0 -1
  538. package/dist/src/cli/commands/eval/gold/validate.d.ts.map +0 -1
  539. package/dist/src/cli/commands/eval/gold/validate.js.map +0 -1
  540. package/dist/src/cli/commands/eval/index.d.ts.map +0 -1
  541. package/dist/src/cli/commands/eval/index.js +0 -465
  542. package/dist/src/cli/commands/eval/index.js.map +0 -1
  543. package/dist/src/cli/commands/evolve.d.ts.map +0 -1
  544. package/dist/src/cli/commands/evolve.js +0 -276
  545. package/dist/src/cli/commands/evolve.js.map +0 -1
  546. package/dist/src/cli/commands/init.d.ts.map +0 -1
  547. package/dist/src/cli/commands/init.js.map +0 -1
  548. package/dist/src/cli/commands/observe/inbox.d.ts.map +0 -1
  549. package/dist/src/cli/commands/observe/inbox.js +0 -258
  550. package/dist/src/cli/commands/observe/inbox.js.map +0 -1
  551. package/dist/src/cli/commands/observe/index.d.ts.map +0 -1
  552. package/dist/src/cli/commands/observe/index.js.map +0 -1
  553. package/dist/src/cli/commands/observe/ingest.d.ts.map +0 -1
  554. package/dist/src/cli/commands/observe/ingest.js +0 -69
  555. package/dist/src/cli/commands/observe/ingest.js.map +0 -1
  556. package/dist/src/cli/commands/observe/show.d.ts.map +0 -1
  557. package/dist/src/cli/commands/observe/show.js.map +0 -1
  558. package/dist/src/cli/commands/sample.d.ts.map +0 -1
  559. package/dist/src/cli/commands/sample.js +0 -488
  560. package/dist/src/cli/commands/sample.js.map +0 -1
  561. package/dist/src/cli/commands/skill-extract.d.ts +0 -20
  562. package/dist/src/cli/commands/skill-extract.d.ts.map +0 -1
  563. package/dist/src/cli/commands/skill-extract.js +0 -84
  564. package/dist/src/cli/commands/skill-extract.js.map +0 -1
  565. package/dist/src/cli/commands/studio.d.ts.map +0 -1
  566. package/dist/src/cli/commands/studio.js +0 -158
  567. package/dist/src/cli/commands/studio.js.map +0 -1
  568. package/dist/src/cli/index.d.ts.map +0 -1
  569. package/dist/src/cli/index.js.map +0 -1
  570. package/dist/src/cli/lib/cli-exit.d.ts.map +0 -1
  571. package/dist/src/cli/lib/cli-exit.js.map +0 -1
  572. package/dist/src/cli/lib/cmd-flags.d.ts +0 -236
  573. package/dist/src/cli/lib/cmd-flags.d.ts.map +0 -1
  574. package/dist/src/cli/lib/cmd-flags.js.map +0 -1
  575. package/dist/src/cli/lib/i18n-dict.d.ts +0 -54
  576. package/dist/src/cli/lib/i18n-dict.d.ts.map +0 -1
  577. package/dist/src/cli/lib/i18n-dict.js +0 -900
  578. package/dist/src/cli/lib/i18n-dict.js.map +0 -1
  579. package/dist/src/cli/lib/i18n.d.ts.map +0 -1
  580. package/dist/src/cli/lib/i18n.js.map +0 -1
  581. package/dist/src/cli/lib/parse-run-config.d.ts +0 -89
  582. package/dist/src/cli/lib/parse-run-config.d.ts.map +0 -1
  583. package/dist/src/cli/lib/parse-run-config.js +0 -252
  584. package/dist/src/cli/lib/parse-run-config.js.map +0 -1
  585. package/dist/src/cli/lib/progress.d.ts.map +0 -1
  586. package/dist/src/cli/lib/progress.js.map +0 -1
  587. package/dist/src/cli/lib/run-tally.d.ts.map +0 -1
  588. package/dist/src/cli/lib/run-tally.js.map +0 -1
  589. package/dist/src/cli/lib/shared.d.ts.map +0 -1
  590. package/dist/src/cli/lib/shared.js.map +0 -1
  591. package/dist/src/cli/lib/update-check.d.ts +0 -3
  592. package/dist/src/cli/lib/update-check.d.ts.map +0 -1
  593. package/dist/src/cli/lib/update-check.js +0 -37
  594. package/dist/src/cli/lib/update-check.js.map +0 -1
  595. package/dist/src/cli/oclif/base-command.d.ts.map +0 -1
  596. package/dist/src/cli/oclif/base-command.js.map +0 -1
  597. package/dist/src/cli/oclif/help.d.ts.map +0 -1
  598. package/dist/src/cli/oclif/help.js.map +0 -1
  599. package/dist/src/cli/oclif/i18n.d.ts.map +0 -1
  600. package/dist/src/cli/oclif/i18n.js.map +0 -1
  601. package/dist/src/cli/oclif/parsers.d.ts.map +0 -1
  602. package/dist/src/cli/oclif/parsers.js.map +0 -1
  603. package/dist/src/cli/oclif/projection.d.ts.map +0 -1
  604. package/dist/src/cli/oclif/projection.js.map +0 -1
  605. package/dist/src/cli/oclif/run.d.ts.map +0 -1
  606. package/dist/src/cli/oclif/run.js +0 -11
  607. package/dist/src/cli/oclif/run.js.map +0 -1
  608. package/dist/src/diagnosis/observe-mapper.d.ts.map +0 -1
  609. package/dist/src/diagnosis/observe-mapper.js.map +0 -1
  610. package/dist/src/diagnosis/observe-producer.d.ts.map +0 -1
  611. package/dist/src/diagnosis/observe-producer.js +0 -197
  612. package/dist/src/diagnosis/observe-producer.js.map +0 -1
  613. package/dist/src/diagnosis/studio-projection.d.ts +0 -14
  614. package/dist/src/diagnosis/studio-projection.d.ts.map +0 -1
  615. package/dist/src/diagnosis/studio-projection.js.map +0 -1
  616. package/dist/src/diagnosis/types.d.ts +0 -79
  617. package/dist/src/diagnosis/types.d.ts.map +0 -1
  618. package/dist/src/diagnosis/types.js +0 -20
  619. package/dist/src/diagnosis/types.js.map +0 -1
  620. package/dist/src/doctor/fixer.d.ts.map +0 -1
  621. package/dist/src/doctor/fixer.js.map +0 -1
  622. package/dist/src/doctor/health/builtin-dimensions.d.ts.map +0 -1
  623. package/dist/src/doctor/health/builtin-dimensions.js.map +0 -1
  624. package/dist/src/doctor/health/composer.d.ts.map +0 -1
  625. package/dist/src/doctor/health/composer.js +0 -296
  626. package/dist/src/doctor/health/composer.js.map +0 -1
  627. package/dist/src/doctor/health/dimension-registry.d.ts.map +0 -1
  628. package/dist/src/doctor/health/dimension-registry.js.map +0 -1
  629. package/dist/src/doctor/health/dimension-spec.d.ts +0 -47
  630. package/dist/src/doctor/health/dimension-spec.d.ts.map +0 -1
  631. package/dist/src/doctor/health/dimension-spec.js.map +0 -1
  632. package/dist/src/doctor/health/parser.d.ts.map +0 -1
  633. package/dist/src/doctor/health/parser.js.map +0 -1
  634. package/dist/src/doctor/health/prompt-builder.d.ts.map +0 -1
  635. package/dist/src/doctor/health/prompt-builder.js.map +0 -1
  636. package/dist/src/doctor/health/register.d.ts.map +0 -1
  637. package/dist/src/doctor/health/register.js.map +0 -1
  638. package/dist/src/doctor/html-renderer.d.ts +0 -20
  639. package/dist/src/doctor/html-renderer.d.ts.map +0 -1
  640. package/dist/src/doctor/html-renderer.js +0 -376
  641. package/dist/src/doctor/html-renderer.js.map +0 -1
  642. package/dist/src/doctor/index.d.ts.map +0 -1
  643. package/dist/src/doctor/index.js.map +0 -1
  644. package/dist/src/doctor/preflight.d.ts.map +0 -1
  645. package/dist/src/doctor/preflight.js.map +0 -1
  646. package/dist/src/doctor/renderer.d.ts +0 -17
  647. package/dist/src/doctor/renderer.d.ts.map +0 -1
  648. package/dist/src/doctor/renderer.js +0 -123
  649. package/dist/src/doctor/renderer.js.map +0 -1
  650. package/dist/src/doctor/rules.d.ts.map +0 -1
  651. package/dist/src/doctor/rules.js +0 -289
  652. package/dist/src/doctor/rules.js.map +0 -1
  653. package/dist/src/eval-core/bootstrap.d.ts.map +0 -1
  654. package/dist/src/eval-core/bootstrap.js.map +0 -1
  655. package/dist/src/eval-core/cache.d.ts.map +0 -1
  656. package/dist/src/eval-core/cache.js.map +0 -1
  657. package/dist/src/eval-core/comparability.d.ts.map +0 -1
  658. package/dist/src/eval-core/comparability.js.map +0 -1
  659. package/dist/src/eval-core/dependency-checker.d.ts +0 -58
  660. package/dist/src/eval-core/dependency-checker.d.ts.map +0 -1
  661. package/dist/src/eval-core/dependency-checker.js.map +0 -1
  662. package/dist/src/eval-core/evaluation-execution.d.ts.map +0 -1
  663. package/dist/src/eval-core/evaluation-execution.js.map +0 -1
  664. package/dist/src/eval-core/evaluation-job.d.ts.map +0 -1
  665. package/dist/src/eval-core/evaluation-job.js.map +0 -1
  666. package/dist/src/eval-core/evaluation-reporting.d.ts.map +0 -1
  667. package/dist/src/eval-core/evaluation-reporting.js.map +0 -1
  668. package/dist/src/eval-core/execution-strategy.d.ts.map +0 -1
  669. package/dist/src/eval-core/execution-strategy.js +0 -165
  670. package/dist/src/eval-core/execution-strategy.js.map +0 -1
  671. package/dist/src/eval-core/fact-checker.d.ts.map +0 -1
  672. package/dist/src/eval-core/fact-checker.js.map +0 -1
  673. package/dist/src/eval-core/layer-gates.d.ts.map +0 -1
  674. package/dist/src/eval-core/layer-gates.js.map +0 -1
  675. package/dist/src/eval-core/mocks-runtime.d.ts.map +0 -1
  676. package/dist/src/eval-core/mocks-runtime.js +0 -559
  677. package/dist/src/eval-core/mocks-runtime.js.map +0 -1
  678. package/dist/src/eval-core/schema.d.ts.map +0 -1
  679. package/dist/src/eval-core/schema.js.map +0 -1
  680. package/dist/src/eval-core/statistics.d.ts.map +0 -1
  681. package/dist/src/eval-core/statistics.js.map +0 -1
  682. package/dist/src/eval-core/task-planner.d.ts.map +0 -1
  683. package/dist/src/eval-core/task-planner.js.map +0 -1
  684. package/dist/src/eval-core/verdict.d.ts.map +0 -1
  685. package/dist/src/eval-core/verdict.js.map +0 -1
  686. package/dist/src/eval-workflows/batch-evaluation-workflow.d.ts.map +0 -1
  687. package/dist/src/eval-workflows/batch-evaluation-workflow.js.map +0 -1
  688. package/dist/src/eval-workflows/evaluation-pipeline.d.ts +0 -109
  689. package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +0 -1
  690. package/dist/src/eval-workflows/evaluation-pipeline.js +0 -373
  691. package/dist/src/eval-workflows/evaluation-pipeline.js.map +0 -1
  692. package/dist/src/eval-workflows/evaluation-preparation.d.ts.map +0 -1
  693. package/dist/src/eval-workflows/evaluation-preparation.js.map +0 -1
  694. package/dist/src/eval-workflows/run-evaluation.d.ts.map +0 -1
  695. package/dist/src/eval-workflows/run-evaluation.js +0 -528
  696. package/dist/src/eval-workflows/run-evaluation.js.map +0 -1
  697. package/dist/src/executors/anthropic-api.d.ts.map +0 -1
  698. package/dist/src/executors/anthropic-api.js.map +0 -1
  699. package/dist/src/executors/claude-cli.d.ts.map +0 -1
  700. package/dist/src/executors/claude-cli.js.map +0 -1
  701. package/dist/src/executors/claude-sdk-trace.d.ts.map +0 -1
  702. package/dist/src/executors/claude-sdk-trace.js.map +0 -1
  703. package/dist/src/executors/claude-sdk.d.ts.map +0 -1
  704. package/dist/src/executors/claude-sdk.js.map +0 -1
  705. package/dist/src/executors/codex-cli-trace.d.ts.map +0 -1
  706. package/dist/src/executors/codex-cli-trace.js.map +0 -1
  707. package/dist/src/executors/codex-cli.d.ts.map +0 -1
  708. package/dist/src/executors/codex-cli.js.map +0 -1
  709. package/dist/src/executors/codex-sdk.d.ts.map +0 -1
  710. package/dist/src/executors/codex-sdk.js.map +0 -1
  711. package/dist/src/executors/gemini.d.ts.map +0 -1
  712. package/dist/src/executors/gemini.js.map +0 -1
  713. package/dist/src/executors/index.d.ts.map +0 -1
  714. package/dist/src/executors/index.js.map +0 -1
  715. package/dist/src/executors/openai-api.d.ts.map +0 -1
  716. package/dist/src/executors/openai-api.js.map +0 -1
  717. package/dist/src/executors/runtime-fingerprint.d.ts.map +0 -1
  718. package/dist/src/executors/runtime-fingerprint.js.map +0 -1
  719. package/dist/src/executors/script.d.ts.map +0 -1
  720. package/dist/src/executors/script.js.map +0 -1
  721. package/dist/src/executors/shared.d.ts +0 -194
  722. package/dist/src/executors/shared.d.ts.map +0 -1
  723. package/dist/src/executors/shared.js +0 -256
  724. package/dist/src/executors/shared.js.map +0 -1
  725. package/dist/src/grading/assertions.d.ts.map +0 -1
  726. package/dist/src/grading/assertions.js.map +0 -1
  727. package/dist/src/grading/debias-validate.d.ts.map +0 -1
  728. package/dist/src/grading/debias-validate.js.map +0 -1
  729. package/dist/src/grading/diagnostic.d.ts.map +0 -1
  730. package/dist/src/grading/diagnostic.js.map +0 -1
  731. package/dist/src/grading/gold-cli.d.ts.map +0 -1
  732. package/dist/src/grading/gold-cli.js.map +0 -1
  733. package/dist/src/grading/gold-dataset.d.ts.map +0 -1
  734. package/dist/src/grading/gold-dataset.js.map +0 -1
  735. package/dist/src/grading/human-gold.d.ts.map +0 -1
  736. package/dist/src/grading/human-gold.js.map +0 -1
  737. package/dist/src/grading/index.d.ts.map +0 -1
  738. package/dist/src/grading/index.js.map +0 -1
  739. package/dist/src/grading/judge.d.ts.map +0 -1
  740. package/dist/src/grading/judge.js.map +0 -1
  741. package/dist/src/grading/layered-scores.d.ts.map +0 -1
  742. package/dist/src/grading/layered-scores.js.map +0 -1
  743. package/dist/src/inputs/eval-config.d.ts.map +0 -1
  744. package/dist/src/inputs/eval-config.js.map +0 -1
  745. package/dist/src/inputs/load-samples.d.ts.map +0 -1
  746. package/dist/src/inputs/load-samples.js.map +0 -1
  747. package/dist/src/inputs/mcp-resolver.d.ts.map +0 -1
  748. package/dist/src/inputs/mcp-resolver.js.map +0 -1
  749. package/dist/src/inputs/skill-loader.d.ts.map +0 -1
  750. package/dist/src/inputs/skill-loader.js +0 -253
  751. package/dist/src/inputs/skill-loader.js.map +0 -1
  752. package/dist/src/inputs/url-fetcher.d.ts.map +0 -1
  753. package/dist/src/inputs/url-fetcher.js.map +0 -1
  754. package/dist/src/observability/experience.d.ts +0 -441
  755. package/dist/src/observability/experience.d.ts.map +0 -1
  756. package/dist/src/observability/experience.js +0 -2400
  757. package/dist/src/observability/experience.js.map +0 -1
  758. package/dist/src/observability/inbox-view-model.d.ts +0 -30
  759. package/dist/src/observability/inbox-view-model.d.ts.map +0 -1
  760. package/dist/src/observability/inbox-view-model.js +0 -144
  761. package/dist/src/observability/inbox-view-model.js.map +0 -1
  762. package/dist/src/observability/inbox.d.ts +0 -127
  763. package/dist/src/observability/inbox.d.ts.map +0 -1
  764. package/dist/src/observability/inbox.js +0 -755
  765. package/dist/src/observability/inbox.js.map +0 -1
  766. package/dist/src/observability/problem-patterns.d.ts +0 -52
  767. package/dist/src/observability/problem-patterns.d.ts.map +0 -1
  768. package/dist/src/observability/problem-patterns.js +0 -205
  769. package/dist/src/observability/problem-patterns.js.map +0 -1
  770. package/dist/src/observability/resolved-review.d.ts +0 -25
  771. package/dist/src/observability/resolved-review.d.ts.map +0 -1
  772. package/dist/src/observability/resolved-review.js +0 -231
  773. package/dist/src/observability/resolved-review.js.map +0 -1
  774. package/dist/src/observability/review-state.d.ts +0 -61
  775. package/dist/src/observability/review-state.d.ts.map +0 -1
  776. package/dist/src/observability/review-state.js.map +0 -1
  777. package/dist/src/observability/skill-chain-advisories.d.ts +0 -25
  778. package/dist/src/observability/skill-chain-advisories.d.ts.map +0 -1
  779. package/dist/src/observability/skill-chain-advisories.js.map +0 -1
  780. package/dist/src/observability/skill-chain.d.ts +0 -62
  781. package/dist/src/observability/skill-chain.d.ts.map +0 -1
  782. package/dist/src/observability/skill-chain.js +0 -270
  783. package/dist/src/observability/skill-chain.js.map +0 -1
  784. package/dist/src/observability/skill-health-analyzer.d.ts.map +0 -1
  785. package/dist/src/observability/skill-health-analyzer.js.map +0 -1
  786. package/dist/src/observability/soft-standards.d.ts +0 -130
  787. package/dist/src/observability/soft-standards.d.ts.map +0 -1
  788. package/dist/src/observability/soft-standards.js +0 -426
  789. package/dist/src/observability/soft-standards.js.map +0 -1
  790. package/dist/src/observability/text-signals.d.ts +0 -14
  791. package/dist/src/observability/text-signals.d.ts.map +0 -1
  792. package/dist/src/observability/text-signals.js +0 -106
  793. package/dist/src/observability/text-signals.js.map +0 -1
  794. package/dist/src/observability/trace-adapter.d.ts.map +0 -1
  795. package/dist/src/observability/trace-adapter.js.map +0 -1
  796. package/dist/src/observability/trace-attribution.d.ts +0 -56
  797. package/dist/src/observability/trace-attribution.d.ts.map +0 -1
  798. package/dist/src/observability/trace-attribution.js +0 -200
  799. package/dist/src/observability/trace-attribution.js.map +0 -1
  800. package/dist/src/observability/trace-segmenter.d.ts +0 -52
  801. package/dist/src/observability/trace-segmenter.d.ts.map +0 -1
  802. package/dist/src/observability/trace-segmenter.js +0 -322
  803. package/dist/src/observability/trace-segmenter.js.map +0 -1
  804. package/dist/src/observability/trace-source.d.ts +0 -99
  805. package/dist/src/observability/trace-source.d.ts.map +0 -1
  806. package/dist/src/observability/trace-source.js +0 -502
  807. package/dist/src/observability/trace-source.js.map +0 -1
  808. package/dist/src/renderer/html-renderer.d.ts.map +0 -1
  809. package/dist/src/renderer/html-renderer.js.map +0 -1
  810. package/dist/src/renderer/layout.d.ts.map +0 -1
  811. package/dist/src/renderer/layout.js.map +0 -1
  812. package/dist/src/renderer/observation-inbox-renderer.d.ts +0 -4
  813. package/dist/src/renderer/observation-inbox-renderer.d.ts.map +0 -1
  814. package/dist/src/renderer/observation-inbox-renderer.js +0 -10366
  815. package/dist/src/renderer/observation-inbox-renderer.js.map +0 -1
  816. package/dist/src/renderer/skill-detail-renderer.d.ts +0 -15
  817. package/dist/src/renderer/skill-detail-renderer.d.ts.map +0 -1
  818. package/dist/src/renderer/skill-detail-renderer.js.map +0 -1
  819. package/dist/src/renderer/skill-health-renderer.d.ts.map +0 -1
  820. package/dist/src/renderer/skill-health-renderer.js.map +0 -1
  821. package/dist/src/renderer/skill-list-renderer.d.ts +0 -4
  822. package/dist/src/renderer/skill-list-renderer.d.ts.map +0 -1
  823. package/dist/src/renderer/skill-list-renderer.js.map +0 -1
  824. package/dist/src/renderer/summary.d.ts +0 -68
  825. package/dist/src/renderer/summary.d.ts.map +0 -1
  826. package/dist/src/renderer/summary.js +0 -1896
  827. package/dist/src/renderer/summary.js.map +0 -1
  828. package/dist/src/renderer/table.d.ts.map +0 -1
  829. package/dist/src/renderer/table.js.map +0 -1
  830. package/dist/src/renderer/test-view.d.ts.map +0 -1
  831. package/dist/src/renderer/test-view.js.map +0 -1
  832. package/dist/src/renderer/trends.d.ts.map +0 -1
  833. package/dist/src/renderer/trends.js.map +0 -1
  834. package/dist/src/server/job-store.d.ts.map +0 -1
  835. package/dist/src/server/job-store.js.map +0 -1
  836. package/dist/src/server/report-server.d.ts.map +0 -1
  837. package/dist/src/server/report-server.js +0 -894
  838. package/dist/src/server/report-server.js.map +0 -1
  839. package/dist/src/server/report-store.d.ts.map +0 -1
  840. package/dist/src/server/report-store.js.map +0 -1
  841. package/dist/src/server/skill-index.d.ts +0 -81
  842. package/dist/src/server/skill-index.d.ts.map +0 -1
  843. package/dist/src/server/skill-index.js +0 -377
  844. package/dist/src/server/skill-index.js.map +0 -1
  845. package/dist/src/server/skill-insights.d.ts +0 -101
  846. package/dist/src/server/skill-insights.d.ts.map +0 -1
  847. package/dist/src/server/skill-insights.js.map +0 -1
  848. package/dist/src/shared/hard-rules.d.ts.map +0 -1
  849. package/dist/src/shared/hard-rules.js.map +0 -1
  850. package/dist/src/shared/llm-prompts/index.d.ts +0 -14
  851. package/dist/src/shared/llm-prompts/index.d.ts.map +0 -1
  852. package/dist/src/shared/llm-prompts/index.js +0 -35
  853. package/dist/src/shared/llm-prompts/index.js.map +0 -1
  854. package/dist/src/shared/llm-prompts/skill-health.d.ts.map +0 -1
  855. package/dist/src/shared/llm-prompts/skill-health.js.map +0 -1
  856. package/dist/src/shared/time.d.ts.map +0 -1
  857. package/dist/src/shared/time.js.map +0 -1
  858. package/dist/src/shared/tool-search.d.ts.map +0 -1
  859. package/dist/src/shared/tool-search.js.map +0 -1
  860. package/dist/src/types/doctor.d.ts +0 -157
  861. package/dist/src/types/doctor.d.ts.map +0 -1
  862. package/dist/src/types/doctor.js.map +0 -1
  863. package/dist/src/types/eval.d.ts +0 -377
  864. package/dist/src/types/eval.d.ts.map +0 -1
  865. package/dist/src/types/eval.js.map +0 -1
  866. package/dist/src/types/executor.d.ts.map +0 -1
  867. package/dist/src/types/executor.js.map +0 -1
  868. package/dist/src/types/index.d.ts +0 -8
  869. package/dist/src/types/index.d.ts.map +0 -1
  870. package/dist/src/types/index.js +0 -8
  871. package/dist/src/types/index.js.map +0 -1
  872. package/dist/src/types/judge.d.ts.map +0 -1
  873. package/dist/src/types/judge.js.map +0 -1
  874. package/dist/src/types/report.d.ts +0 -557
  875. package/dist/src/types/report.d.ts.map +0 -1
  876. package/dist/src/types/report.js.map +0 -1
  877. package/dist/src/types/shared.d.ts.map +0 -1
  878. package/dist/src/types/shared.js.map +0 -1
  879. package/dist/src/types/storage.d.ts.map +0 -1
  880. package/dist/src/types/storage.js.map +0 -1
  881. package/dist/src/util/safe-slice.d.ts +0 -23
  882. package/dist/src/util/safe-slice.d.ts.map +0 -1
  883. package/dist/src/util/safe-slice.js +0 -33
  884. package/dist/src/util/safe-slice.js.map +0 -1
  885. package/docs/prompts/llm-enhanced-review.prompt.md +0 -111
  886. /package/dist/{src/analysis → analysis}/coverage-analyzer.d.ts +0 -0
  887. /package/dist/{src/analysis → analysis}/coverage-analyzer.js +0 -0
  888. /package/dist/{src/analysis → analysis}/failure-clusterer.d.ts +0 -0
  889. /package/dist/{src/analysis → analysis}/failure-clusterer.js +0 -0
  890. /package/dist/{src/analysis → analysis}/hedging-classifier.d.ts +0 -0
  891. /package/dist/{src/analysis → analysis}/hedging-classifier.js +0 -0
  892. /package/dist/{src/analysis → analysis}/sample-diagnostics.d.ts +0 -0
  893. /package/dist/{src/analysis → analysis}/sample-diagnostics.js +0 -0
  894. /package/dist/{src/analysis → analysis}/saturation.d.ts +0 -0
  895. /package/dist/{src/analysis → analysis}/saturation.js +0 -0
  896. /package/dist/{src/authoring → authoring}/evolver.d.ts +0 -0
  897. /package/dist/{src/authoring → authoring}/evolver.js +0 -0
  898. /package/dist/{src/authoring → authoring}/generator.d.ts +0 -0
  899. /package/dist/{src/authoring → authoring}/sample-fixer.d.ts +0 -0
  900. /package/dist/{src/authoring → authoring}/sample-fixer.js +0 -0
  901. /package/dist/{src/cli → cli}/commands/eval/gold/compare.d.ts +0 -0
  902. /package/dist/{src/cli → cli}/commands/eval/gold/compare.js +0 -0
  903. /package/dist/{src/cli → cli}/commands/eval/gold/index.d.ts +0 -0
  904. /package/dist/{src/cli → cli}/commands/eval/gold/index.js +0 -0
  905. /package/dist/{src/cli → cli}/commands/eval/gold/init.d.ts +0 -0
  906. /package/dist/{src/cli → cli}/commands/eval/gold/init.js +0 -0
  907. /package/dist/{src/cli → cli}/commands/eval/gold/validate.d.ts +0 -0
  908. /package/dist/{src/cli → cli}/commands/eval/gold/validate.js +0 -0
  909. /package/dist/{src/cli → cli}/commands/eval/index.d.ts +0 -0
  910. /package/dist/{src/cli → cli}/commands/evolve.d.ts +0 -0
  911. /package/dist/{src/cli → cli}/commands/init.d.ts +0 -0
  912. /package/dist/{src/cli → cli}/commands/init.js +0 -0
  913. /package/dist/{src/cli → cli}/commands/observe/inbox.d.ts +0 -0
  914. /package/dist/{src/cli → cli}/commands/observe/index.d.ts +0 -0
  915. /package/dist/{src/cli → cli}/commands/observe/index.js +0 -0
  916. /package/dist/{src/cli → cli}/commands/observe/ingest.d.ts +0 -0
  917. /package/dist/{src/cli → cli}/commands/observe/show.d.ts +0 -0
  918. /package/dist/{src/cli → cli}/commands/observe/show.js +0 -0
  919. /package/dist/{src/cli → cli}/commands/sample.d.ts +0 -0
  920. /package/dist/{src/cli → cli}/commands/studio.d.ts +0 -0
  921. /package/dist/{src/cli → cli}/index.d.ts +0 -0
  922. /package/dist/{src/cli → cli}/index.js +0 -0
  923. /package/dist/{src/cli → cli}/lib/cli-exit.d.ts +0 -0
  924. /package/dist/{src/cli → cli}/lib/cli-exit.js +0 -0
  925. /package/dist/{src/cli → cli}/lib/cmd-flags.js +0 -0
  926. /package/dist/{src/cli → cli}/lib/i18n.d.ts +0 -0
  927. /package/dist/{src/cli → cli}/lib/i18n.js +0 -0
  928. /package/dist/{src/cli → cli}/lib/progress.d.ts +0 -0
  929. /package/dist/{src/cli → cli}/lib/progress.js +0 -0
  930. /package/dist/{src/cli → cli}/lib/run-tally.d.ts +0 -0
  931. /package/dist/{src/cli → cli}/lib/run-tally.js +0 -0
  932. /package/dist/{src/cli → cli}/lib/shared.d.ts +0 -0
  933. /package/dist/{src/cli → cli}/lib/shared.js +0 -0
  934. /package/dist/{src/cli → cli}/oclif/base-command.d.ts +0 -0
  935. /package/dist/{src/cli → cli}/oclif/base-command.js +0 -0
  936. /package/dist/{src/cli → cli}/oclif/help.d.ts +0 -0
  937. /package/dist/{src/cli → cli}/oclif/help.js +0 -0
  938. /package/dist/{src/cli → cli}/oclif/i18n.d.ts +0 -0
  939. /package/dist/{src/cli → cli}/oclif/i18n.js +0 -0
  940. /package/dist/{src/cli → cli}/oclif/parsers.d.ts +0 -0
  941. /package/dist/{src/cli → cli}/oclif/parsers.js +0 -0
  942. /package/dist/{src/cli → cli}/oclif/projection.d.ts +0 -0
  943. /package/dist/{src/cli → cli}/oclif/projection.js +0 -0
  944. /package/dist/{src/cli → cli}/oclif/run.d.ts +0 -0
  945. /package/dist/{src/diagnosis → diagnosis}/observe-mapper.d.ts +0 -0
  946. /package/dist/{src/diagnosis → diagnosis}/observe-mapper.js +0 -0
  947. /package/dist/{src/diagnosis → diagnosis}/observe-producer.d.ts +0 -0
  948. /package/dist/{src/diagnosis → diagnosis}/studio-projection.js +0 -0
  949. /package/dist/{src/doctor → doctor}/fixer.d.ts +0 -0
  950. /package/dist/{src/doctor → doctor}/fixer.js +0 -0
  951. /package/dist/{src/doctor → doctor}/health/builtin-dimensions.d.ts +0 -0
  952. /package/dist/{src/doctor → doctor}/health/builtin-dimensions.js +0 -0
  953. /package/dist/{src/doctor → doctor}/health/composer.d.ts +0 -0
  954. /package/dist/{src/doctor → doctor}/health/dimension-registry.d.ts +0 -0
  955. /package/dist/{src/doctor → doctor}/health/dimension-registry.js +0 -0
  956. /package/dist/{src/doctor → doctor}/health/dimension-spec.js +0 -0
  957. /package/dist/{src/doctor → doctor}/health/parser.d.ts +0 -0
  958. /package/dist/{src/doctor → doctor}/health/parser.js +0 -0
  959. /package/dist/{src/doctor → doctor}/health/prompt-builder.d.ts +0 -0
  960. /package/dist/{src/doctor → doctor}/health/prompt-builder.js +0 -0
  961. /package/dist/{src/doctor → doctor}/health/register.d.ts +0 -0
  962. /package/dist/{src/doctor → doctor}/health/register.js +0 -0
  963. /package/dist/{src/doctor → doctor}/index.d.ts +0 -0
  964. /package/dist/{src/doctor → doctor}/index.js +0 -0
  965. /package/dist/{src/doctor → doctor}/preflight.d.ts +0 -0
  966. /package/dist/{src/doctor → doctor}/preflight.js +0 -0
  967. /package/dist/{src/doctor → doctor}/rules.d.ts +0 -0
  968. /package/dist/{src/eval-core → eval-core}/bootstrap.d.ts +0 -0
  969. /package/dist/{src/eval-core → eval-core}/bootstrap.js +0 -0
  970. /package/dist/{src/eval-core → eval-core}/cache.d.ts +0 -0
  971. /package/dist/{src/eval-core → eval-core}/cache.js +0 -0
  972. /package/dist/{src/eval-core → eval-core}/comparability.d.ts +0 -0
  973. /package/dist/{src/eval-core → eval-core}/comparability.js +0 -0
  974. /package/dist/{src/eval-core → eval-core}/dependency-checker.js +0 -0
  975. /package/dist/{src/eval-core → eval-core}/evaluation-execution.d.ts +0 -0
  976. /package/dist/{src/eval-core → eval-core}/evaluation-execution.js +0 -0
  977. /package/dist/{src/eval-core → eval-core}/evaluation-job.d.ts +0 -0
  978. /package/dist/{src/eval-core → eval-core}/evaluation-job.js +0 -0
  979. /package/dist/{src/eval-core → eval-core}/evaluation-reporting.d.ts +0 -0
  980. /package/dist/{src/eval-core → eval-core}/evaluation-reporting.js +0 -0
  981. /package/dist/{src/eval-core → eval-core}/execution-strategy.d.ts +0 -0
  982. /package/dist/{src/eval-core → eval-core}/fact-checker.d.ts +0 -0
  983. /package/dist/{src/eval-core → eval-core}/fact-checker.js +0 -0
  984. /package/dist/{src/eval-core → eval-core}/layer-gates.d.ts +0 -0
  985. /package/dist/{src/eval-core → eval-core}/layer-gates.js +0 -0
  986. /package/dist/{src/eval-core → eval-core}/mock-hook.cjs +0 -0
  987. /package/dist/{src/eval-core → eval-core}/mocks-runtime.d.ts +0 -0
  988. /package/dist/{src/eval-core → eval-core}/schema.d.ts +0 -0
  989. /package/dist/{src/eval-core → eval-core}/schema.js +0 -0
  990. /package/dist/{src/eval-core → eval-core}/statistics.d.ts +0 -0
  991. /package/dist/{src/eval-core → eval-core}/statistics.js +0 -0
  992. /package/dist/{src/eval-core → eval-core}/task-planner.d.ts +0 -0
  993. /package/dist/{src/eval-core → eval-core}/task-planner.js +0 -0
  994. /package/dist/{src/eval-core → eval-core}/verdict.d.ts +0 -0
  995. /package/dist/{src/eval-core → eval-core}/verdict.js +0 -0
  996. /package/dist/{src/eval-workflows → eval-workflows}/batch-evaluation-workflow.d.ts +0 -0
  997. /package/dist/{src/eval-workflows → eval-workflows}/batch-evaluation-workflow.js +0 -0
  998. /package/dist/{src/eval-workflows → eval-workflows}/evaluation-preparation.d.ts +0 -0
  999. /package/dist/{src/eval-workflows → eval-workflows}/evaluation-preparation.js +0 -0
  1000. /package/dist/{src/eval-workflows → eval-workflows}/run-evaluation.d.ts +0 -0
  1001. /package/dist/{src/executors → executors}/anthropic-api.d.ts +0 -0
  1002. /package/dist/{src/executors → executors}/anthropic-api.js +0 -0
  1003. /package/dist/{src/executors → executors}/claude-cli.d.ts +0 -0
  1004. /package/dist/{src/executors → executors}/claude-cli.js +0 -0
  1005. /package/dist/{src/executors → executors}/claude-sdk-trace.d.ts +0 -0
  1006. /package/dist/{src/executors → executors}/claude-sdk-trace.js +0 -0
  1007. /package/dist/{src/executors → executors}/claude-sdk.d.ts +0 -0
  1008. /package/dist/{src/executors → executors}/claude-sdk.js +0 -0
  1009. /package/dist/{src/executors → executors}/codex-cli-trace.d.ts +0 -0
  1010. /package/dist/{src/executors → executors}/codex-cli-trace.js +0 -0
  1011. /package/dist/{src/executors → executors}/codex-cli.d.ts +0 -0
  1012. /package/dist/{src/executors → executors}/codex-cli.js +0 -0
  1013. /package/dist/{src/executors → executors}/codex-sdk.d.ts +0 -0
  1014. /package/dist/{src/executors → executors}/codex-sdk.js +0 -0
  1015. /package/dist/{src/executors → executors}/gemini.d.ts +0 -0
  1016. /package/dist/{src/executors → executors}/gemini.js +0 -0
  1017. /package/dist/{src/executors → executors}/index.d.ts +0 -0
  1018. /package/dist/{src/executors → executors}/index.js +0 -0
  1019. /package/dist/{src/executors → executors}/openai-api.d.ts +0 -0
  1020. /package/dist/{src/executors → executors}/openai-api.js +0 -0
  1021. /package/dist/{src/executors → executors}/runtime-fingerprint.d.ts +0 -0
  1022. /package/dist/{src/executors → executors}/runtime-fingerprint.js +0 -0
  1023. /package/dist/{src/executors → executors}/script.d.ts +0 -0
  1024. /package/dist/{src/executors → executors}/script.js +0 -0
  1025. /package/dist/{src/grading → grading}/assertions.d.ts +0 -0
  1026. /package/dist/{src/grading → grading}/assertions.js +0 -0
  1027. /package/dist/{src/grading → grading}/debias-validate.d.ts +0 -0
  1028. /package/dist/{src/grading → grading}/debias-validate.js +0 -0
  1029. /package/dist/{src/grading → grading}/diagnostic.d.ts +0 -0
  1030. /package/dist/{src/grading → grading}/diagnostic.js +0 -0
  1031. /package/dist/{src/grading → grading}/gold-cli.d.ts +0 -0
  1032. /package/dist/{src/grading → grading}/gold-cli.js +0 -0
  1033. /package/dist/{src/grading → grading}/gold-dataset.d.ts +0 -0
  1034. /package/dist/{src/grading → grading}/gold-dataset.js +0 -0
  1035. /package/dist/{src/grading → grading}/human-gold.d.ts +0 -0
  1036. /package/dist/{src/grading → grading}/human-gold.js +0 -0
  1037. /package/dist/{src/grading → grading}/index.d.ts +0 -0
  1038. /package/dist/{src/grading → grading}/index.js +0 -0
  1039. /package/dist/{src/grading → grading}/judge.d.ts +0 -0
  1040. /package/dist/{src/grading → grading}/judge.js +0 -0
  1041. /package/dist/{src/grading → grading}/layered-scores.d.ts +0 -0
  1042. /package/dist/{src/grading → grading}/layered-scores.js +0 -0
  1043. /package/dist/{src/inputs → inputs}/eval-config.d.ts +0 -0
  1044. /package/dist/{src/inputs → inputs}/eval-config.js +0 -0
  1045. /package/dist/{src/inputs → inputs}/load-samples.d.ts +0 -0
  1046. /package/dist/{src/inputs → inputs}/load-samples.js +0 -0
  1047. /package/dist/{src/inputs → inputs}/mcp-resolver.d.ts +0 -0
  1048. /package/dist/{src/inputs → inputs}/mcp-resolver.js +0 -0
  1049. /package/dist/{src/inputs → inputs}/skill-loader.d.ts +0 -0
  1050. /package/dist/{src/inputs → inputs}/url-fetcher.d.ts +0 -0
  1051. /package/dist/{src/inputs → inputs}/url-fetcher.js +0 -0
  1052. /package/dist/{src/observability → observability}/review-state.js +0 -0
  1053. /package/dist/{src/observability → observability}/skill-chain-advisories.js +0 -0
  1054. /package/dist/{src/observability → observability}/skill-health-analyzer.d.ts +0 -0
  1055. /package/dist/{src/observability → observability}/skill-health-analyzer.js +0 -0
  1056. /package/dist/{src/observability → observability}/trace-adapter.d.ts +0 -0
  1057. /package/dist/{src/observability → observability}/trace-adapter.js +0 -0
  1058. /package/dist/{src/renderer → renderer}/html-renderer.d.ts +0 -0
  1059. /package/dist/{src/renderer → renderer}/html-renderer.js +0 -0
  1060. /package/dist/{src/renderer → renderer}/layout.d.ts +0 -0
  1061. /package/dist/{src/renderer → renderer}/layout.js +0 -0
  1062. /package/dist/{src/renderer → renderer}/skill-detail-renderer.js +0 -0
  1063. /package/dist/{src/renderer → renderer}/skill-health-renderer.d.ts +0 -0
  1064. /package/dist/{src/renderer → renderer}/skill-health-renderer.js +0 -0
  1065. /package/dist/{src/renderer → renderer}/skill-list-renderer.js +0 -0
  1066. /package/dist/{src/renderer → renderer}/table.d.ts +0 -0
  1067. /package/dist/{src/renderer → renderer}/table.js +0 -0
  1068. /package/dist/{src/renderer → renderer}/test-view.d.ts +0 -0
  1069. /package/dist/{src/renderer → renderer}/test-view.js +0 -0
  1070. /package/dist/{src/renderer → renderer}/trends.d.ts +0 -0
  1071. /package/dist/{src/renderer → renderer}/trends.js +0 -0
  1072. /package/dist/{src/server → server}/job-store.d.ts +0 -0
  1073. /package/dist/{src/server → server}/job-store.js +0 -0
  1074. /package/dist/{src/server → server}/report-server.d.ts +0 -0
  1075. /package/dist/{src/server → server}/report-store.d.ts +0 -0
  1076. /package/dist/{src/server → server}/report-store.js +0 -0
  1077. /package/dist/{src/server → server}/skill-insights.js +0 -0
  1078. /package/dist/{src/shared → shared}/hard-rules.d.ts +0 -0
  1079. /package/dist/{src/shared → shared}/hard-rules.js +0 -0
  1080. /package/dist/{src/shared → shared}/llm-prompts/skill-health.d.ts +0 -0
  1081. /package/dist/{src/shared → shared}/llm-prompts/skill-health.js +0 -0
  1082. /package/dist/{src/shared → shared}/time.d.ts +0 -0
  1083. /package/dist/{src/shared → shared}/time.js +0 -0
  1084. /package/dist/{src/shared → shared}/tool-search.d.ts +0 -0
  1085. /package/dist/{src/shared → shared}/tool-search.js +0 -0
  1086. /package/dist/{src/types → types}/doctor.js +0 -0
  1087. /package/dist/{src/types → types}/eval.js +0 -0
  1088. /package/dist/{src/types → types}/executor.d.ts +0 -0
  1089. /package/dist/{src/types → types}/executor.js +0 -0
  1090. /package/dist/{src/types → types}/judge.d.ts +0 -0
  1091. /package/dist/{src/types → types}/judge.js +0 -0
  1092. /package/dist/{src/types → types}/report.js +0 -0
  1093. /package/dist/{src/types → types}/shared.d.ts +0 -0
  1094. /package/dist/{src/types → types}/shared.js +0 -0
  1095. /package/dist/{src/types → types}/storage.d.ts +0 -0
  1096. /package/dist/{src/types → types}/storage.js +0 -0
@@ -1,1896 +0,0 @@
1
- import { e, fmtNum, fmtCost, fmtDuration, COLORS, t } from './layout.js';
2
- import { generateAnalysisSummary } from '../analysis/report-diagnostics.js';
3
- import { pValueCategory } from '../eval-core/statistics.js';
4
- import { computeVerdict } from '../eval-core/verdict.js';
5
- /**
6
- * Verdict pill — sticky banner at the top of the HTML report giving the same
7
- * one-line conclusion as the `omk eval` CLI. Both surfaces share the
8
- * computeVerdict rule engine so they can never disagree.
9
- *
10
- * Color coding follows traffic-light convention plus a yellow band for
11
- * UNDERPOWERED / NOISE / CAUTIOUS — none of which are "ship" but none of which
12
- * are "regress" either. SOLO is grey because it's an info pill, not a verdict.
13
- */
14
- // v0.21 B.4 — verdict 一句话总结. 把 level 标签 + Δ 数字 + action 推荐 + 原因
15
- // 压成一个完整的中文/英文句子, 用 variant 名直接说"X 比 Y 怎么样, 怎么办".
16
- // 用户读完一句话就懂结论, 不需要先学 jargon (Δ / CI / NOISE / SHIP).
17
- // 数字细节让"六维对比"表展示, verdict pill 只给 takeaway.
18
- function verdictOneLine(level, lang, treatment, control) {
19
- const t = treatment ?? (lang === 'zh' ? '实验组' : 'treatment');
20
- const c = control ?? (lang === 'zh' ? '对照组' : 'control');
21
- if (lang === 'zh') {
22
- switch (level) {
23
- case 'PROGRESS': return `${t} 比 ${c} 明显更好——可以发布`;
24
- case 'CAUTIOUS': return `${t} 比 ${c} 略好——但建议再仔细看,差距很小或某层未达标`;
25
- case 'REGRESS': return `${t} 比 ${c} 明显更差——不要发布`;
26
- case 'NOISE': return `${t} 和 ${c} 没看出明显差别——可以多加几条用例再试`;
27
- case 'UNDERPOWERED': return `评测用例数太少,看不出 ${t} 和 ${c} 的差别——多跑几个再看`;
28
- case 'SOLO': return `只跑了一组,需要加对照组才能对比`;
29
- }
30
- }
31
- switch (level) {
32
- case 'PROGRESS': return `${t} is clearly better than ${c} — ready to ship`;
33
- case 'CAUTIOUS': return `${t} is slightly better than ${c} — investigate, gap is small or a layer is below threshold`;
34
- case 'REGRESS': return `${t} is clearly worse than ${c} — do not ship`;
35
- case 'NOISE': return `No clear difference between ${t} and ${c} — try more samples`;
36
- case 'UNDERPOWERED': return `Not enough data to compare ${t} and ${c} — run more samples`;
37
- case 'SOLO': return `Single variant — add a control to compare`;
38
- }
39
- }
40
- // v0.21 B.4 — 中文标签用"程度副词 + 方向" 三段式 (明显进步 / 略微进步 / 基本
41
- // 持平 / 明显退步) 让用户一眼读懂,不必先学 jargon. 英文保留 level code (开发者
42
- // 用户多, code 反而无歧义).
43
- export function levelLabel(level, lang) {
44
- if (lang === 'zh') {
45
- switch (level) {
46
- case 'PROGRESS': return '明显进步';
47
- case 'CAUTIOUS': return '略微进步';
48
- case 'REGRESS': return '明显退步';
49
- case 'NOISE': return '基本持平';
50
- case 'UNDERPOWERED': return '用例不足';
51
- case 'SOLO': return '无法对比';
52
- }
53
- }
54
- return level;
55
- }
56
- // v0.21 B.4 — Tooltip 写人话,不是 jargon 重复. 出现在 listing 页的 status pill
57
- // title 和 detail 页 banner 的 aria-label 上. 用户 hover 立刻知道这个 status
58
- // 是怎么算出来的, 不用查文档.
59
- export function levelTooltip(level, lang) {
60
- if (lang === 'zh') {
61
- switch (level) {
62
- case 'PROGRESS': return '实验组分数显著优于对照组';
63
- case 'REGRESS': return '实验组分数显著劣于对照组';
64
- case 'CAUTIOUS': return '实验组略优于对照组,但差距小或某层未达 gate';
65
- case 'NOISE': return '两组分数差距置信区间跨过 0,统计上分辨不出效果';
66
- case 'UNDERPOWERED': return '评测用例数太少,需要多加几条再看';
67
- case 'SOLO': return '只跑了一组,没做对比';
68
- }
69
- }
70
- switch (level) {
71
- case 'PROGRESS': return 'Treatment scores significantly above control';
72
- case 'REGRESS': return 'Treatment scores significantly below control';
73
- case 'CAUTIOUS': return 'Treatment slightly above control — small gap or layer below gate';
74
- case 'NOISE': return 'Diff CI spans 0; effect not separable from noise';
75
- case 'UNDERPOWERED': return 'Sample size too small to detect effect';
76
- case 'SOLO': return 'Single variant — no comparison';
77
- }
78
- }
79
- // Verdict hero — verdict 是报告的「答案」,应在头部抢眼可读:
80
- // 行 1: 状态 badge (明显进步 / PROGRESS) + 自然语言句子 (含 ship action)
81
- // 行 2: 分差 + 评测规模 — 最直观的两个数字, "差多少 / 跑了多少"
82
- // machine-readable enum 通过 data-verdict-level 属性挂在 section 上,给 CI/工具用;
83
- // ship-action token 写进 aria-label 给 screen reader,但不放进 badge 显示文字 ——
84
- // 因为 verdictOneLine 的中文句子已经包含 "可以发布 / 不要发布" 这类 action,
85
- // badge 再放一遍是重复。CI / CV 这种统计学专名挪到 variance 表 + tooltip,hero
86
- // 不堆 jargon. 想看精确边界值的工程师下滑到下方的配对对比 / 波动检验表里看。
87
- // computeVerdict 在历史混合批量 report 上有 NPE 风险,try/catch 让 renderer 不 crash。
88
- const SHIP_LABEL = {
89
- PROGRESS: { zh: '可发布', en: 'SHIP' },
90
- CAUTIOUS: { zh: '需排查', en: 'INVESTIGATE' },
91
- REGRESS: { zh: '勿发布', en: 'DO NOT SHIP' },
92
- NOISE: { zh: '不下结论', en: 'NO CALL' },
93
- UNDERPOWERED: { zh: '数据不足', en: 'INSUFFICIENT DATA' },
94
- SOLO: { zh: '缺对照', en: 'ADD CONTROL' },
95
- };
96
- function computeMedianCVPercent(report) {
97
- const variance = report.variance;
98
- if (!variance || (variance.runs ?? 0) < 2)
99
- return null;
100
- const cvs = [];
101
- for (const v of Object.values(variance.perVariant ?? {})) {
102
- if (typeof v.stddev === 'number' && typeof v.mean === 'number' && v.mean > 0) {
103
- cvs.push((v.stddev / v.mean) * 100);
104
- }
105
- }
106
- if (cvs.length === 0)
107
- return null;
108
- cvs.sort((a, b) => a - b);
109
- return cvs[Math.floor(cvs.length / 2)];
110
- }
111
- export function renderVerdictPill(report, lang) {
112
- let result;
113
- try {
114
- result = computeVerdict(report);
115
- }
116
- catch {
117
- return '';
118
- }
119
- const level = result.level;
120
- const pair = result.perPair?.[0];
121
- const oneLine = verdictOneLine(level, lang, pair?.treatment, pair?.control);
122
- const tooltip = levelTooltip(level, lang);
123
- const prefix = lang === 'zh' ? '测评结论' : 'Verdict';
124
- // 机器可读 enum 永远是 level token; 显示给用户的文字按 lang i18n.
125
- const levelDisplay = lang === 'zh' ? levelLabel(level, lang) : level;
126
- const shipAria = lang === 'zh' ? SHIP_LABEL[level].zh : SHIP_LABEL[level].en;
127
- // hero 只放「答案」: 分差是 verdict 的核心证据数字, 单独一枚 chip。
128
- // 评测规模 (用例数 × 轮次) 走「实验配置」section 的 subtitle 那条 canonical 路径,
129
- // 不在 hero 里重复; CV / CI 走 chip tooltip + 方法学审计 / 波动表。
130
- const ci = report.meta?.pairComparisons?.[0]?.diffBootstrapCI;
131
- const cvPct = computeMedianCVPercent(report);
132
- const metrics = [];
133
- if (ci) {
134
- const sign = ci.estimate >= 0 ? '+' : '';
135
- const cvSuffix = cvPct != null
136
- ? (lang === 'zh' ? `;多轮稳定性 CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? '稳' : cvPct < 15 ? '中' : '不稳'})` : `; CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? 'stable' : cvPct < 15 ? 'moderate' : 'unstable'})`)
137
- : '';
138
- const ciTipBase = lang === 'zh'
139
- ? `实验组与对照组综合分均值差(Δ)。bootstrap 95% 可信区间 [${ci.low}, ${ci.high}],${ci.significant ? '不含 0 = 差异显著' : '跨过 0 = 差异不显著'}${cvSuffix}`
140
- : `Treatment minus control mean composite score (Δ). Bootstrap 95% CI [${ci.low}, ${ci.high}], ${ci.significant ? 'excludes 0 ⇒ significant' : 'spans 0 ⇒ not significant'}${cvSuffix}`;
141
- metrics.push({
142
- label: lang === 'zh' ? '分差' : 'Δ',
143
- value: `${sign}${ci.estimate}`,
144
- tip: ciTipBase,
145
- });
146
- }
147
- const metricChips = metrics.map((m) => `<span class="verdict-metric"${m.tip ? ` title="${e(m.tip)}"` : ''}>` +
148
- `<span class="verdict-metric-label">${e(m.label)}</span>` +
149
- `<span class="verdict-metric-value">${e(m.value)}</span></span>`).join('');
150
- return `<section class="page-verdict verdict-${level}" role="status" data-verdict-level="${e(level)}" aria-label="${e(prefix)}: ${e(levelDisplay)} · ${e(shipAria)} · ${e(oneLine)}" title="${e(tooltip)}">
151
- <div class="page-verdict-head">
152
- <span class="page-verdict-badge"><span class="page-verdict-badge-dot" aria-hidden="true">●</span>${e(levelDisplay)}</span>
153
- <span class="page-verdict-text">${e(oneLine)}</span>
154
- </div>
155
- ${metricChips ? `<div class="page-verdict-metrics">${metricChips}</div>` : ''}
156
- </section>`;
157
- }
158
- /**
159
- * Pairwise diff (treatment vs control) bootstrap CI table — populated only when
160
- * --bootstrap was used and at least 2 variants ran. Each row shows whether
161
- * treatment significantly outperformed control on compositeScore mean.
162
- */
163
- export function renderPairwiseDiff(pairs, lang) {
164
- if (!pairs || pairs.length === 0)
165
- return '';
166
- const validPairs = pairs.filter((p) => p.diffBootstrapCI);
167
- if (validPairs.length === 0)
168
- return '';
169
- const rows = validPairs.map((p) => {
170
- const ci = p.diffBootstrapCI;
171
- const sigClass = ci.significant ? 'green' : 'text-muted';
172
- const sigText = ci.significant ? t('bootstrapDiffSignificant', lang) : t('bootstrapDiffNotSignificant', lang);
173
- const estColor = ci.estimate > 0 ? 'var(--green)' : ci.estimate < 0 ? 'var(--red)' : 'var(--text-muted)';
174
- return `<tr>
175
- <td><strong>${e(p.treatment)}</strong> ${lang === 'zh' ? 'vs' : 'vs'} ${e(p.control)}</td>
176
- <td style="text-align:center;color:${estColor}"><strong>${ci.estimate >= 0 ? '+' : ''}${ci.estimate}</strong></td>
177
- <td style="text-align:center;font-size:11px">[${ci.low}, ${ci.high}]</td>
178
- <td style="text-align:center;color:var(--${sigClass})">${sigText}</td>
179
- <td style="text-align:center;color:var(--text-muted);font-size:11px">${ci.samples}</td>
180
- </tr>`;
181
- }).join('');
182
- return `
183
- <h2 style="margin-top:24px">${lang === 'zh' ? '配对对比 (Bootstrap CI)' : 'Pairwise comparison (bootstrap CI)'}</h2>
184
- <p style="font-size:13px;color:var(--text-secondary);margin:4px 0 12px">${lang === 'zh' ? 'control vs treatment 的均值差 95% CI。CI 不含 0 = 显著差异。bootstrap 不假设分布,适合 LLM 序数评分。' : '95% CI on (treatment - control) mean diff. 0 outside CI = significant. Bootstrap is distribution-free, fits ordinal LLM scores.'}</p>
185
- <div class="table-wrap">
186
- <table class="summary-table">
187
- <thead><tr>
188
- <th>${lang === 'zh' ? '对照' : 'Pair'}</th>
189
- <th title="${t('bootstrapDiffLabel', lang)}">${t('bootstrapDiffLabel', lang)}</th>
190
- <th>95% CI</th>
191
- <th>${lang === 'zh' ? '显著性' : 'Significance'}</th>
192
- <th>${lang === 'zh' ? '重采样数' : 'samples'}</th>
193
- </tr></thead>
194
- <tbody>${rows}</tbody>
195
- </table>
196
- </div>`;
197
- }
198
- /**
199
- * Human-gold agreement section — rendered when --gold-dir was used at run time
200
- * and the report has been re-persisted with `meta.humanAgreement`. Shows α
201
- * (primary), bootstrap CI on α, weighted κ, Pearson — and surfaces a
202
- * contamination warning prominently when the gold annotator id overlaps with
203
- * the judge model id.
204
- */
205
- export function renderHumanAgreement(agreement, lang) {
206
- if (!agreement)
207
- return '';
208
- if (agreement.sampleCount === 0)
209
- return '';
210
- const fmt = (x) => Number.isNaN(x) ? 'NaN' : x.toFixed(3);
211
- const a = agreement;
212
- const alphaColor = Number.isNaN(a.alpha)
213
- ? 'var(--text-muted)'
214
- : a.alpha >= 0.8 ? 'var(--green)'
215
- : a.alpha >= 0.667 ? 'var(--yellow)'
216
- : 'var(--red)';
217
- const interpret = lang === 'zh'
218
- ? (Number.isNaN(a.alpha)
219
- ? '评分多样性不足,α 未定义'
220
- : a.alpha >= 0.8 ? '高度一致——结论可放心使用'
221
- : a.alpha >= 0.667 ? '可接受——谨慎结论'
222
- : a.alpha >= 0.4 ? '较弱一致——结论需配合 CI 与人工抽检'
223
- : a.alpha >= 0 ? '偏差较大——排查 rubric / prompt'
224
- : '系统性反向——重新审视判分逻辑')
225
- : (Number.isNaN(a.alpha)
226
- ? 'insufficient rating variance for α'
227
- : a.alpha >= 0.8 ? 'high agreement — conclusions trustworthy'
228
- : a.alpha >= 0.667 ? 'acceptable — conclude cautiously'
229
- : a.alpha >= 0.4 ? 'weak agreement — pair with CI + spot-checks'
230
- : a.alpha >= 0 ? 'large divergence — investigate rubric / prompt'
231
- : 'systematic inversion — judge logic needs review');
232
- const warningBlock = a.contaminationWarning
233
- ? `<div style="margin:8px 0;padding:8px 12px;background:rgba(255, 200, 80, 0.12);border-left:3px solid var(--yellow);font-size:13px"><strong>${lang === 'zh' ? '污染警告' : 'Contamination warning'}:</strong> ${e(a.contaminationWarning)}</div>`
234
- : '';
235
- const missingNote = (a.missingCount > 0 || a.unscoredCount > 0)
236
- ? `<p style="font-size:12px;color:var(--text-muted);margin:4px 0">${lang === 'zh' ? '注: 报告缺' : 'Note: report missing'} ${a.missingCount} ${lang === 'zh' ? '条 sample' : 'samples'}${a.unscoredCount > 0 ? `, ${a.unscoredCount} ${lang === 'zh' ? '条无评分' : 'unscored'}` : ''}</p>`
237
- : '';
238
- return `
239
- <h2 style="margin-top:24px">${lang === 'zh' ? '人工锚点 (Human Gold)' : 'Human gold anchor'}</h2>
240
- <p style="font-size:13px;color:var(--text-secondary);margin:4px 0 12px">${lang === 'zh' ? '对比 LLM 评委分与外部标注的一致性。α 解决"评委对不对",区别于 CI 解决"评委稳不稳"。' : 'Agreement between the LLM judge and external annotations. α addresses "is the judge correct"; the bootstrap CI addresses "is it consistent".'}</p>
241
- ${warningBlock}
242
- <div class="table-wrap">
243
- <table class="summary-table">
244
- <thead><tr>
245
- <th>${lang === 'zh' ? '指标' : 'Metric'}</th>
246
- <th style="text-align:center">${lang === 'zh' ? '值' : 'Value'}</th>
247
- <th style="text-align:center">95% CI</th>
248
- <th>${lang === 'zh' ? '说明' : 'Note'}</th>
249
- </tr></thead>
250
- <tbody>
251
- <tr>
252
- <td><strong>Krippendorff α</strong></td>
253
- <td style="text-align:center;color:${alphaColor}"><strong>${fmt(a.alpha)}</strong></td>
254
- <td style="text-align:center;font-size:11px">[${fmt(a.alphaCI.low)}, ${fmt(a.alphaCI.high)}]</td>
255
- <td style="font-size:12px;color:var(--text-secondary)">${lang === 'zh' ? '主指标,序数加权' : 'primary, ordinal-weighted'}</td>
256
- </tr>
257
- <tr>
258
- <td>${lang === 'zh' ? '加权 κ' : 'weighted κ'}</td>
259
- <td style="text-align:center">${fmt(a.weightedKappa)}</td>
260
- <td style="text-align:center;color:var(--text-muted)">—</td>
261
- <td style="font-size:12px;color:var(--text-secondary)">${lang === 'zh' ? '副指标,平方加权' : 'secondary, quadratic'}</td>
262
- </tr>
263
- <tr>
264
- <td>Pearson r</td>
265
- <td style="text-align:center">${fmt(a.pearson)}</td>
266
- <td style="text-align:center;color:var(--text-muted)">—</td>
267
- <td style="font-size:12px;color:var(--text-secondary)">${lang === 'zh' ? '只看 rank order' : 'rank-order only'}</td>
268
- </tr>
269
- </tbody>
270
- </table>
271
- </div>
272
- <p style="margin:8px 0 0;font-size:13px"><strong>${lang === 'zh' ? '解读' : 'Reading'}:</strong> ${e(interpret)}</p>
273
- <p style="margin:4px 0 0;font-size:12px;color:var(--text-muted)">${lang === 'zh' ? '标注者' : 'annotator'}: ${e(a.goldAnnotator)} (v${e(a.goldVersion)}) · variant: <strong>${e(a.variant)}</strong> · n=${a.sampleCount}</p>
274
- ${missingNote}`;
275
- }
276
- /**
277
- * Saturation curve — answers "did I run enough samples?". Renders an inline
278
- * SVG of mean ± 95% CI shading vs cumulative N, one curve per variant. The
279
- * verdict line below the chart calls out the saturation point (when the curve
280
- * has flattened enough that more samples don't materially shrink the CI).
281
- *
282
- * Hidden when variance is absent (single-run reports) or when there are fewer
283
- * than 2 checkpoints. We keep the chart visible at 2-4 checkpoints (= repeat
284
- * < 5) so users can see the trajectory; the verdict is only computed at
285
- * repeat ≥ 5.
286
- */
287
- export function renderSaturationCurve(saturation, variants, lang) {
288
- if (!saturation)
289
- return '';
290
- const checkpoints = saturation.checkpointSampleCounts;
291
- if (!checkpoints || checkpoints.length < 2)
292
- return '';
293
- // Layout: 600 × 280 SVG, leave 50px on the left for y-axis and 30px at the
294
- // bottom for x-axis labels. Score is the 1-5 Likert range; pin axes to that.
295
- const width = 600, height = 280, padL = 50, padR = 20, padT = 16, padB = 32;
296
- const plotW = width - padL - padR;
297
- const plotH = height - padT - padB;
298
- const xMax = Math.max(...checkpoints);
299
- const xMin = Math.min(...checkpoints);
300
- const yMin = 1, yMax = 5;
301
- const xScale = (x) => padL + (plotW * (x - xMin)) / Math.max(1, xMax - xMin);
302
- const yScale = (y) => padT + plotH - (plotH * (y - yMin)) / (yMax - yMin);
303
- // Per-variant curves and CI ribbons.
304
- const seriesParts = [];
305
- variants.forEach((variant, i) => {
306
- const trace = saturation.perVariant[variant];
307
- if (!trace || trace.length === 0)
308
- return;
309
- const color = COLORS[i % COLORS.length];
310
- // CI ribbon: polygon walking forward along upper bound and back along lower.
311
- const upper = trace.map((p) => `${xScale(p.n).toFixed(1)},${yScale(p.ciHigh).toFixed(1)}`);
312
- const lower = trace.map((p) => `${xScale(p.n).toFixed(1)},${yScale(p.ciLow).toFixed(1)}`).reverse();
313
- const ribbonPoints = [...upper, ...lower].join(' ');
314
- seriesParts.push(`<polygon points="${ribbonPoints}" fill="${color}" fill-opacity="0.15" stroke="none" />`);
315
- // Mean line.
316
- const linePoints = trace.map((p) => `${xScale(p.n).toFixed(1)},${yScale(p.mean).toFixed(1)}`).join(' ');
317
- seriesParts.push(`<polyline points="${linePoints}" fill="none" stroke="${color}" stroke-width="2" />`);
318
- // Mean dots.
319
- for (const p of trace) {
320
- seriesParts.push(`<circle cx="${xScale(p.n).toFixed(1)}" cy="${yScale(p.mean).toFixed(1)}" r="3" fill="${color}" />`);
321
- }
322
- });
323
- // Y-axis ticks at 1..5; x-axis ticks at every checkpoint.
324
- const yTicks = [1, 2, 3, 4, 5].map((y) => `<line x1="${padL}" y1="${yScale(y)}" x2="${width - padR}" y2="${yScale(y)}" stroke="var(--border)" stroke-width="0.5" />
325
- <text x="${padL - 6}" y="${yScale(y) + 4}" font-size="10" text-anchor="end" fill="var(--text-muted)">${y}</text>`).join('');
326
- const xTicks = checkpoints.map((n) => `<text x="${xScale(n)}" y="${height - padB + 14}" font-size="10" text-anchor="middle" fill="var(--text-muted)">${n}</text>`).join('');
327
- // Per-variant verdict block underneath.
328
- const verdictRows = [];
329
- if (saturation.verdicts) {
330
- for (const variant of variants) {
331
- const v = saturation.verdicts[variant];
332
- if (!v)
333
- continue;
334
- const flagColor = v.saturated ? 'var(--green)' : 'var(--yellow)';
335
- const flagText = v.saturated
336
- ? (lang === 'zh' ? `已饱和 (N=${v.atN})` : `saturated at N=${v.atN}`)
337
- : (lang === 'zh' ? '尚未饱和' : 'not yet saturated');
338
- const conf = v.confidence === 'low'
339
- ? `<span style="color:var(--yellow)">⚠ ${lang === 'zh' ? '用例太少,结论参考意义有限' : 'low confidence, interpret cautiously'}</span>`
340
- : '';
341
- verdictRows.push(`<tr><td><strong>${e(variant)}</strong></td>
342
- <td style="color:${flagColor}"><strong>${flagText}</strong></td>
343
- <td style="font-size:12px;color:var(--text-secondary)">${e(v.reason)} ${conf}</td></tr>`);
344
- }
345
- }
346
- const noteHtml = saturation.verdicts
347
- ? `<div class="table-wrap"><table class="summary-table">
348
- <thead><tr>
349
- <th>${lang === 'zh' ? '变体' : 'Variant'}</th>
350
- <th>${lang === 'zh' ? '判定' : 'Verdict'}</th>
351
- <th>${lang === 'zh' ? '依据' : 'Rationale'}</th>
352
- </tr></thead><tbody>${verdictRows.join('')}</tbody></table></div>`
353
- : `<p style="font-size:13px;color:var(--text-muted);margin:8px 0 0">${lang === 'zh' ? '提示:repeat ≥ 5 时才会输出饱和判定。当前数据只够画曲线,不足以下结论。' : 'Note: saturation verdict needs repeat ≥ 5. Current data plots the curve only.'}</p>`;
354
- return `
355
- <h2 style="margin-top:24px">${lang === 'zh' ? '饱和曲线 (Saturation curve)' : 'Saturation curve'}</h2>
356
- <p style="font-size:13px;color:var(--text-secondary);margin:4px 0 12px">${lang === 'zh' ? '随累积评测用例数 N 增长的均值与 95% CI。CI 宽度衰减放缓即饱和——再多评测用例对结论无实质收益。' : 'Mean and 95% CI as cumulative N grows. When CI shrink rate flattens, the evidence has saturated — more samples buy little.'}</p>
357
- <svg viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg" style="width:100%;max-width:${width}px;height:auto;display:block">
358
- ${yTicks}
359
- <text x="${padL - 36}" y="${padT + plotH / 2}" font-size="11" text-anchor="middle" fill="var(--text-muted)" transform="rotate(-90 ${padL - 36} ${padT + plotH / 2})">${lang === 'zh' ? '均值' : 'mean'}</text>
360
- <text x="${padL + plotW / 2}" y="${height - 4}" font-size="11" text-anchor="middle" fill="var(--text-muted)">${lang === 'zh' ? '累积评测用例数 N' : 'cumulative N'}</text>
361
- ${xTicks}
362
- ${seriesParts.join('\n')}
363
- </svg>
364
- ${noteHtml}`;
365
- }
366
- export function renderSummaryCards(variants, summary, lang, variance) {
367
- // 六维对比表格:事实 / 行为 / LLM 评价 / 成本 / 效率 / 稳定性。
368
- // 前三列(事实/行为/LLM 评价)是 task 级原始指标的 variant 聚合;成本/效率同。
369
- // 稳定性是 variant 级散度度量(需 --repeat ≥ 2)。
370
- //
371
- // composite 合成分 (= (fact + behavior + judge) / 3) 从 v0.16 起不再在此表主视觉呈现,
372
- // 仅保留在 report JSON 数据层 + Variance & Significance 表顶层 flat 字段(legacy)。
373
- // 综合分放在第一列, 紧贴 实验分组. 它是 fact/behavior/judge 三层的等权均值,
374
- // 在 omk 内部承担「跨 run 排序 / bootstrap CI / verdict 比较信号」三个角色 ——
375
- // 这里展示 + 表头 ? button 弹 modal 公开计算方式 + 局限, 让用户知情消费。
376
- // 其余六列保持 layer-first (fact / behavior / judge / cost / efficiency / stability)。
377
- const headerCols = [
378
- { key: 'dimFact', label: t('dimFact', lang) },
379
- { key: 'dimBehavior', label: t('dimBehavior', lang) },
380
- { key: 'dimJudge', label: t('dimJudge', lang) },
381
- { key: 'dimCost', label: t('dimCost', lang) },
382
- { key: 'dimEfficiency', label: t('dimEfficiency', lang) },
383
- { key: 'dimStability', label: t('dimStability', lang) },
384
- ];
385
- const scoringModalId = 'guide-scoring';
386
- // v0.30 — 拆 hero(每变体综合分一行)+ heatmap 表(去 composite,加色块)。
387
- // composite 从表里挪出来当 hero,大字号 + delta 跟 baseline 对比;
388
- // 维度细节继续用表,但每格按 score 浅色填充,便于扫读弱维度。
389
- const thead = `<tr><th data-i18n="variants">${t('variants', lang)}</th>${headerCols.map((c) => `<th data-i18n="${c.key}">${c.label}</th>`).join('')}</tr>`;
390
- // 渲染单层分数 cell(事实/行为/LLM 评价通用)— 加 heatmap 浅色背景
391
- function renderLayerCell(varianceMean, summaryValue, detailHtml = '') {
392
- const v = varianceMean ?? summaryValue;
393
- const hasValue = typeof v === 'number' && v > 0;
394
- const heatClass = hasValue
395
- ? (v >= 4 ? 'sm-heat-pass' : v >= 3 ? 'sm-heat-warn' : 'sm-heat-fail')
396
- : '';
397
- const color = hasValue
398
- ? (v >= 4 ? 'var(--green)' : v >= 3 ? 'var(--yellow)' : 'var(--red)')
399
- : 'var(--text-muted)';
400
- const display = hasValue ? v.toFixed(2) : '—';
401
- return `<td class="summary-cell ${heatClass}"><div class="summary-value summary-value-primary" style="color:${color}">${display}</div>${detailHtml}</td>`;
402
- }
403
- const rows = variants.map((v, i) => {
404
- const s = summary[v] || {};
405
- const vd = variance?.perVariant[v];
406
- const color = COLORS[i % COLORS.length];
407
- // 事实层 cell(含事实验证率 detail,如果有)
408
- const factDetailParts = [];
409
- if (s.avgFactVerifiedRate != null) {
410
- const pct = Math.round(s.avgFactVerifiedRate * 100);
411
- factDetailParts.push(`${lang === 'zh' ? '验证率' : 'Verified'} ${pct}%`);
412
- }
413
- const factDetail = factDetailParts.length > 0 ? `<div class="summary-detail">${factDetailParts.join(' · ')}</div>` : '';
414
- const factCell = renderLayerCell(vd?.byLayer?.fact?.mean, s.avgFactScore, factDetail);
415
- // 行为层 cell(暂无 detail,后续可按工具调用成功率之类扩展)
416
- const behaviorCell = renderLayerCell(vd?.byLayer?.behavior?.mean, s.avgBehaviorScore);
417
- // LLM 评价层 cell
418
- const judgeCell = renderLayerCell(vd?.byLayer?.judge?.mean, s.avgJudgeScore);
419
- // Cost — only show execution cost (judge cost is tool overhead, not skill cost)
420
- // execCostReported === false 时(如 codex executor)显示「—」并 tooltip 解释,
421
- // 跟"真的花了 $0(全 cached / 短 prompt)"区分开。
422
- const execCost = s.totalExecCostUSD || 0;
423
- const costReported = s.execCostReported !== false;
424
- const hasCost = execCost > 0 || (s.avgTotalTokens || 0) > 0;
425
- const costUnreportedTooltip = lang === 'zh'
426
- ? 'executor 不报 USD 成本(如 codex CLI),无法估算'
427
- : 'executor does not report USD cost (e.g. codex CLI); not measurable';
428
- const costCell = hasCost
429
- ? (costReported
430
- ? `<td class="summary-cell"><div class="summary-value">${fmtCost(execCost)}</div><div class="summary-detail">${fmtNum(s.avgTotalTokens)} tokens/${t('tokPerReq', lang).replace('tokens/', '')}</div></td>`
431
- : `<td class="summary-cell" title="${e(costUnreportedTooltip)}"><div class="summary-value" style="color:var(--text-muted)">${fmtCost(0, false)}</div><div class="summary-detail">${fmtNum(s.avgTotalTokens)} tokens/${t('tokPerReq', lang).replace('tokens/', '')}</div></td>`)
432
- : `<td class="summary-cell"><span style="color:var(--text-muted)">N/A</span></td>`;
433
- // Efficiency
434
- const effDetails = [];
435
- const displayTurns = s.avgFullNumTurns ?? s.avgNumTurns;
436
- if ((displayTurns || 0) > 0)
437
- effDetails.push(`${displayTurns} ${t('turnsPerReq', lang)}`);
438
- if (s.avgToolCalls != null && s.avgToolCalls > 0) {
439
- const srPct = s.toolSuccessRate != null ? ` (${(s.toolSuccessRate * 100).toFixed(0)}% OK)` : '';
440
- effDetails.push(`${s.avgToolCalls} tools/req${srPct}`);
441
- }
442
- const totalDurationMs = (s.avgDurationMs || 0) * (s.successCount || 0);
443
- if (totalDurationMs > 0)
444
- effDetails.push(`${lang === 'zh' ? '总计' : 'total'} ${fmtDuration(totalDurationMs)}`);
445
- const effDetail = effDetails.length > 0 ? `<div class="summary-detail">${effDetails.join(' · ')}</div>` : '';
446
- const avgLabel = lang === 'zh' ? '次' : 'req';
447
- const effCell = `<td class="summary-cell"><div class="summary-value">${fmtDuration(s.avgDurationMs)}<span class="summary-unit">/${avgLabel}</span></div>${effDetail}</td>`;
448
- // Stability — 多次运行分数一致性(test-retest reliability),统计学定义的稳定性。
449
- // 主视觉:白话定性词 + ±σ 直观量级,让读者一眼判断。
450
- // 副区:CV (变异系数 = σ / mean) 分两行展示 + 95% CI(置信区间)。
451
- // 无 --repeat(variance 缺失) 时主值显示 "—" + 明示需多跑,不虚报 100%——
452
- // 符合 omk 叙事底线"诚实交代测不到什么"。
453
- //
454
- // 跨用例 min~max range 不是稳定性(反映用例难度差异,非 variant 波动),已从此列移除。
455
- // 成功率 ≠ 稳定性(执行完成率,不是分数一致性),降级到 < 100% 时的副区 alert。
456
- const total = s.totalSamples || 0;
457
- const successCount = s.successCount || 0;
458
- const errorCount = s.errorCount || 0;
459
- const successRate = total > 0 ? Number((successCount / total * 100).toFixed(1)) : 0;
460
- const stabDetails = [];
461
- let stabValue;
462
- let stabColor;
463
- // CV = σ / mean,当 mean 过小(接近 0)时 CV 发散,数值无参考价值——1-5 分数量纲下
464
- // mean < 0.5 已属全灭场景,直接降级显示"—"。负 mean(理论上不会出现)也走降级。
465
- if (vd && typeof vd.stddev === 'number' && typeof vd.mean === 'number' && vd.mean >= 0.5) {
466
- const cv = Math.abs(vd.stddev / vd.mean);
467
- const cvPct = cv * 100;
468
- const sigma = vd.stddev;
469
- // 主值用白话定性 + ±σ 直观量级,让非统计背景读者一眼判断。
470
- // CV 和 CI 下沉到副区,给懂的人细看。阈值面向 1-5 分数量纲的经验值。
471
- let label;
472
- if (cvPct < 5) {
473
- label = lang === 'zh' ? '稳定' : 'Stable';
474
- stabColor = 'var(--green)';
475
- }
476
- else if (cvPct < 15) {
477
- label = lang === 'zh' ? '较稳' : 'Moderate';
478
- stabColor = 'var(--yellow)';
479
- }
480
- else {
481
- label = lang === 'zh' ? '波动大' : 'Variable';
482
- stabColor = 'var(--red)';
483
- }
484
- stabValue = `${label} · ±${fmtNum(sigma, 2)}`;
485
- const ciLo = fmtNum(vd.lower, 2);
486
- const ciHi = fmtNum(vd.upper, 2);
487
- stabDetails.push(`CV ${cvPct.toFixed(1)}% · 95% CI [${ciLo}, ${ciHi}]`);
488
- }
489
- else {
490
- // No cross-run data → make the gap loud, not silent. 之前用灰色 "—" 让单轮
491
- // 报告读者误以为"无显示 = 没问题",实际是 omk 测不到这个维度。改红 +
492
- // "未测量" 字样让缺失可见 — 鼓励用户加 --repeat ≥ 2 而不是默默 ship。
493
- stabValue = lang === 'zh' ? '⚠ 未测量' : '⚠ Not measured';
494
- stabColor = 'var(--red)';
495
- stabDetails.push(`<span style="color:var(--red)">${lang === 'zh' ? '单轮评测,加 --repeat ≥ 2 才能测 CV' : 'single-run; needs --repeat ≥ 2 to measure CV'}</span>`);
496
- }
497
- // Execution-completion alerts:success rate < 100% 时降级到此处,避免和"稳定性"语义混淆。
498
- if (errorCount > 0) {
499
- stabDetails.unshift(`<span style="color:var(--red)">${successRate}% ${lang === 'zh' ? '完成率' : 'completed'} · ${errorCount} ${t('errors', lang)}</span>`);
500
- }
501
- // 正常情况副区只一行:`CV X.X% · 95% CI [...]`。
502
- // 有 alert(成功率 < 100%) 时把 alert 放第一行、CV+CI 放第二行,让异常信息第一眼看到。
503
- const stabDetail = stabDetails.length > 0
504
- ? stabDetails.map((d) => `<div class="summary-detail">${d}</div>`).join('')
505
- : '';
506
- const stabCell = `<td class="summary-cell"><div class="summary-value" style="color:${stabColor}">${stabValue}</div>${stabDetail}</td>`;
507
- return `<tr><td style="border-left:3px solid ${color};padding-left:12px"><strong>${e(v)}</strong></td>${factCell}${behaviorCell}${judgeCell}${costCell}${effCell}${stabCell}</tr>`;
508
- }).join('');
509
- // ──────────── Hero 行:每 variant 一行,综合分大字号 + delta vs baseline ────────────
510
- // 每变体一行,左 variant 名 + 颜色条;中 大字号 composite;右 delta(vs variants[0])
511
- const baselineComposite = summary[variants[0]]?.avgCompositeScore;
512
- const heroRows = variants.map((v, i) => {
513
- const s = summary[v] || {};
514
- const composite = s.avgCompositeScore;
515
- const compositeHasValue = typeof composite === 'number' && composite > 0;
516
- const compositeColor = compositeHasValue
517
- ? (composite >= 4 ? 'var(--green)' : composite >= 3 ? 'var(--yellow)' : 'var(--red)')
518
- : 'var(--text-muted)';
519
- const compositeDisplay = compositeHasValue ? composite.toFixed(2) : '—';
520
- const color = COLORS[i % COLORS.length];
521
- let deltaHtml = '';
522
- if (i > 0 && compositeHasValue && typeof baselineComposite === 'number' && baselineComposite > 0) {
523
- const diff = composite - baselineComposite;
524
- if (Math.abs(diff) >= 0.01) {
525
- const sign = diff > 0 ? '+' : '';
526
- const dColor = diff > 0 ? 'var(--green)' : 'var(--red)';
527
- const arrow = diff > 0 ? '↑' : '↓';
528
- deltaHtml = `<span class="sm-hero-delta" style="color:${dColor}">${sign}${diff.toFixed(2)} ${arrow}</span>`;
529
- }
530
- }
531
- return `<div class="sm-hero" style="border-left:3px solid ${color}">
532
- <div class="sm-hero-name">${e(v)}</div>
533
- <div class="sm-hero-score" style="color:${compositeColor}">${compositeDisplay}<span class="sm-hero-unit">/ 5</span></div>
534
- ${deltaHtml || '<span class="sm-hero-delta-placeholder"></span>'}
535
- </div>`;
536
- }).join('');
537
- const heroBlock = `<div class="sm-hero-list">
538
- <div class="sm-hero-h">
539
- ${lang === 'zh' ? '综合分' : 'Composite'}
540
- <button type="button" class="hint-btn" onclick="openModal('${scoringModalId}')" aria-label="${e(lang === 'zh' ? '综合分怎么算的?' : 'How is composite computed?')}" aria-haspopup="dialog">?</button>
541
- </div>
542
- ${heroRows}
543
- </div>`;
544
- const guideModalId = 'guide-six-dims';
545
- const guideTitle = lang === 'zh' ? '如何阅读六维对比?' : 'How to read this six-dimension comparison?';
546
- const guideIntro = lang === 'zh'
547
- ? '每行是一个实验分组(Variant),六列分别衡量不同维度:'
548
- : 'Each row is an experiment variant. Six columns measure independent dimensions:';
549
- const icon = (emoji) => `<span aria-hidden="true">${emoji}</span>`;
550
- // 维度分隔加粗(border-top 2px),让六维的视觉边界更明显。sub 缩进从 28 收到 22。
551
- const dim = 'style="padding:12px 0 4px;border-top:2px solid var(--border);color:var(--text-primary);font-weight:600"';
552
- const dimDesc = 'style="padding:12px 0 4px;border-top:2px solid var(--border);color:var(--text-secondary)"';
553
- const dimFirst = 'style="padding:4px 0 4px;color:var(--text-primary);font-weight:600"';
554
- const dimFirstDesc = 'style="padding:4px 0 4px;color:var(--text-secondary)"';
555
- const sub = 'style="padding:2px 0 2px 22px;font-size:12px;color:var(--text-secondary);font-weight:500"';
556
- const subDesc = 'style="padding:2px 0;font-size:12px;color:var(--text-muted)"';
557
- const guideRows = lang === 'zh' ? `
558
- <tr><td ${dimFirst}>${icon('📋')} <strong>事实</strong></td><td ${dimFirstDesc}>任务执行模型的输出说得对不对(事实声明层面)。靠规则断言判:关键词是否出现、JSON 格式是否合法等,答错了直接不给分。</td></tr>
559
- <tr><td ${dim}>${icon('🛠️')} <strong>行为</strong></td><td ${dimDesc}>任务执行模型做事的过程有没有走对路。靠规则断言判:该调的工具有没有调、有没有超过轮次/成本上限。</td></tr>
560
- <tr><td ${dim}>${icon('💬')} <strong>LLM 评价</strong></td><td ${dimDesc}>请一个 LLM 当评委,让它读任务执行模型的输出内容,按预先写好的评分规则(英文叫 rubric)打个 1-5 分。主观但能抓到规则断言判不了的"整体好不好"——比如回答是否清晰、有没有答非所问。</td></tr>
561
- <tr><td ${dim}>${icon('💰')} <strong>执行成本</strong></td><td ${dimDesc}>任务执行模型跑这次评测花了多少 API 调用费。评委模型成本单独作为评测开销记录,不算进 skill 自身成本。</td></tr>
562
- <tr><td ${dim}>${icon('⚡')} <strong>效率</strong></td><td ${dimDesc}>一次评测平均跑多久;附带轮次数和工具调用次数。</td></tr>
563
- <tr><td ${dim}>${icon('🛡️')} <strong>稳定性</strong></td><td ${dimDesc}>同一份测试跑很多次,分数抖不抖。抖得越少越稳定。<strong>跑一次看不出稳定性</strong>——至少要 <code>--repeat ≥ 2</code>,不然显示"—"。</td></tr>
564
- <tr><td ${sub}>稳定 / 较稳 / 波动大</td><td ${subDesc}>分数波动比例 &lt;5% = 稳定 · 5~15% = 一般 · &gt;15% = 波动大</td></tr>
565
- <tr><td ${sub}>±σ</td><td ${subDesc}>每次跑出的分数,大概在平均分上下浮动多少。1-5 分数里 ±0.05 几乎不抖、±0.5 抖得很厉害</td></tr>
566
- <tr><td ${sub}>CV</td><td ${subDesc}>分数抖动幅度占平均分的比例(例:CV 2% = 分数波动大约是平均分的 2%)</td></tr>
567
- <tr><td ${sub}>95% CI</td><td ${subDesc}>如果跑无数次求平均,真实平均分有 95% 概率落在这个范围里——范围越窄,这次测出的均值越可信</td></tr>
568
- ` : `
569
- <tr><td ${dimFirst}>${icon('📋')} <strong>Fact</strong></td><td ${dimFirstDesc}>Whether the task execution model's output is factually correct. Checked by rule-based assertions — keyword matches, JSON schema validity, etc. Wrong = zero.</td></tr>
570
- <tr><td ${dim}>${icon('🛠️')} <strong>Behavior</strong></td><td ${dimDesc}>Whether the task execution model followed the right process. Checked by rule-based assertions — did it call the expected tools, stay within turn/cost limits.</td></tr>
571
- <tr><td ${dim}>${icon('💬')} <strong>LLM judge</strong></td><td ${dimDesc}>A separate LLM acts as judge: it reads the task execution model's output and scores it 1-5 against a predefined rubric. Subjective, but catches "overall feel" that rule-based assertions miss — e.g., whether the answer is clear, whether it's on-topic.</td></tr>
572
- <tr><td ${dim}>${icon('💰')} <strong>Exec cost</strong></td><td ${dimDesc}>API cost for the task execution model. Judge model cost is tracked as evaluation overhead, not skill cost.</td></tr>
573
- <tr><td ${dim}>${icon('⚡')} <strong>Efficiency</strong></td><td ${dimDesc}>Average time per evaluation, plus turn counts and tool call stats.</td></tr>
574
- <tr><td ${dim}>${icon('🛡️')} <strong>Stability</strong></td><td ${dimDesc}>How much the score swings when you repeat the same test. Less swing = more stable. <strong>You can't measure stability from a single run</strong> — need <code>--repeat ≥ 2</code>, otherwise shows "—".</td></tr>
575
- <tr><td ${sub}>Stable / Moderate / Variable</td><td ${subDesc}>Score swing as % of mean: &lt;5% = Stable · 5~15% = Moderate · &gt;15% = Variable</td></tr>
576
- <tr><td ${sub}>±σ</td><td ${subDesc}>How much each run's score typically swings around the mean. On a 1-5 scale, ±0.05 barely moves, ±0.5 swings a lot</td></tr>
577
- <tr><td ${sub}>CV</td><td ${subDesc}>Score swing as a percentage of the mean (e.g., CV 2% = swings are about 2% of the mean)</td></tr>
578
- <tr><td ${sub}>95% CI</td><td ${subDesc}>If you ran infinitely many times, the true mean has a 95% chance of falling in this range — narrower = you can trust the measured mean more</td></tr>
579
- `;
580
- const scoringModalHtml = renderScoringModal(scoringModalId, lang);
581
- return `
582
- <h2 style="display:flex;align-items:center;gap:4px">${lang === 'zh' ? '六维对比' : 'Six-Dimension Comparison'} <button type="button" class="hint-btn" onclick="openModal('${guideModalId}')" aria-label="${e(guideTitle)}" aria-haspopup="dialog">?</button></h2>
583
- <div id="${guideModalId}" class="modal-overlay" role="dialog" aria-modal="true" aria-labelledby="${guideModalId}-title" onclick="if(event.target===this)closeModal('${guideModalId}')">
584
- <div class="modal-content">
585
- <div class="modal-header">
586
- <strong id="${guideModalId}-title" style="font-size:1rem">${e(guideTitle)}</strong>
587
- <button type="button" class="modal-close" onclick="closeModal('${guideModalId}')" aria-label="${lang === 'zh' ? '关闭' : 'Close'}">✕</button>
588
- </div>
589
- <p style="font-size:13px;color:var(--text-secondary);margin:4px 0 16px">${e(guideIntro)}</p>
590
- <table class="modal-table"><tbody>${guideRows}</tbody></table>
591
- </div>
592
- </div>
593
- ${scoringModalHtml}
594
- ${heroBlock}
595
- <div class="table-wrap">
596
- <table class="summary-table">
597
- <thead>${thead}</thead>
598
- <tbody>${rows}</tbody>
599
- </table>
600
- </div>
601
- <style>
602
- /* 六维对比 — Hero 区(每 variant 综合分 + delta) */
603
- .sm-hero-list { display:flex;flex-direction:column;gap:8px;margin:14px 0 18px }
604
- .sm-hero-h { font-size:13px;color:var(--text-muted);margin-bottom:4px;letter-spacing:0.02em }
605
- .sm-hero { display:flex;align-items:baseline;gap:16px;padding:14px 18px;background:var(--bg-surface);border-radius:var(--radius);box-shadow:var(--shadow-sm) }
606
- .sm-hero-name { flex:1;font-size:14.5px;font-weight:500;color:var(--text-primary) }
607
- .sm-hero-score { font-size:28px;font-weight:600;font-variant-numeric:tabular-nums;line-height:1 }
608
- .sm-hero-unit { font-size:13px;color:var(--text-muted);margin-left:4px;font-weight:400 }
609
- .sm-hero-delta { font-size:14px;font-weight:600;font-variant-numeric:tabular-nums;font-family:"SF Mono",Menlo,monospace;min-width:60px;text-align:right }
610
- .sm-hero-delta-placeholder { display:inline-block;min-width:60px }
611
- /* 六维 heatmap — 每格按分数浅色填充,便于远看识别弱维度 */
612
- .sm-heat-pass { background:linear-gradient(0deg,var(--green-bg),var(--green-bg)) }
613
- .sm-heat-warn { background:linear-gradient(0deg,var(--yellow-bg),var(--yellow-bg)) }
614
- .sm-heat-fail { background:linear-gradient(0deg,var(--red-bg),var(--red-bg)) }
615
- </style>`;
616
- }
617
- /**
618
- * 评分说明 modal — 公开 composite 计算方式 + 局限,所有展示综合分的地方都通向同一份说明
619
- * (六维对比表头 / 列表页 score 列 / 趋势页 chart caption)。
620
- *
621
- * 测量学诚实性 ≠ 藏起来怕用户误用,而是展示 + 说清楚边界。这个 modal 是简版 (~10 行),
622
- * 完整推导 (五层评分管道架构 / ratioToScore 公式 / 多层 gate 与 composite 关系)
623
- * 在 docs/zh/scoring.md。
624
- */
625
- export function renderScoringModal(id, lang) {
626
- const title = lang === 'zh' ? '综合分怎么算的?' : 'How is composite computed?';
627
- const intro = lang === 'zh'
628
- ? '综合分(composite)= 等权均值({事实, 行为, LLM 评价})。1-5 制。'
629
- : 'Composite = unweighted mean of {fact, behavior, LLM judge}. Scale 1-5.';
630
- const sectionHead = (text) => `<tr><td colspan="2" style="padding:14px 0 6px;color:var(--text-primary);font-weight:600;font-size:13px;border-top:1px solid var(--border)">${e(text)}</td></tr>`;
631
- const row = (label, body) => `<tr><td style="padding:6px 12px 6px 0;color:var(--text-secondary);font-size:12px;font-weight:500;white-space:nowrap;vertical-align:top;width:120px">${e(label)}</td><td style="padding:6px 0;color:var(--text-secondary);font-size:12px;line-height:1.6">${body}</td></tr>`;
632
- const calcRows = lang === 'zh' ? [
633
- row('事实 / 行为', '断言通过率经 <code>1 + 通过率 × 4</code> 线性映射到 1-5 制'),
634
- row('LLM 评价', '评委模型直接给 1-5 分'),
635
- row('缺失某层', '自动降维(参与计算的层取均值)'),
636
- ] : [
637
- row('Fact / Behavior', 'Assertion pass-rate mapped to 1-5 via <code>1 + ratio × 4</code>'),
638
- row('LLM judge', 'Judge model returns 1-5 score directly'),
639
- row('Missing layer', 'Auto-collapse (mean over present layers)'),
640
- ];
641
- const limitRows = lang === 'zh' ? [
642
- row('① 等权聚合', '三层各 1/3 权重不是从需求 derive 的,无显式构造效度论证'),
643
- row('② 量尺不一致', '事实是 binary 通过率 stretch 到 1-5;评委是真序数评分。直接均值违反量表理论(measurement scale homogeneity)'),
644
- row('③ 缺失自动降维', '不同 variant / skill 配的层数不同时,综合分数字相同但 construct 不同,<strong>不可机械跨 variant / 跨 skill 比较</strong>'),
645
- ] : [
646
- row('① Equal-weight aggregation', 'The 1/3 weight per layer is ad hoc, not derived from stakeholder needs; no explicit construct validity argument'),
647
- row('② Mixed scales added directly', 'Fact is binary pass-rate stretched to 1-5; judge is true ordinal. Direct mean violates measurement scale homogeneity'),
648
- row('③ Auto-collapse on missing', 'When variants/skills have different layer coverage, composite numbers look same but represent different constructs — <strong>not mechanically comparable across variants/skills</strong>'),
649
- ];
650
- const usageRows = lang === 'zh' ? [
651
- row('✓ 适用', '同一份 eval-samples 上 A/B 比较,看「分差 + bootstrap CI + 三层独立 gate」联合判断'),
652
- row('✗ 不适用', '当作 absolute psychometric measure(说「这 skill 4.28/5 分」)'),
653
- ] : [
654
- row('✓ Use for', 'A/B comparison on same eval-samples — read "diff + bootstrap CI + per-layer gate" jointly'),
655
- row('✗ Don\'t use as', 'Absolute psychometric measure (claiming "this skill is 4.28/5")'),
656
- ];
657
- const calcSection = lang === 'zh' ? '怎么算的' : 'How it\'s computed';
658
- const limitSection = lang === 'zh' ? '局限(直白说)' : 'Limitations (frankly)';
659
- const usageSection = lang === 'zh' ? '推荐用法' : 'Recommended usage';
660
- const docsLink = lang === 'zh'
661
- ? '完整推导 / 五层评分管道架构 / 多层 gate 与综合分的关系:<a href="https://github.com/lizhiyao/oh-my-knowledge/blob/main/docs/zh/scoring.md" target="_blank" rel="noopener">docs/zh/scoring.md</a>'
662
- : 'Full derivation / five-layer scoring pipeline / multi-layer gate & composite relationship: <a href="https://github.com/lizhiyao/oh-my-knowledge/blob/main/docs/zh/scoring.md" target="_blank" rel="noopener">docs/zh/scoring.md</a>';
663
- return `<div id="${id}" class="modal-overlay" role="dialog" aria-modal="true" aria-labelledby="${id}-title" onclick="if(event.target===this)closeModal('${id}')">
664
- <div class="modal-content">
665
- <div class="modal-header">
666
- <strong id="${id}-title" style="font-size:1rem">${e(title)}</strong>
667
- <button type="button" class="modal-close" onclick="closeModal('${id}')" aria-label="${lang === 'zh' ? '关闭' : 'Close'}">✕</button>
668
- </div>
669
- <p style="font-size:13px;color:var(--text-secondary);margin:4px 0 12px">${e(intro)}</p>
670
- <table class="modal-table"><tbody>
671
- ${sectionHead(calcSection)}
672
- ${calcRows.join('')}
673
- ${sectionHead(limitSection)}
674
- ${limitRows.join('')}
675
- ${sectionHead(usageSection)}
676
- ${usageRows.join('')}
677
- </tbody></table>
678
- <p style="font-size:11px;color:var(--text-muted);margin:14px 0 0;line-height:1.6">${docsLink}</p>
679
- </div>
680
- </div>`;
681
- }
682
- /**
683
- * Cross-sample inter-judge agreement table — only renders when at least one variant
684
- * has multi-judge ensemble data. This is the v0.20.2 "blog headline" view: shows
685
- * Pearson + MAD across the whole sample set, the metric that refutes "Claude judge
686
- * Claude same-modality bias".
687
- */
688
- export function renderJudgeAgreementBlock(variants, summary, lang) {
689
- const variantsWithEnsemble = variants.filter((v) => summary[v]?.judgeAgreement);
690
- if (variantsWithEnsemble.length === 0)
691
- return '';
692
- const rows = variantsWithEnsemble.map((v) => {
693
- const s = summary[v];
694
- const ag = s.judgeAgreement;
695
- const judgeList = (s.judgeModels || []).map((j) => `<code>${e(`${j.executor}:${j.model}`)}</code>`).join(', ');
696
- const pearsonCell = ag.pearson != null
697
- ? `<span title="${t('pearsonDesc', lang)}" style="color:${ag.pearson >= 0.7 ? 'var(--green)' : ag.pearson >= 0.4 ? 'var(--yellow)' : 'var(--red)'}"><strong>${ag.pearson}</strong></span>`
698
- : `<span style="color:var(--text-muted)">—</span>`;
699
- const madCell = `<span title="${t('madDesc', lang)}" style="color:${ag.meanAbsDiff < 0.5 ? 'var(--green)' : ag.meanAbsDiff < 1.5 ? 'var(--yellow)' : 'var(--red)'}"><strong>${ag.meanAbsDiff}</strong></span>`;
700
- return `<tr>
701
- <td><strong>${e(v)}</strong></td>
702
- <td style="font-size:11px;color:var(--text-muted)">${judgeList}</td>
703
- <td style="text-align:center">${pearsonCell}</td>
704
- <td style="text-align:center">${madCell}</td>
705
- <td style="text-align:center;color:var(--text-muted)">${ag.sampleCount}</td>
706
- <td style="text-align:center;color:var(--text-muted)">${ag.pairCount}</td>
707
- </tr>`;
708
- }).join('');
709
- return `
710
- <h2 style="margin-top:24px">${t('agreementHeader', lang)}</h2>
711
- <p style="font-size:13px;color:var(--text-secondary);margin:4px 0 12px">${t('agreementDesc', lang)}</p>
712
- <div class="table-wrap">
713
- <table class="summary-table">
714
- <thead><tr>
715
- <th>${t('variants', lang)}</th>
716
- <th>${t('judgeModelsLabel', lang)}</th>
717
- <th title="${t('pearsonDesc', lang)}">${t('pearsonLabel', lang)}</th>
718
- <th title="${t('madDesc', lang)}">${t('madLabel', lang)}</th>
719
- <th>${lang === 'zh' ? '评测用例数' : 'Samples'}</th>
720
- <th>${lang === 'zh' ? 'Judge 对数' : 'Pairs'}</th>
721
- </tr></thead>
722
- <tbody>${rows}</tbody>
723
- </table>
724
- </div>`;
725
- }
726
- function badgeIcon(status) {
727
- switch (status) {
728
- case 'pass': return '✓';
729
- case 'warn': return '⚠';
730
- case 'fail': return '✗';
731
- case 'skip': return '—';
732
- }
733
- }
734
- function computeJudgeAgreementBadge(variants, summary, lang) {
735
- const treatment = variants[1] ?? variants[0];
736
- const ag = summary[treatment]?.judgeAgreement;
737
- if (!ag || ag.pearson == null)
738
- return null;
739
- const p = ag.pearson;
740
- const status = p >= 0.7 ? 'pass' : p >= 0.4 ? 'warn' : 'fail';
741
- const verdict = status === 'pass'
742
- ? (lang === 'zh' ? '评委一致' : 'judges agree')
743
- : status === 'warn'
744
- ? (lang === 'zh' ? '评委偏弱一致' : 'judges weakly agree')
745
- : (lang === 'zh' ? '评委分歧大' : 'judges diverge');
746
- return {
747
- key: 'judge',
748
- label: verdict,
749
- status,
750
- detail: lang === 'zh' ? `Pearson ${p}(≥0.7 一致 / 0.4-0.7 偏弱 / <0.4 分歧)` : `Pearson ${p} (≥0.7 agree / 0.4-0.7 weak / <0.4 diverge)`,
751
- };
752
- }
753
- function computeSignificanceBadge(report, lang) {
754
- const ci = report.meta?.pairComparisons?.[0]?.diffBootstrapCI;
755
- if (!ci)
756
- return null;
757
- const status = ci.significant ? 'pass' : 'warn';
758
- const label = ci.significant
759
- ? (lang === 'zh' ? '差异显著' : 'significant')
760
- : (lang === 'zh' ? '差异不显著' : 'not significant');
761
- return {
762
- key: 'sig',
763
- label,
764
- status,
765
- detail: lang === 'zh' ? `bootstrap 95% CI [${ci.low}, ${ci.high}]${ci.significant ? '(不含 0)' : '(跨过 0)'}` : `Bootstrap 95% CI [${ci.low}, ${ci.high}]${ci.significant ? ' (excludes 0)' : ' (spans 0)'}`,
766
- };
767
- }
768
- function computeSaturationBadge(report, variants, lang) {
769
- const sat = report.variance?.saturation?.verdicts;
770
- if (!sat)
771
- return null;
772
- const treatment = variants[1] ?? variants[0];
773
- const v = sat[treatment];
774
- if (!v)
775
- return null;
776
- const status = v.saturated
777
- ? (v.confidence === 'high' ? 'pass' : 'warn')
778
- : 'warn';
779
- const label = v.saturated
780
- ? (lang === 'zh' ? '已饱和' : 'saturated')
781
- : (lang === 'zh' ? '未饱和' : 'not saturated');
782
- return {
783
- key: 'sat',
784
- label,
785
- status,
786
- detail: lang === 'zh'
787
- ? `${v.saturated ? `于 N=${v.atN ?? '?'} 饱和` : '尚未饱和'}(${v.confidence} confidence)`
788
- : `${v.saturated ? `saturates at N=${v.atN ?? '?'}` : 'not yet saturated'} (${v.confidence} confidence)`,
789
- };
790
- }
791
- function computeHumanAlignmentBadge(report, lang) {
792
- const a = report.meta?.humanAgreement;
793
- if (!a || a.sampleCount === 0 || Number.isNaN(a.alpha))
794
- return null;
795
- const status = a.alpha >= 0.667 ? 'pass' : a.alpha >= 0.4 ? 'warn' : 'fail';
796
- const label = status === 'pass'
797
- ? (lang === 'zh' ? '人工对齐' : 'aligns with humans')
798
- : status === 'warn'
799
- ? (lang === 'zh' ? '人工对齐偏弱' : 'weakly aligns')
800
- : (lang === 'zh' ? '与人工分歧' : 'diverges from humans');
801
- return {
802
- key: 'human',
803
- label,
804
- status,
805
- detail: lang === 'zh' ? `Krippendorff α=${a.alpha.toFixed(3)} vs ${a.goldAnnotator}` : `Krippendorff α=${a.alpha.toFixed(3)} vs ${a.goldAnnotator}`,
806
- };
807
- }
808
- export function renderMethodologyAudit(report, variants, summary, lang) {
809
- const judgeBlock = renderJudgeAgreementBlock(variants, summary, lang);
810
- const pairwiseBlock = renderPairwiseDiff(report.meta?.pairComparisons, lang);
811
- const humanBlock = renderHumanAgreement(report.meta?.humanAgreement, lang);
812
- const saturationBlock = renderSaturationCurve(report.variance?.saturation, variants, lang);
813
- if (!judgeBlock && !pairwiseBlock && !humanBlock && !saturationBlock)
814
- return '';
815
- const badges = [
816
- computeJudgeAgreementBadge(variants, summary, lang),
817
- computeSignificanceBadge(report, lang),
818
- computeSaturationBadge(report, variants, lang),
819
- computeHumanAlignmentBadge(report, lang),
820
- ].filter((b) => b !== null);
821
- // 默认折叠;只有 ✗ fail (红色 badge) 才强制展开 — summary 行已经显示
822
- // ⚠ warn 状态, 用户能自决是否展开; 但 fail 是危险信号必须让用户看证据。
823
- const hasFailure = badges.some((b) => b.status === 'fail');
824
- const openAttr = hasFailure ? ' open' : '';
825
- const summaryLabel = lang === 'zh' ? '测评可信度' : 'Reliability check';
826
- const summaryHint = lang === 'zh' ? '点击展开看支撑证据(评委一致 / 差异显著 / 饱和度 / 人工对齐)' : 'click to expand evidence (judge agreement / significance / saturation / human alignment)';
827
- const badgesHtml = badges.length > 0
828
- ? `<span class="methodology-badges">${badges.map((b) => `<span class="methodology-badge methodology-badge-${b.status}" title="${e(b.detail)}">${badgeIcon(b.status)} ${e(b.label)}</span>`).join('')}</span>`
829
- : '';
830
- const innerSections = [judgeBlock, pairwiseBlock, humanBlock, saturationBlock].filter(Boolean).join('');
831
- return `<details class="methodology-audit"${openAttr}>
832
- <summary><span class="methodology-summary-label">${e(summaryLabel)}</span>${badgesHtml}<span class="methodology-summary-hint">${e(summaryHint)}</span></summary>
833
- <div class="methodology-body">${innerSections}</div>
834
- </details>`;
835
- }
836
- function buildDiagnostic(metric, cfg, winner, lang) {
837
- const es = metric.effectSize;
838
- if (!es || es.primary === 'none') {
839
- return {
840
- icon: '—',
841
- text: lang === 'zh' ? '数据不足,无法判断' : 'insufficient data',
842
- color: 'var(--text-muted)',
843
- };
844
- }
845
- const isStrong = es.magnitude === 'medium' || es.magnitude === 'large';
846
- const strongLabelZh = es.magnitude === 'large' ? '大' : '中';
847
- const strongLabelEn = es.magnitude;
848
- // Direction word bound into the diagnostic text so readers cannot misread
849
- // green ✓ as "good for v2" when it actually means "v1 is significantly
850
- // cheaper / faster / higher-quality". Each metric uses its own natural verb.
851
- const winnerPhrase = winner
852
- ? (lang === 'zh'
853
- ? `${winner} 显著${cfg.winnerWordZh}`
854
- : `${winner} significantly ${cfg.winnerWordEn.replace(' by', '')}`)
855
- : '';
856
- if (metric.significant && isStrong) {
857
- return {
858
- icon: '✓',
859
- text: lang === 'zh'
860
- ? `${winnerPhrase}(${strongLabelZh}差异)`
861
- : `${winnerPhrase} (${strongLabelEn} effect)`,
862
- color: 'var(--green)',
863
- };
864
- }
865
- if (metric.significant && !isStrong) {
866
- return {
867
- icon: '⚠',
868
- text: lang === 'zh'
869
- ? '统计显著但效应微弱,别过度解读'
870
- : 'significant but effect is trivial — do not overinterpret',
871
- color: 'var(--yellow)',
872
- };
873
- }
874
- if (!metric.significant && isStrong) {
875
- const leadPhrase = winner
876
- ? (lang === 'zh'
877
- ? `${winner} 看似${cfg.winnerWordZh}但用例不足,建议加大 --repeat`
878
- : `${winner} looks ${cfg.winnerWordEn.replace(' by', '')} but underpowered — increase --repeat`)
879
- : (lang === 'zh'
880
- ? `${strongLabelZh}差异但用例不足,建议加大 --repeat`
881
- : `${strongLabelEn} effect but underpowered — increase --repeat`);
882
- return {
883
- icon: '⚠',
884
- text: leadPhrase,
885
- color: 'var(--yellow)',
886
- };
887
- }
888
- return {
889
- icon: '—',
890
- text: lang === 'zh' ? '两变体相当,无实质差异' : 'no meaningful difference',
891
- color: 'var(--text-muted)',
892
- };
893
- }
894
- // Three-layer breakdown labels (PR-2). Rendered inside the expandable
895
- // `<details>` beneath each comparison; composite still lives on the top table.
896
- // UI 命名:事实 / 行为 / LLM 评价(字段 judge)——前两层规则验证,第三层 LLM 主观评分。
897
- const LAYER_LABELS = {
898
- fact: { zh: '事实', en: 'Fact' },
899
- behavior: { zh: '行为', en: 'Behavior' },
900
- judge: { zh: 'LLM 评价', en: 'LLM judge' },
901
- };
902
- const METRIC_CONFIGS = [
903
- {
904
- key: 'composite',
905
- labelZh: '质量',
906
- labelEn: 'Quality',
907
- higherIsBetter: true,
908
- winnerWordZh: '胜出',
909
- winnerWordEn: 'wins by',
910
- formatValue: (v) => `${v.toFixed(2)} 分`,
911
- showRawEffectSize: true,
912
- showPercent: true,
913
- percentWordZh: '高',
914
- percentWordEn: 'higher',
915
- },
916
- {
917
- key: 'cost',
918
- labelZh: '成本',
919
- labelEn: 'Cost',
920
- higherIsBetter: false,
921
- winnerWordZh: '更便宜',
922
- winnerWordEn: 'cheaper by',
923
- formatValue: (v) => `$${v.toFixed(4)}`,
924
- showRawEffectSize: true,
925
- showPercent: true,
926
- percentWordZh: '便宜',
927
- percentWordEn: 'cheaper',
928
- },
929
- {
930
- key: 'efficiency',
931
- labelZh: '效率',
932
- labelEn: 'Efficiency',
933
- higherIsBetter: false,
934
- winnerWordZh: '更快',
935
- winnerWordEn: 'faster by',
936
- formatValue: (v) => `${(v / 1000).toFixed(1)}s`,
937
- showRawEffectSize: true,
938
- showPercent: true,
939
- percentWordZh: '快',
940
- percentWordEn: 'faster',
941
- },
942
- ];
943
- function pickMetricFromComparison(comp, key) {
944
- if (key === 'composite') {
945
- return {
946
- meanDiff: comp.meanDiff,
947
- tStatistic: comp.tStatistic,
948
- df: comp.df,
949
- significant: comp.significant,
950
- effectSize: comp.effectSize,
951
- };
952
- }
953
- return comp.byMetric?.[key] ?? null;
954
- }
955
- function getVariantMetricMean(variance, variant, key) {
956
- const v = variance.perVariant[variant];
957
- if (!v)
958
- return null;
959
- if (key === 'composite')
960
- return v.mean;
961
- return v.byMetric?.[key]?.mean ?? null;
962
- }
963
- export function renderVarianceComparisons(variance, lang, layeredStatsOpen = false, summary) {
964
- if (!variance || !variance.comparisons || variance.comparisons.length === 0)
965
- return '';
966
- // 任一 variant 的 exec 或 judge cost 未报告 → cost 行跳过(显示「—」会跟"持平"难区分,
967
- // 索性直接不渲染该指标行)。判 cost 列前看 summary,缺 summary 当全 reported 走旧路径。
968
- const allCostReported = !summary || Object.values(summary).every((v) => v.execCostReported !== false && v.judgeCostReported !== false);
969
- const modalId = 'guide-variance-comparisons';
970
- const title = lang === 'zh' ? '方差与显著性' : 'Variance & Significance';
971
- const guideTitle = lang === 'zh' ? '如何阅读这张表?' : 'How to read this table?';
972
- const headerLabels = lang === 'zh'
973
- ? ['对比', '维度', '差距', '效应量', '显著性', '诊断']
974
- : ['Comparison', 'Metric', 'Gap', 'Effect size', 'Significance', 'Diagnostic'];
975
- const thead = `<tr>${headerLabels.map((h, i) => {
976
- const cls = i === 5 ? ' class="diagnostic-cell"' : '';
977
- return `<th${cls}>${h}</th>`;
978
- }).join('')}</tr>`;
979
- const magnitudeZh = { negligible: '可忽略', small: '小效应', medium: '中效应', large: '大效应' };
980
- const magnitudeEn = { negligible: 'negligible', small: 'small effect', medium: 'medium effect', large: 'large effect' };
981
- // Determine the winning variant for a given metric. Returns null if tied.
982
- function pickWinner(metric, cfg, a, b) {
983
- const diffAbs = Math.abs(metric.meanDiff);
984
- if (diffAbs < 1e-9)
985
- return null;
986
- const rawHigherVariant = metric.meanDiff > 0 ? a : b;
987
- return cfg.higherIsBetter ? rawHigherVariant : (metric.meanDiff > 0 ? b : a);
988
- }
989
- function buildMetricRowCells(metric, cfg, a, b, aMean, bMean, fadeDiagnostic) {
990
- // Gap cell — metric-aware direction word + optional relative percent
991
- const winner = pickWinner(metric, cfg, a, b);
992
- const diffAbs = Math.abs(metric.meanDiff);
993
- let gapCell;
994
- if (!winner) {
995
- gapCell = `<div class="verdict-line">${lang === 'zh' ? '持平' : 'tied'}</div>`;
996
- }
997
- else {
998
- const winnerWord = lang === 'zh' ? cfg.winnerWordZh : cfg.winnerWordEn;
999
- const detailParts = [cfg.formatValue(diffAbs)];
1000
- if (cfg.showPercent && aMean != null && bMean != null) {
1001
- const denom = Math.max(Math.abs(aMean), Math.abs(bMean));
1002
- if (denom > 0) {
1003
- const pct = (diffAbs / denom) * 100;
1004
- const pctWord = lang === 'zh' ? cfg.percentWordZh : cfg.percentWordEn;
1005
- detailParts.push(lang === 'zh' ? `${pctWord} ${pct.toFixed(0)}%` : `${pct.toFixed(0)}% ${pctWord}`);
1006
- }
1007
- }
1008
- gapCell = `
1009
- <div class="verdict-line"><strong>${e(winner)}</strong> ${winnerWord}</div>
1010
- <div class="detail-line">${detailParts.join(' · ')}</div>`;
1011
- }
1012
- // Effect size cell
1013
- const es = metric.effectSize;
1014
- let esCell;
1015
- if (!es || es.primary === 'none') {
1016
- esCell = `<div class="verdict-line">${lang === 'zh' ? '数据不足' : 'insufficient data'}</div>`;
1017
- }
1018
- else {
1019
- const magnitudeLabel = (lang === 'zh' ? magnitudeZh : magnitudeEn)[es.magnitude] || es.magnitude;
1020
- const detail = cfg.showRawEffectSize
1021
- ? `g=${Math.abs(es.hedgesG).toFixed(2)} · d=${Math.abs(es.cohensD).toFixed(2)} · n=${es.n1}+${es.n2}`
1022
- : `n=${es.n1}+${es.n2}`;
1023
- esCell = `
1024
- <div class="verdict-line">${magnitudeLabel}</div>
1025
- <div class="detail-line">${detail}</div>`;
1026
- }
1027
- // Significance cell — uniform binary verdict. Both sides just say
1028
- // "显著" or "不显著"; the p-value bucket is shown in the detail line
1029
- // so readers can distinguish "barely significant" from "strongly significant".
1030
- const sigText = metric.significant
1031
- ? (lang === 'zh' ? '显著' : 'significant')
1032
- : (lang === 'zh' ? '不显著' : 'not significant');
1033
- const pBucket = pValueCategory(metric.tStatistic, metric.df);
1034
- const sigCell = `
1035
- <div class="verdict-line">${sigText}</div>
1036
- <div class="detail-line">p${pBucket} · t=${metric.tStatistic.toFixed(2)} · df=${metric.df}</div>`;
1037
- // Diagnostic cell: colored icon + bold colored text with direction bound in.
1038
- // Visually fade if this diagnostic text is a consecutive duplicate of the row above.
1039
- const diag = buildDiagnostic(metric, cfg, winner, lang);
1040
- const fadeClass = fadeDiagnostic ? ' diag-faded' : '';
1041
- // inline-flex so the td's text-align:center centers the icon+text unit as
1042
- // a whole (a plain flex div would stretch to fill the cell and align left).
1043
- const diagCell = `
1044
- <div class="diag-cell${fadeClass}" style="display:inline-flex;align-items:center;gap:6px;line-height:1.5;text-align:left">
1045
- <span style="color:${diag.color};font-size:14px;flex-shrink:0">${diag.icon}</span>
1046
- <strong style="color:${diag.color}">${diag.text}</strong>
1047
- </div>`;
1048
- const metricLabel = lang === 'zh' ? cfg.labelZh : cfg.labelEn;
1049
- return `
1050
- <td class="verdict-line">${metricLabel}</td>
1051
- <td>${gapCell}</td>
1052
- <td>${esCell}</td>
1053
- <td>${sigCell}</td>
1054
- <td class="diagnostic-cell">${diagCell}</td>`;
1055
- }
1056
- // Per-layer display config — same shape as composite quality, just re-labeled.
1057
- // Composite rows live in METRIC_CONFIGS (the outer variance table); layer rows
1058
- // live inside an expandable <details> and only render when byLayer data exists.
1059
- function layerCfg(key) {
1060
- const labels = LAYER_LABELS[key];
1061
- return {
1062
- labelZh: labels.zh,
1063
- labelEn: labels.en,
1064
- higherIsBetter: true,
1065
- winnerWordZh: '胜出',
1066
- winnerWordEn: 'wins by',
1067
- formatValue: (v) => lang === 'zh' ? `${v.toFixed(2)} 分` : `${v.toFixed(2)} pts`,
1068
- showRawEffectSize: true,
1069
- showPercent: true,
1070
- percentWordZh: '高',
1071
- percentWordEn: 'higher',
1072
- };
1073
- }
1074
- function renderLayerBreakdown(comp) {
1075
- if (!comp.byLayer || Object.keys(comp.byLayer).length === 0)
1076
- return '';
1077
- const layerRows = ['fact', 'behavior', 'judge'].map((key) => {
1078
- const m = comp.byLayer?.[key];
1079
- if (!m)
1080
- return '';
1081
- const aMean = variance.perVariant[comp.a]?.byLayer?.[key]?.mean ?? null;
1082
- const bMean = variance.perVariant[comp.b]?.byLayer?.[key]?.mean ?? null;
1083
- const cfg = layerCfg(key);
1084
- return `<tr>${buildMetricRowCells(m, cfg, comp.a, comp.b, aMean, bMean, false)}</tr>`;
1085
- }).filter(Boolean).join('');
1086
- if (!layerRows)
1087
- return '';
1088
- const summaryLabel = lang === 'zh'
1089
- ? '展开三层独立显著性(fact / behavior / judge)'
1090
- : 'Show three-layer independent significance (fact / behavior / judge)';
1091
- const openAttr = layeredStatsOpen ? ' open' : '';
1092
- // 多重比较 disclaimer:三层独立 t 检验,family-wise error 未矫正;小样本下 Cohen's d 不稳(stats 术语,见 terminology-spec §6 例外)。
1093
- // 不默默修改 significant 判定(避免用户被"自动矫正"误导),而是把判读责任明示交给读者。
1094
- const disclaimerText = lang === 'zh'
1095
- ? '⚠ 三层独立检验:p 值未做多重比较矫正(建议按 Bonferroni α/3 = 0.017 判断显著);小样本(n ≤ 10)下 Cohen\'s d 效应量标签仅供探索参考,不作结论'
1096
- : '⚠ Three independent tests: p values are NOT corrected for multiple comparisons (use Bonferroni α/3 = 0.017 as the stricter threshold). With small samples (n ≤ 10), Cohen\'s d magnitude labels are exploratory only';
1097
- return `
1098
- <tr class="layer-breakdown-row">
1099
- <td colspan="6">
1100
- <details class="layer-breakdown"${openAttr}>
1101
- <summary>${e(summaryLabel)}</summary>
1102
- <div class="layer-breakdown-disclaimer">${e(disclaimerText)}</div>
1103
- <table class="summary-table variance-table layer-sub-table">
1104
- <tbody>${layerRows}</tbody>
1105
- </table>
1106
- </details>
1107
- </td>
1108
- </tr>`;
1109
- }
1110
- const rows = variance.comparisons.map((comp) => {
1111
- // Collect available metric rows for this comparison
1112
- const availableMetrics = [];
1113
- for (const cfg of METRIC_CONFIGS) {
1114
- // cost 行:任一 variant 不报 cost 时跳过(数据是占位 0,显示"持平"误导)
1115
- if (cfg.key === 'cost' && !allCostReported)
1116
- continue;
1117
- const m = pickMetricFromComparison(comp, cfg.key);
1118
- if (m)
1119
- availableMetrics.push({ cfg, metric: m });
1120
- }
1121
- if (availableMetrics.length === 0)
1122
- return '';
1123
- const rowspan = availableMetrics.length;
1124
- const comparisonCell = `<td rowspan="${rowspan}" class="verdict-line comparison-cell"><strong>${e(comp.a)}</strong> <span style="color:var(--text-muted)">vs</span> <strong>${e(comp.b)}</strong></td>`;
1125
- // Build all metric rows first so we can detect consecutive duplicate diagnostics
1126
- // (for visual fade), then emit.
1127
- const preBuilt = availableMetrics.map((row) => {
1128
- const winner = pickWinner(row.metric, row.cfg, comp.a, comp.b);
1129
- const diag = buildDiagnostic(row.metric, row.cfg, winner, lang);
1130
- return { ...row, diagText: `${diag.icon}|${diag.text}` };
1131
- });
1132
- let prevDiagKey = '';
1133
- const mainRows = preBuilt.map((row, idx) => {
1134
- const lead = idx === 0 ? comparisonCell : '';
1135
- const aMean = getVariantMetricMean(variance, comp.a, row.cfg.key);
1136
- const bMean = getVariantMetricMean(variance, comp.b, row.cfg.key);
1137
- const fade = idx > 0 && row.diagText === prevDiagKey;
1138
- prevDiagKey = row.diagText;
1139
- return `<tr>${lead}${buildMetricRowCells(row.metric, row.cfg, comp.a, comp.b, aMean, bMean, fade)}</tr>`;
1140
- }).join('');
1141
- return mainRows + renderLayerBreakdown(comp);
1142
- }).join('');
1143
- const glossaryZh = [
1144
- { label: '差距', desc: '跨轮均值胜出者 + 绝对差值(原始单位)' },
1145
- { label: '效应量', desc: '差距相对标准差的倍数。阈值:0.2=小 / 0.5=中 / 0.8=大' },
1146
- { label: "Hedges' g", desc: '小样本修正版(统计术语);n1+n2<20 时优先参考', sub: true },
1147
- { label: "Cohen's d", desc: '未修正版,n1+n2≥20 时是学术惯例', sub: true },
1148
- { label: '显著性', desc: 't 检验结论,基于 p<0.05 阈值。回答"差异真不真",和效应量"差多大"互补' },
1149
- { label: 'p 值', desc: '假设真的没差异时,观察到当前差距的概率。越小越可信。0.05 只是约定阈值', sub: true },
1150
- { label: 't 值', desc: '均值差 ÷ 估计误差,需配合 df 和效应量解读,不能单独看', sub: true },
1151
- { label: 'df 自由度', desc: '≈"有效评测用例数"。--repeat 3 时通常 2~4;想达到 20+ 需 --repeat 10+', sub: true },
1152
- ];
1153
- const glossaryEn = [
1154
- { label: 'Gap', desc: 'Cross-run mean winner + absolute difference (raw units)' },
1155
- { label: 'Effect size', desc: 'Gap measured in standard deviations. Thresholds: 0.2=small / 0.5=medium / 0.8=large' },
1156
- { label: "Hedges' g", desc: 'Small-sample corrected; preferred when n1+n2<20', sub: true },
1157
- { label: "Cohen's d", desc: 'Uncorrected; conventional when n1+n2≥20', sub: true },
1158
- { label: 'Significance', desc: 't-test verdict based on p<0.05 threshold. Complements effect size ("how real" vs "how big")' },
1159
- { label: 'p value', desc: 'Probability of seeing the current gap if variants were truly identical. Smaller = more confident. 0.05 is a convention', sub: true },
1160
- { label: 't value', desc: 'Mean diff ÷ estimated error. Must be read alongside df and effect size', sub: true },
1161
- { label: 'df', desc: '≈"effective sample size". --repeat 3 usually lands at 2~4; df 20+ needs --repeat 10+', sub: true },
1162
- ];
1163
- const glossaryRows = lang === 'zh' ? glossaryZh : glossaryEn;
1164
- const glossaryHtml = glossaryRows.map((r) => {
1165
- if (r.sub) {
1166
- return `<div class="modal-glossary-sub"><div class="modal-glossary-sub-label">${e(r.label)}</div><div class="modal-glossary-sub-desc">${e(r.desc)}</div></div>`;
1167
- }
1168
- return `<div class="modal-glossary-row"><div class="modal-glossary-label">${e(r.label)}</div><div class="modal-glossary-desc">${e(r.desc)}</div></div>`;
1169
- }).join('');
1170
- const diagRulesZhData = [
1171
- { variant: 'good', icon: '✓', title: '显著差异(中/大效应)', desc: '差异真实且有实际意义,可作为结论', example: '示例:v1 更便宜 · 显著 · g=1.04' },
1172
- { variant: 'warn', icon: '⚠', title: '显著但效应微弱', desc: '差异真实但太小没实际价值,别过度解读', example: '示例:p<0.05 但 g≈0.1(--repeat 很大时易出现)' },
1173
- { variant: 'warn', icon: '⚠', title: '大效应但用例不足', desc: '差距看似大但用例太少,建议加大 --repeat 再判断', example: '示例:v2 胜出 0.30 · g=1.04 · 不显著' },
1174
- { variant: 'neutral', icon: '—', title: '两变体相当', desc: '既不显著也效应微弱,可视为无差异', example: '示例:Δ≈0 · 不显著 · g<0.2' },
1175
- ];
1176
- const diagRulesEnData = [
1177
- { variant: 'good', icon: '✓', title: 'Significant, medium/large effect', desc: 'Real and meaningful — acceptable as a conclusion', example: 'e.g. v1 cheaper · significant · g=1.04' },
1178
- { variant: 'warn', icon: '⚠', title: 'Significant but trivial effect', desc: 'Real but tiny — do not overinterpret', example: 'e.g. p<0.05 but g≈0.1 (common with large --repeat)' },
1179
- { variant: 'warn', icon: '⚠', title: 'Large effect but underpowered', desc: 'Gap looks real but sample is too small — increase --repeat', example: 'e.g. v2 leads by 0.30 · g=1.04 · not significant' },
1180
- { variant: 'neutral', icon: '—', title: 'No meaningful difference', desc: 'Neither significant nor large — treat as equivalent', example: 'e.g. Δ≈0 · not significant · g<0.2' },
1181
- ];
1182
- const diagRulesData = lang === 'zh' ? diagRulesZhData : diagRulesEnData;
1183
- const diagRulesHtml = diagRulesData.map((rule) => `
1184
- <div class="diag-rule-row rule-${rule.variant}">
1185
- <span class="diag-rule-icon rule-${rule.variant}">${rule.icon}</span>
1186
- <div class="diag-rule-body">
1187
- <div class="diag-rule-title">${e(rule.title)}</div>
1188
- <div class="diag-rule-desc">${e(rule.desc)}</div>
1189
- <div class="diag-rule-example">${e(rule.example)}</div>
1190
- </div>
1191
- </div>`).join('');
1192
- const orderHint = lang === 'zh' ? '以下按表格列顺序排列' : 'Below follows the table column order';
1193
- const sectionTitle = lang === 'zh' ? '四象限诊断规则' : 'Four-quadrant diagnostic rules';
1194
- const closeLabel = lang === 'zh' ? '关闭' : 'Close';
1195
- return `
1196
- <section style="margin-top:24px">
1197
- <h2 style="display:flex;align-items:center;gap:4px">${title} <button type="button" class="hint-btn" onclick="openModal('${modalId}')" aria-label="${e(guideTitle)}" aria-haspopup="dialog">?</button></h2>
1198
- <div id="${modalId}" class="modal-overlay" role="dialog" aria-modal="true" aria-labelledby="${modalId}-title" onclick="if(event.target===this)closeModal('${modalId}')">
1199
- <div class="modal-content">
1200
- <div class="modal-header">
1201
- <strong id="${modalId}-title" style="font-size:1rem">${e(guideTitle)}</strong>
1202
- <button type="button" class="modal-close" onclick="closeModal('${modalId}')" aria-label="${closeLabel}">✕</button>
1203
- </div>
1204
- <p class="modal-glossary-hint">${e(orderHint)}</p>
1205
- <div class="modal-glossary">${glossaryHtml}</div>
1206
- <div class="modal-section">
1207
- <div class="modal-section-title">${e(sectionTitle)}</div>
1208
- <div class="diag-rules">${diagRulesHtml}</div>
1209
- </div>
1210
- </div>
1211
- </div>
1212
- <div class="table-wrap">
1213
- <table class="summary-table variance-table">
1214
- <thead>${thead}</thead>
1215
- <tbody>${rows}</tbody>
1216
- </table>
1217
- </div>
1218
- </section>`;
1219
- }
1220
- const CONCLUSION_TYPES = new Set([
1221
- 'efficiency_gap', 'tool_count_gap', 'high_cost_sample',
1222
- ]);
1223
- function isConclusion(insight) {
1224
- return CONCLUSION_TYPES.has(insight.type);
1225
- }
1226
- function severityDot(severity) {
1227
- const color = {
1228
- error: 'var(--red)',
1229
- warning: 'var(--yellow)',
1230
- info: 'var(--accent)',
1231
- };
1232
- return `<span style="display:inline-block;width:6px;height:6px;border-radius:50%;background:${color[severity] || 'var(--text-muted)'};margin-right:8px;flex-shrink:0;margin-top:6px"></span>`;
1233
- }
1234
- function renderSummaryStructured(summary, lang) {
1235
- const markerRegex = /【([^】]+)】/g;
1236
- const markers = [];
1237
- let match;
1238
- while ((match = markerRegex.exec(summary)) !== null) {
1239
- markers.push({ label: match[1], start: match.index, contentStart: match.index + match[0].length });
1240
- }
1241
- const sections = [];
1242
- for (let i = 0; i < markers.length; i++) {
1243
- const end = i + 1 < markers.length ? markers[i + 1].start : summary.length;
1244
- sections.push({ label: markers[i].label, content: summary.slice(markers[i].contentStart, end).trim() });
1245
- }
1246
- if (sections.length === 0) {
1247
- return `<div style="padding:14px 18px;font-size:13px;line-height:1.8;color:var(--text-secondary);background:var(--bg-surface);border-radius:var(--radius);border:1px solid var(--border)">${e(summary)}</div>`;
1248
- }
1249
- const labelMap = {
1250
- 结论: { zh: '结论', en: 'Conclusion' },
1251
- Conclusion: { zh: '结论', en: 'Conclusion' },
1252
- '关键差异': { zh: '关键差异', en: 'Key differences' },
1253
- 'Key differences': { zh: '关键差异', en: 'Key differences' },
1254
- 综合洞察: { zh: '综合洞察', en: 'Synthesis' },
1255
- Synthesis: { zh: '综合洞察', en: 'Synthesis' },
1256
- };
1257
- const sectionHtml = sections.map((section) => `
1258
- <div style="display:flex;gap:12px;align-items:baseline">
1259
- <span style="flex-shrink:0;font-size:11px;font-weight:600;color:var(--text-muted);letter-spacing:0.03em;min-width:56px">${e(labelMap[section.label]?.[lang] || section.label)}</span>
1260
- <span style="color:var(--text-secondary);font-size:13px;line-height:1.7">${e(section.content)}</span>
1261
- </div>`).join('');
1262
- return `
1263
- <div style="padding:14px 18px;background:var(--bg-surface);border-radius:var(--radius);border:1px solid var(--border);display:flex;flex-direction:column;gap:8px">
1264
- ${sectionHtml}
1265
- </div>`;
1266
- }
1267
- function asRecord(value) {
1268
- return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
1269
- }
1270
- function asArray(value) {
1271
- return Array.isArray(value) ? value : [];
1272
- }
1273
- function num(value) {
1274
- return typeof value === 'number' && Number.isFinite(value) ? value : null;
1275
- }
1276
- function localizedInsightMessage(insight, report, lang) {
1277
- const details = asRecord(insight.details);
1278
- const detailItems = asArray(insight.details);
1279
- const totalSamples = report?.results?.length || detailItems.length || num(details.total) || 0;
1280
- const type = insight.type;
1281
- switch (type) {
1282
- case 'low_discrimination_all_passed': {
1283
- const count = detailItems.length;
1284
- return lang === 'zh'
1285
- ? `${count} 个断言所有变体均通过,区分度低`
1286
- : `${count} assertions passed for every variant, so they have low discrimination power`;
1287
- }
1288
- case 'low_discrimination_all_failed': {
1289
- const count = detailItems.length;
1290
- return lang === 'zh'
1291
- ? `${count} 个断言所有变体均失败,断言可能过严或配置有误`
1292
- : `${count} assertions failed for every variant; they may be too strict or misconfigured`;
1293
- }
1294
- case 'uniform_scores': {
1295
- const count = detailItems.length;
1296
- return lang === 'zh'
1297
- ? `${count}/${totalSamples} 个评测用例在变体间分差 < 0.5,区分度较低`
1298
- : `${count}/${totalSamples} samples differ by less than 0.5 across variants, so they have low discrimination power`;
1299
- }
1300
- case 'all_pass':
1301
- return lang === 'zh'
1302
- ? '所有断言在所有变体上全部通过,断言可能过于宽松'
1303
- : 'All assertions passed for all variants; the assertions may be too loose';
1304
- case 'all_fail':
1305
- return lang === 'zh'
1306
- ? '所有断言在所有变体上全部失败,请检查断言配置'
1307
- : 'All assertions failed for all variants; check the assertion configuration';
1308
- case 'suggest_repeat': {
1309
- const variant = String(details.variant || 'variant');
1310
- const min = num(details.min);
1311
- const max = num(details.max);
1312
- const range = min != null && max != null ? `${min}~${max}` : '';
1313
- return lang === 'zh'
1314
- ? `${variant} 的分数跨度较大${range ? `(${range})` : ''},建议使用 --repeat 多轮评测`
1315
- : `${variant} has a wide score range${range ? ` (${range})` : ''}; run with --repeat to measure variance`;
1316
- }
1317
- case 'low_tool_success_rate': {
1318
- const variant = String(details.variant || 'variant');
1319
- const rate = num(details.toolSuccessRate);
1320
- const pct = rate != null ? `${(rate * 100).toFixed(0)}%` : 'low';
1321
- return lang === 'zh'
1322
- ? `${variant} 的工具调用成功率仅 ${pct},可能存在工具选择或参数问题`
1323
- : `${variant} has only ${pct} tool-call success, which may indicate tool-selection or parameter issues`;
1324
- }
1325
- case 'tool_permission_error': {
1326
- const count = detailItems.length;
1327
- return lang === 'zh'
1328
- ? `检测到 ${count} 次工具权限错误,实验结论可能被环境问题污染`
1329
- : `${count} tool permission errors were detected; the conclusion may be contaminated by environment issues`;
1330
- }
1331
- case 'trace_integrity_gap': {
1332
- const count = detailItems.length;
1333
- return lang === 'zh'
1334
- ? `${count} 个 variant 的 trace 覆盖率低于 75%,报告可能不足以解释 agent 行为差异`
1335
- : `${count} variants have trace coverage below 75%, so the report may not fully explain agent behavior differences`;
1336
- }
1337
- case 'agent_assertion_discrimination_low': {
1338
- const total = num(details.total) || 0;
1339
- const discriminative = num(details.discriminative) || 0;
1340
- const pct = total > 0 ? Math.round((discriminative / total) * 100) : 0;
1341
- return lang === 'zh'
1342
- ? `agent 断言区分度偏低,只有 ${pct}% 的断言真正拉开了变体差异`
1343
- : `Agent assertion discrimination is low; only ${pct}% of assertions separate variants`;
1344
- }
1345
- case 'agent_assertion_discrimination_ok': {
1346
- const total = num(details.total) || 0;
1347
- const discriminative = num(details.discriminative) || 0;
1348
- const pct = total > 0 ? Math.round((discriminative / total) * 100) : 0;
1349
- return lang === 'zh'
1350
- ? `agent 断言区分度达标,${pct}% 的断言能区分变体差异`
1351
- : `Agent assertion discrimination is healthy; ${pct}% of assertions separate variants`;
1352
- }
1353
- case 'efficiency_gap': {
1354
- const variant = String(details.variant || 'variant');
1355
- const baseline = String(details.baseline || 'baseline');
1356
- return lang === 'zh'
1357
- ? `${variant} 与 ${baseline} 在效率维度存在明显差异`
1358
- : `${variant} differs materially from ${baseline} on efficiency`;
1359
- }
1360
- case 'tool_count_gap': {
1361
- const variant = String(details.variant || 'variant');
1362
- const baseline = String(details.baseline || 'baseline');
1363
- const baseTools = num(details.baseTools);
1364
- const otherTools = num(details.otherTools);
1365
- const delta = baseTools != null && otherTools != null ? Math.abs(otherTools - baseTools).toFixed(1) : '';
1366
- return lang === 'zh'
1367
- ? `${variant} 与 ${baseline} 的工具调用次数差异明显${delta ? `(差 ${delta} 次)` : ''}`
1368
- : `${variant} differs materially from ${baseline} in tool-call count${delta ? ` (${delta} calls apart)` : ''}`;
1369
- }
1370
- case 'high_cost_sample': {
1371
- const count = detailItems.length;
1372
- return lang === 'zh'
1373
- ? `${count} 个评测用例成本显著高于平均值`
1374
- : `${count} samples cost materially more than average`;
1375
- }
1376
- default:
1377
- return lang === 'zh'
1378
- ? `结构化诊断:${type}`
1379
- : `Structured diagnostic: ${type}`;
1380
- }
1381
- }
1382
- function localizedSuggestion(insight, lang) {
1383
- switch (insight.type) {
1384
- case 'low_discrimination_all_passed':
1385
- return lang === 'zh'
1386
- ? '把全通过断言替换为能检测 skill 独有细节的断言,例如特定参数名、配置值或流程要求'
1387
- : 'Replace always-passing assertions with checks for artifact-specific details, such as parameter names, config values, or workflow requirements';
1388
- case 'low_discrimination_all_failed':
1389
- return lang === 'zh'
1390
- ? '检查断言条件是否正确,或降低匹配要求,避免 broken 用例污染结论'
1391
- : 'Check whether the assertion condition is correct, or relax the matcher to avoid broken samples contaminating the conclusion';
1392
- case 'uniform_scores':
1393
- return lang === 'zh'
1394
- ? '增加更有挑战性的评测用例,或把 rubric 写得更能区分优劣'
1395
- : 'Add more challenging samples, or make the rubric more discriminative';
1396
- case 'all_pass':
1397
- return lang === 'zh'
1398
- ? '增加更严格的断言来区分不同变体的质量'
1399
- : 'Add stricter assertions that can distinguish variant quality';
1400
- case 'all_fail':
1401
- return lang === 'zh'
1402
- ? '优先检查评测配置和断言目标,确认用例不是整体不可达'
1403
- : 'Check the evaluation config and assertion target first; make sure the sample is reachable';
1404
- case 'suggest_repeat':
1405
- return lang === 'zh'
1406
- ? '使用 --repeat 3 或更多轮次获取方差、置信区间和显著性检验'
1407
- : 'Run with --repeat 3 or more to obtain variance, confidence intervals, and significance tests';
1408
- case 'low_tool_success_rate':
1409
- return lang === 'zh'
1410
- ? '检查失败工具调用,必要时在 skill 中补充工具选择和参数约束'
1411
- : 'Inspect failed tool calls and, if needed, add tool-selection and parameter guidance to the artifact';
1412
- case 'tool_permission_error':
1413
- return lang === 'zh'
1414
- ? '先解决工具权限或工作目录问题,再解读分数差异'
1415
- : 'Fix tool permission or working-directory issues before interpreting score differences';
1416
- case 'trace_integrity_gap':
1417
- return lang === 'zh'
1418
- ? '补齐 turns、toolCalls、timing 和完整输出采集,确保报告能解释行为差异'
1419
- : 'Capture turns, tool calls, timing, and full output so the report can explain behavior differences';
1420
- case 'agent_assertion_discrimination_low':
1421
- return lang === 'zh'
1422
- ? '重写 agent 断言时优先约束工具路径、关键文件读取和 turns 上限'
1423
- : 'When rewriting agent assertions, prioritize tool path, key file reads, and turn-limit constraints';
1424
- default:
1425
- return lang === 'zh'
1426
- ? '查看结构化 details 字段定位原因'
1427
- : 'Inspect the structured details field for the underlying evidence';
1428
- }
1429
- }
1430
- export function renderAnalysis(report, lang) {
1431
- const analysis = report?.analysis;
1432
- if (!analysis)
1433
- return '';
1434
- const { insights } = analysis;
1435
- if ((!insights || insights.length === 0) && !report)
1436
- return '';
1437
- const issues = (insights || []).filter((insight) => !isConclusion(insight));
1438
- const issueLabel = lang === 'zh' ? '问题与建议' : 'Issues & Suggestions';
1439
- const safeSuggestions = issues.map((issue) => localizedSuggestion(issue, lang));
1440
- let issuesHtml = '';
1441
- if (issues.length > 0) {
1442
- const maxRows = issues.length;
1443
- const rows = [];
1444
- for (let i = 0; i < maxRows; i++) {
1445
- const issue = issues[i];
1446
- const suggestion = safeSuggestions[i];
1447
- const issueContent = issue
1448
- ? `${severityDot(issue.severity)}<span>${e(localizedInsightMessage(issue, report, lang))}</span>`
1449
- : '';
1450
- const suggestionContent = suggestion
1451
- ? e(suggestion)
1452
- : `<span style="color:var(--text-faint)">—</span>`;
1453
- rows.push(`
1454
- <div style="display:grid;grid-template-columns:1fr 1fr;gap:16px;padding:10px 16px;border-bottom:1px solid var(--border)">
1455
- <div style="display:flex;align-items:flex-start;color:var(--text-secondary);font-size:12.5px;line-height:1.6">${issueContent}</div>
1456
- <div style="color:var(--text-muted);font-size:12.5px;line-height:1.6">${suggestionContent}</div>
1457
- </div>`);
1458
- }
1459
- issuesHtml = `
1460
- <div style="margin-top:12px">
1461
- <h3 style="font-size:12px;color:var(--text-muted);font-weight:600;margin:0 0 8px;letter-spacing:0.03em">${issueLabel}</h3>
1462
- <div style="background:var(--bg-surface);border:1px solid var(--border);border-radius:var(--radius);overflow:hidden">
1463
- <div style="display:grid;grid-template-columns:1fr 1fr;gap:16px;padding:8px 16px;border-bottom:1px solid var(--border)">
1464
- <div style="font-size:11px;font-weight:500;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em">${lang === 'zh' ? '问题' : 'Issue'}</div>
1465
- <div style="font-size:11px;font-weight:500;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em">${lang === 'zh' ? '建议' : 'Suggestion'}</div>
1466
- </div>
1467
- ${rows.join('')}
1468
- </div>
1469
- </div>
1470
- `;
1471
- }
1472
- const generatedSummary = report ? generateAnalysisSummary(report, lang) : undefined;
1473
- const summaryHtml = generatedSummary
1474
- ? renderSummaryStructured(generatedSummary, lang)
1475
- : '';
1476
- if (!summaryHtml && !issuesHtml)
1477
- return '';
1478
- return `
1479
- <h2 data-i18n="autoAnalysis">${t('autoAnalysis', lang)}</h2>
1480
- ${summaryHtml}
1481
- ${issuesHtml}
1482
- `;
1483
- }
1484
- export function renderAgentOverview(variants, summary, lang) {
1485
- const hasAgentData = variants.some((variant) => summary[variant]?.avgToolCalls != null && summary[variant].avgToolCalls > 0);
1486
- if (!hasAgentData)
1487
- return '';
1488
- // v0.30 — Agent 执行概览弱化为 ctx-row(单行 inline)样式,跟实验配置并排放在 context-strip 里。
1489
- // 主要数字:轮次 / 工具调用数 / 成功率,加 tooltip 显示工具分布。
1490
- const variantRows = variants.map((variant, i) => {
1491
- const stats = summary[variant];
1492
- if (!stats)
1493
- return '';
1494
- const color = COLORS[i % COLORS.length];
1495
- const avgTools = stats.avgToolCalls ?? 0;
1496
- const successRate = stats.toolSuccessRate != null ? `${(stats.toolSuccessRate * 100).toFixed(0)}%` : '—';
1497
- const successRateColor = (stats.toolSuccessRate ?? 1) >= 0.8 ? 'var(--green)' : 'var(--red)';
1498
- const turns = stats.avgFullNumTurns ?? stats.avgNumTurns ?? 0;
1499
- const distributionEntries = Object.entries(stats.toolDistribution || {}).sort((a, b) => b[1] - a[1]);
1500
- const distSummary = distributionEntries.length > 0
1501
- ? distributionEntries.slice(0, 5).map(([tool, count]) => `${tool}×${count}`).join(' · ')
1502
- : '';
1503
- const tooltip = distSummary ? `${t('agentToolDist', lang)}: ${distSummary}` : '';
1504
- return `<div class="ctx-row" style="border-left:3px solid ${color}"${tooltip ? ` title="${e(tooltip)}"` : ''}>
1505
- <span class="ctx-row-name">${e(variant)}</span>
1506
- <span class="ctx-row-bits">
1507
- <span>${turns} ${t('turnsPerReq', lang)}</span>
1508
- <span class="ctx-sep">·</span>
1509
- <span>${avgTools} ${lang === 'zh' ? '工具/次' : 'tools/req'}</span>
1510
- <span class="ctx-sep">·</span>
1511
- <span style="color:${successRateColor}">${successRate} OK</span>
1512
- </span>
1513
- </div>`;
1514
- }).join('');
1515
- return `
1516
- <div class="ctx-block">
1517
- <div class="ctx-h">${t('agentOverview', lang)}</div>
1518
- <div class="ctx-sub">${lang === 'zh' ? '工具调用统计 · 轮次 / 调用数 / 成功率' : 'Tool call stats · turns / calls / success rate'}</div>
1519
- <div class="ctx-rows">${variantRows}</div>
1520
- </div>
1521
- `;
1522
- }
1523
- export function renderCoverageSection(coverage, lang) {
1524
- if (!coverage || Object.keys(coverage).length === 0)
1525
- return '';
1526
- const variantSections = Object.entries(coverage).map(([variant, knowledgeCoverage]) => {
1527
- if (knowledgeCoverage.filesTotal === 0)
1528
- return '';
1529
- const pct = Math.round(knowledgeCoverage.fileCoverageRate * 100);
1530
- const pctColor = pct >= 80 ? 'var(--green)' : pct >= 50 ? 'var(--yellow)' : 'var(--red)';
1531
- const barW = Math.max(2, pct);
1532
- const fileRows = knowledgeCoverage.entries
1533
- .sort((a, b) => (a.accessed === b.accessed ? 0 : a.accessed ? -1 : 1))
1534
- .map((entry) => {
1535
- const icon = entry.accessed ? '✓' : '✗';
1536
- const color = entry.accessed ? 'var(--green)' : 'var(--text-muted)';
1537
- const countBadge = entry.accessCount > 1
1538
- ? `<span style="font-size:10px;color:var(--accent);margin-left:4px">×${entry.accessCount}</span>`
1539
- : '';
1540
- const lines = entry.lineCount ? `<span style="font-size:10px;color:var(--text-muted);margin-left:4px">${entry.lineCount}L</span>` : '';
1541
- const typeTag = `<span style="font-size:10px;padding:1px 4px;border-radius:2px;background:var(--bg-surface);color:var(--text-muted);margin-left:4px">${e(entry.type)}</span>`;
1542
- return `<div style="display:flex;align-items:center;gap:4px;padding:2px 0;font-size:12px">
1543
- <span style="color:${color};width:16px;text-align:center">${icon}</span>
1544
- <span style="color:${entry.accessed ? 'var(--text-primary)' : 'var(--text-muted)'};${entry.accessed ? '' : 'text-decoration:line-through;opacity:0.6'}">${e(entry.path)}</span>
1545
- ${typeTag}${lines}${countBadge}
1546
- </div>`;
1547
- }).join('');
1548
- const uncoveredByType = {};
1549
- for (const entry of knowledgeCoverage.entries.filter((item) => !item.accessed)) {
1550
- const category = entry.path.startsWith('repos/') ? 'code' : entry.type;
1551
- (uncoveredByType[category] = uncoveredByType[category] || []).push(entry.path);
1552
- }
1553
- const hintLines = [];
1554
- const typeLabels = lang === 'zh'
1555
- ? { principle: '原则文件', semantic: '语义索引', design: '设计文档', code: '代码路径', script: '脚本工具', other: '其他知识' }
1556
- : { principle: 'Principles', semantic: 'Semantic index', design: 'Design docs', code: 'Code paths', script: 'Scripts', other: 'Other' };
1557
- for (const [type, files] of Object.entries(uncoveredByType)) {
1558
- const label = typeLabels[type] || type;
1559
- hintLines.push(`<strong>${label}</strong>(${files.length}):${files.slice(0, 3).map((file) => `<code>${e(file)}</code>`).join(', ')}${files.length > 3 ? ` +${files.length - 3}` : ''}`);
1560
- }
1561
- const uncoveredHint = hintLines.length > 0
1562
- ? `<div style="font-size:11px;color:var(--text-muted);margin-top:10px;border-top:1px solid var(--border);padding-top:8px">
1563
- <div style="margin-bottom:4px">${lang === 'zh' ? '💡 以下知识未被任何用例覆盖,建议补充评测用例:' : '💡 These knowledge files were not accessed by any sample — consider adding test cases:'}</div>
1564
- ${hintLines.map((line) => `<div style="margin:2px 0">${line}</div>`).join('')}
1565
- </div>`
1566
- : '';
1567
- return `<div style="flex:1;min-width:280px;padding:16px;background:var(--bg-surface);border:1px solid var(--border);border-radius:var(--radius)">
1568
- <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px">
1569
- <strong>${e(variant)}</strong>
1570
- <span style="font-size:20px;font-weight:600;color:${pctColor}">${pct}%</span>
1571
- </div>
1572
- <div style="height:8px;background:var(--bg-card);border-radius:4px;margin-bottom:8px">
1573
- <div style="width:${barW}%;height:100%;background:${pctColor};border-radius:4px"></div>
1574
- </div>
1575
- <div style="font-size:11px;color:var(--text-muted);margin-bottom:8px">${knowledgeCoverage.filesCovered}/${knowledgeCoverage.filesTotal} ${lang === 'zh' ? '个文件被访问' : 'files accessed'} · ${knowledgeCoverage.grepPatternsUsed} ${lang === 'zh' ? '次搜索' : 'searches'}</div>
1576
- ${fileRows}
1577
- ${uncoveredHint}
1578
- </div>`;
1579
- }).join('');
1580
- const title = lang === 'zh' ? '本次测评的知识使用情况' : 'Knowledge usage in this evaluation';
1581
- const desc = lang === 'zh'
1582
- ? '本次测评中,哪些知识没有被使用。数字低说明评测用例没覆盖到的角落多,不是知识库内容缺失——配合下方"本次测评的知识盲区"一起看才完整。'
1583
- : 'Which knowledge files were NOT exercised by this evaluation. Low coverage means test cases leave KB corners untouched, not that the KB is incomplete. Pair with "knowledge gaps" below for the full picture.';
1584
- return `
1585
- <section style="margin-top:24px">
1586
- <h2>${title}</h2>
1587
- <p style="font-size:12px;color:var(--text-muted);margin-bottom:12px">${desc}</p>
1588
- <div style="display:flex;gap:16px;flex-wrap:wrap">
1589
- ${variantSections}
1590
- </div>
1591
- </section>
1592
- `;
1593
- }
1594
- /**
1595
- * Render the knowledge gap section: per-variant gap rate + mandatory test set
1596
- * watermark + signal classification + inventory of individual signals.
1597
- * See docs/knowledge-gap-signal-spec.md for the semantics.
1598
- */
1599
- /**
1600
- * Combined knowledge-interaction section: coverage + gap side-by-side per variant.
1601
- *
1602
- * v0.17 起替代原独立的 renderCoverageSection + renderGapSection。两者都是"测评集
1603
- * × 知识库交互"的产物(spec §二),分开展示会让读者只看一个指标得出误判结论。合并后
1604
- * 每个 variant 一张 card,左右两栏并排展示"用了多少"vs"撞了多少",形成完整诊断画像。
1605
- */
1606
- export function renderKnowledgeInteractionSection(coverage, gapReports, lang) {
1607
- const hasCov = coverage && Object.keys(coverage).length > 0;
1608
- const hasGap = gapReports && Object.keys(gapReports).length > 0;
1609
- if (!hasCov && !hasGap)
1610
- return '';
1611
- // 所有 variant 的 coverage 都不可用 (filesTotal === 0) 且所有 gap 都是 0%(无信号)
1612
- // 时,整个 section 没有可读信息 — 渲染只会占大段纵向空间显示「数据不可用 / 0%」
1613
- // 干扰用户阅读流。直接跳过比强行展示「显式无信号」更尊重读者。
1614
- const usefulCoverage = hasCov && Object.values(coverage).some((c) => c.filesTotal > 0);
1615
- const usefulGap = hasGap && Object.values(gapReports).some((g) => g.gapRate > 0 || g.signals.length > 0);
1616
- if (!usefulCoverage && !usefulGap)
1617
- return '';
1618
- const title = lang === 'zh' ? '本次测评:评测用例 × 知识库' : 'This Evaluation: Test Set × Knowledge Base';
1619
- const desc = lang === 'zh'
1620
- ? '展示本次评测用例和知识库的交互画像——用到哪些知识(使用情况)· 哪些知识想找但没找到或任务执行模型表达不确定(盲区)。'
1621
- : 'How this test set interacts with the KB — which knowledge was exercised (usage) · which knowledge the task execution model missed or flagged as uncertain (gaps).';
1622
- const readHint = lang === 'zh'
1623
- ? '💡 读表:两者同时高 → 知识库内容有问题(有文件但答不出)· 同时低 → 评测用例太浅(没触到复杂场景)· 使用高 + 盲区低 → 理想但警惕用例驯化'
1624
- : '💡 Read together: both high → KB content issues (files exist but can\'t answer) · both low → test set too shallow · high use + low gap → ideal, but beware sample-set overfitting';
1625
- // 聚合所有 variant(coverage / gap 任一侧存在即纳入)
1626
- const allVariants = Array.from(new Set([
1627
- ...(hasCov ? Object.keys(coverage) : []),
1628
- ...(hasGap ? Object.keys(gapReports) : []),
1629
- ]));
1630
- const signalTypeLabels = {
1631
- failed_search: { zh: '搜索未命中', en: 'Search miss' },
1632
- explicit_marker: { zh: '任务执行模型标记缺口', en: 'Execution model flag' },
1633
- hedging: { zh: '表达不确定', en: 'Hedging' },
1634
- repeated_failure: { zh: '反复未命中', en: 'Repeated miss' },
1635
- };
1636
- const pickSignalLabel = (key) => signalTypeLabels[key][lang === 'zh' ? 'zh' : 'en'];
1637
- // severity 按 SIGNAL_WEIGHTS 映射:strong (weight 1.0) → 红色 · weak (0.5) → 灰/黄
1638
- const signalSeverity = {
1639
- failed_search: 'strong',
1640
- repeated_failure: 'strong',
1641
- explicit_marker: 'medium',
1642
- hedging: 'weak',
1643
- };
1644
- const cards = allVariants.map((variant, i) => {
1645
- const cov = coverage?.[variant];
1646
- const gap = gapReports?.[variant];
1647
- const variantColor = COLORS[i % COLORS.length];
1648
- // ─── 左栏:知识使用(coverage)────────────────────────
1649
- let covInner = '';
1650
- if (cov && cov.filesTotal > 0) {
1651
- const pct = Math.round(cov.fileCoverageRate * 100);
1652
- const pctColor = pct >= 80 ? 'var(--green)' : pct >= 50 ? 'var(--yellow)' : 'var(--red)';
1653
- const barW = Math.max(2, pct);
1654
- const barLabel = lang === 'zh' ? '知识使用' : 'Knowledge used';
1655
- const fileRows = cov.entries
1656
- .slice()
1657
- .sort((a, b) => (a.accessed === b.accessed ? 0 : a.accessed ? -1 : 1))
1658
- .map((entry) => {
1659
- const icon = entry.accessed ? '✓' : '✗';
1660
- const color = entry.accessed ? 'var(--green)' : 'var(--text-muted)';
1661
- const countBadge = entry.accessCount > 1
1662
- ? `<span style="font-size:10px;color:var(--accent);margin-left:4px">×${entry.accessCount}</span>`
1663
- : '';
1664
- const lines = entry.lineCount ? `<span style="font-size:10px;color:var(--text-muted);margin-left:4px">${entry.lineCount}L</span>` : '';
1665
- const typeTag = `<span style="font-size:10px;padding:1px 4px;border-radius:2px;background:var(--bg-card);color:var(--text-muted);margin-left:4px">${e(entry.type)}</span>`;
1666
- return `<div style="display:flex;align-items:center;gap:4px;padding:2px 0;font-size:12px">
1667
- <span style="color:${color};width:16px;text-align:center">${icon}</span>
1668
- <span style="color:${entry.accessed ? 'var(--text-primary)' : 'var(--text-muted)'};${entry.accessed ? '' : 'text-decoration:line-through;opacity:0.6'};word-break:break-all">${e(entry.path)}</span>
1669
- ${typeTag}${lines}${countBadge}
1670
- </div>`;
1671
- }).join('');
1672
- const uncoveredByType = {};
1673
- for (const entry of cov.entries.filter((item) => !item.accessed)) {
1674
- const category = entry.path.startsWith('repos/') ? 'code' : entry.type;
1675
- (uncoveredByType[category] = uncoveredByType[category] || []).push(entry.path);
1676
- }
1677
- const typeLabels = lang === 'zh'
1678
- ? { principle: '原则文件', semantic: '语义索引', design: '设计文档', code: '代码路径', script: '脚本工具', other: '其他知识' }
1679
- : { principle: 'Principles', semantic: 'Semantic index', design: 'Design docs', code: 'Code paths', script: 'Scripts', other: 'Other' };
1680
- const hintLines = [];
1681
- for (const [type, files] of Object.entries(uncoveredByType)) {
1682
- const label = typeLabels[type] || type;
1683
- hintLines.push(`<strong>${label}</strong>(${files.length}):${files.slice(0, 3).map((file) => `<code>${e(file)}</code>`).join(', ')}${files.length > 3 ? ` +${files.length - 3}` : ''}`);
1684
- }
1685
- const uncoveredHint = hintLines.length > 0
1686
- ? `<div style="font-size:11px;color:var(--text-muted);margin-top:10px;border-top:1px solid var(--border);padding-top:8px">
1687
- <div style="margin-bottom:4px">${lang === 'zh' ? '💡 以下知识未被任何用例覆盖,建议补充评测用例:' : '💡 These knowledge files were not accessed — consider adding test cases:'}</div>
1688
- ${hintLines.map((line) => `<div style="margin:2px 0">${line}</div>`).join('')}
1689
- </div>`
1690
- : '';
1691
- const uncoveredCount = cov.filesTotal - cov.filesCovered;
1692
- const summaryParts = [
1693
- `${cov.filesCovered} ${lang === 'zh' ? '命中' : 'hit'}`,
1694
- `${uncoveredCount} ${lang === 'zh' ? '未命中' : 'miss'}`,
1695
- `${cov.grepPatternsUsed} ${lang === 'zh' ? '次搜索' : 'searches'}`,
1696
- ].join(' · ');
1697
- const detailsLabel = lang === 'zh' ? `展开 ${cov.filesTotal} 个文件清单` : `Show all ${cov.filesTotal} files`;
1698
- covInner = `
1699
- <div class="ki-col-header">
1700
- <span class="ki-col-title">${lang === 'zh' ? '知识使用' : 'Knowledge used'}</span>
1701
- <span class="ki-col-value" style="color:${pctColor}">${pct}%</span>
1702
- </div>
1703
- <div class="ki-bar" role="progressbar" aria-valuenow="${pct}" aria-valuemin="0" aria-valuemax="100" aria-label="${e(barLabel)}">
1704
- <div class="ki-bar-fill" style="width:${barW}%;background:${pctColor}"></div>
1705
- </div>
1706
- <div style="font-size:11px;color:var(--text-muted)">${summaryParts}</div>
1707
- <details class="ki-details"><summary>${detailsLabel}</summary>
1708
- ${fileRows}
1709
- ${uncoveredHint}
1710
- </details>`;
1711
- }
1712
- else {
1713
- covInner = `<div style="color:var(--text-muted);font-size:12px">${lang === 'zh' ? '知识使用数据不可用' : 'Coverage data unavailable'}</div>`;
1714
- }
1715
- // ─── 右栏:知识盲区(gap)────────────────────────────
1716
- let gapInner = '';
1717
- if (gap) {
1718
- const pct = Math.round(gap.gapRate * 100);
1719
- const pctColor = pct <= 10 ? 'var(--green)' : pct <= 30 ? 'var(--yellow)' : 'var(--red)';
1720
- const barW = Math.max(2, pct);
1721
- const barLabel = lang === 'zh' ? '知识盲区' : 'Knowledge gaps';
1722
- const weightedPct = Math.round(gap.weightedGapRate * 100);
1723
- const softShare = pct - weightedPct;
1724
- const weightedHint = lang === 'zh'
1725
- ? (softShare >= 10
1726
- ? `<strong>实际盲区 ${weightedPct}%</strong> · 另 ${softShare}% 为任务执行模型表达不确定(软信号,建议对照清单复核)`
1727
- : `<strong>实际盲区 ${weightedPct}%</strong> · 主要来自确定的搜索未命中`)
1728
- : (softShare >= 10
1729
- ? `<strong>real gaps ${weightedPct}%</strong> · another ${softShare}% is hedging (review list below)`
1730
- : `<strong>real gaps ${weightedPct}%</strong> · mostly confirmed search misses`);
1731
- const typeBadges = Object.keys(gap.byType)
1732
- .filter((key) => gap.byType[key] > 0)
1733
- .map((key) => `<span style="display:inline-block;padding:2px 8px;border-radius:var(--radius);background:var(--bg-card);font-size:var(--fs-micro);color:var(--text-secondary);margin:2px 4px 2px 0">${e(pickSignalLabel(key))} × ${gap.byType[key]}</span>`)
1734
- .join('');
1735
- const INVENTORY_CAP = 6;
1736
- // inventory 行用 border-left-color 按 signal severity 上色,不再重复标"类型"文字
1737
- const inventory = gap.signals.slice(0, INVENTORY_CAP).map((sig) => {
1738
- const severity = signalSeverity[sig.type];
1739
- const turnPart = sig.turn != null ? ` · ${lang === 'zh' ? '第' : 'turn'} ${sig.turn}${lang === 'zh' ? ' 轮' : ''}` : '';
1740
- return `<div class="ki-inventory-item" data-severity="${severity}">
1741
- <div class="ki-inventory-item-meta"><strong style="color:var(--text-secondary)">${e(sig.sampleId)}</strong>${turnPart}</div>
1742
- <div class="ki-inventory-item-ctx">${e(sig.context)}</div>
1743
- </div>`;
1744
- }).join('');
1745
- const overflowHint = gap.signals.length > INVENTORY_CAP
1746
- ? `<div style="font-size:var(--fs-micro);color:var(--text-muted);margin-top:6px">${lang === 'zh' ? `另 ${gap.signals.length - INVENTORY_CAP} 条未展示` : `+${gap.signals.length - INVENTORY_CAP} more not shown`}</div>`
1747
- : '';
1748
- const detailsLabel = lang === 'zh'
1749
- ? `展开 ${gap.signals.length} 条证据(按严重度上色: 红=确定 / 黄=执行模型自述 / 灰=犹豫)`
1750
- : `Show all ${gap.signals.length} evidence items (red=confirmed · yellow=self-flagged · gray=hedging)`;
1751
- gapInner = `
1752
- <div class="ki-col-header">
1753
- <span class="ki-col-title">${lang === 'zh' ? '知识盲区' : 'Knowledge gaps'}</span>
1754
- <span class="ki-col-value" style="color:${pctColor}">${pct}%</span>
1755
- </div>
1756
- <div class="ki-bar" role="progressbar" aria-valuenow="${pct}" aria-valuemin="0" aria-valuemax="100" aria-label="${e(barLabel)}">
1757
- <div class="ki-bar-fill" style="width:${barW}%;background:${pctColor}"></div>
1758
- </div>
1759
- <div style="font-size:11px;color:var(--text-muted);margin-bottom:4px">${gap.samplesWithGap}/${gap.sampleCount} ${lang === 'zh' ? '个评测用例出现搜索未命中或表达不确定' : 'samples with search miss / hedging'}</div>
1760
- <div style="font-size:var(--fs-detail);color:var(--text-secondary);margin-bottom:8px">${weightedHint}</div>
1761
- ${typeBadges ? `<div>${typeBadges}</div>` : ''}
1762
- ${inventory ? `<details class="ki-details"><summary>${detailsLabel}</summary>
1763
- ${inventory}
1764
- ${overflowHint}
1765
- </details>` : ''}`;
1766
- }
1767
- else {
1768
- gapInner = `<div style="color:var(--text-muted);font-size:12px">${lang === 'zh' ? '知识盲区数据不可用' : 'Gap data unavailable'}</div>`;
1769
- }
1770
- // ─── Watermark(spec §7.1):test set 标识 ───
1771
- const watermarkBits = [];
1772
- if (gap?.testSetPath) {
1773
- const basename = gap.testSetPath.split('/').pop() || gap.testSetPath;
1774
- watermarkBits.push(`<span style="font-family:ui-monospace,SFMono-Regular,Menlo,monospace">${e(basename)}</span>`);
1775
- }
1776
- if (gap)
1777
- watermarkBits.push(`n=${gap.sampleCount}`);
1778
- if (gap?.testSetHash)
1779
- watermarkBits.push(`sha:${e(gap.testSetHash)}`);
1780
- const watermark = watermarkBits.length > 0
1781
- ? `<div class="ki-card-meta">${watermarkBits.join(' · ')}</div>`
1782
- : '';
1783
- return `<div class="ki-card" style="border-left:3px solid ${variantColor}">
1784
- <div class="ki-card-header">
1785
- <span class="ki-card-title">${e(variant)}</span>
1786
- ${watermark}
1787
- </div>
1788
- <div class="ki-columns">
1789
- <div class="ki-col">${covInner}</div>
1790
- <div class="ki-col">${gapInner}</div>
1791
- </div>
1792
- </div>`;
1793
- }).join('');
1794
- return `
1795
- <section style="margin-top:28px;padding-top:20px;border-top:1px solid var(--border)">
1796
- <h2>${title}</h2>
1797
- <p class="ki-desc">${desc}</p>
1798
- <div class="ki-desc-hint">${readHint}</div>
1799
- ${cards}
1800
- </section>
1801
- `;
1802
- }
1803
- export function renderGapSection(gapReports, lang) {
1804
- if (!gapReports || Object.keys(gapReports).length === 0)
1805
- return '';
1806
- const title = lang === 'zh' ? '本次测评的知识盲区' : 'Knowledge gaps in this evaluation';
1807
- const desc = lang === 'zh'
1808
- ? '本次测评中,哪些知识想找但没找到、或任务执行模型表达不确定。数字高不一定代表知识库不全——也可能是评测用例问的领域知识库未覆盖。'
1809
- : 'Which knowledge the task execution model tried to find but missed, or expressed uncertainty about. High numbers do not necessarily mean the KB is incomplete — the test set may be asking about areas the KB never covered.';
1810
- const signalTypeLabels = {
1811
- failed_search: { zh: '搜索未命中', en: 'Search miss' },
1812
- explicit_marker: { zh: '任务执行模型标记缺口', en: 'Execution model flag' },
1813
- hedging: { zh: '表达不确定', en: 'Hedging' },
1814
- repeated_failure: { zh: '反复未命中', en: 'Repeated miss' },
1815
- };
1816
- const pickLabel = (key) => signalTypeLabels[key][lang === 'zh' ? 'zh' : 'en'];
1817
- const variantSections = Object.entries(gapReports).map(([variant, report]) => {
1818
- const pct = Math.round(report.gapRate * 100);
1819
- // Color: lower gap rate is better (inverse of coverage). Gate at 10% / 30%.
1820
- const pctColor = pct <= 10 ? 'var(--green)' : pct <= 30 ? 'var(--yellow)' : 'var(--red)';
1821
- const barW = Math.max(2, pct);
1822
- // v0.2 加权严重度 (spec §6):weightedGapRate 按用例最强信号权重聚合,总是 ≤ gapRate,
1823
- // 差值反映"软信号(hedging / explicit_marker)占比"——若差值大,说明 gap_rate 被软信号
1824
- // 拉高,读者该复核弱信号的真实含义。若差值小,说明信号以硬证据为主,结论可信度高。
1825
- const weightedPct = Math.round(report.weightedGapRate * 100);
1826
- const softSignalShare = pct - weightedPct;
1827
- const weightedHint = lang === 'zh'
1828
- ? (softSignalShare >= 10
1829
- ? `<strong>实际盲区 ${weightedPct}%</strong> · 另外 ${softSignalShare}% 为任务执行模型表达不确定(软信号,建议对照右侧清单复核)`
1830
- : `<strong>实际盲区 ${weightedPct}%</strong> · 主要来自确定的搜索未命中`)
1831
- : (softSignalShare >= 10
1832
- ? `<strong>real gaps ${weightedPct}%</strong> · another ${softSignalShare}% is hedging (soft signals — review the list on the right)`
1833
- : `<strong>real gaps ${weightedPct}%</strong> · mostly from confirmed search misses`);
1834
- // Watermark (spec §7.1): test set path + sample count + hash + explicit caveat
1835
- const watermarkBits = [];
1836
- if (report.testSetPath) {
1837
- const basename = report.testSetPath.split('/').pop() || report.testSetPath;
1838
- watermarkBits.push(`<span style="font-family:ui-monospace,SFMono-Regular,Menlo,monospace">${e(basename)}</span>`);
1839
- }
1840
- watermarkBits.push(`n=${report.sampleCount}`);
1841
- if (report.testSetHash)
1842
- watermarkBits.push(`sha:${e(report.testSetHash)}`);
1843
- const watermark = `<div style="font-size:var(--fs-micro);color:var(--text-muted);margin-bottom:8px;font-weight:400">${watermarkBits.join(' · ')}</div>`;
1844
- // Classification: per-type counts
1845
- const typeBadges = Object.keys(report.byType)
1846
- .filter((key) => report.byType[key] > 0)
1847
- .map((key) => `<span style="display:inline-block;padding:2px 8px;border-radius:var(--radius);background:var(--bg-card);font-size:var(--fs-micro);color:var(--text-secondary);margin:2px 4px 2px 0">${e(pickLabel(key))} × ${report.byType[key]}</span>`)
1848
- .join('');
1849
- // Inventory: list of specific signals (cap at 8 to keep the panel compact)
1850
- const INVENTORY_CAP = 8;
1851
- const inventory = report.signals.slice(0, INVENTORY_CAP).map((sig) => {
1852
- const typeLabel = pickLabel(sig.type);
1853
- const turnPart = sig.turn != null ? ` / ${lang === 'zh' ? '第' : 'turn'} ${sig.turn}${lang === 'zh' ? ' 轮' : ''}` : '';
1854
- return `<div style="padding:6px 10px;margin:4px 0;background:var(--bg-card);border-left:2px solid var(--border-hover);border-radius:4px;font-size:var(--fs-detail);line-height:1.5">
1855
- <div style="color:var(--text-muted);font-size:var(--fs-micro);margin-bottom:2px">
1856
- <strong style="color:var(--text-secondary)">${e(sig.sampleId)}</strong>${turnPart} · ${e(typeLabel)}
1857
- </div>
1858
- <div style="color:var(--text-secondary);word-break:break-all">${e(sig.context)}</div>
1859
- </div>`;
1860
- }).join('');
1861
- const overflowHint = report.signals.length > INVENTORY_CAP
1862
- ? `<div style="font-size:var(--fs-micro);color:var(--text-muted);margin-top:6px">${lang === 'zh' ? `还有 ${report.signals.length - INVENTORY_CAP} 条未展示` : `+${report.signals.length - INVENTORY_CAP} more not shown`}</div>`
1863
- : '';
1864
- // caveat 已从底部移除:副标题已经讲清楚"撞墙多不等于知识库不全"。重复说一遍反而稀释主信号。
1865
- return `<div style="flex:1;min-width:320px;padding:16px;background:var(--bg-surface);border:1px solid var(--border);border-radius:var(--radius)">
1866
- <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:4px">
1867
- <strong>${e(variant)}</strong>
1868
- <span style="font-size:20px;font-weight:600;color:${pctColor}">${pct}%</span>
1869
- </div>
1870
- ${watermark}
1871
- <div style="height:8px;background:var(--bg-card);border-radius:4px;margin-bottom:10px">
1872
- <div style="width:${barW}%;height:100%;background:${pctColor};border-radius:4px"></div>
1873
- </div>
1874
- <div style="font-size:var(--fs-detail);color:var(--text-muted);margin-bottom:4px">
1875
- ${report.samplesWithGap} / ${report.sampleCount} ${lang === 'zh' ? '个评测用例出现搜索未命中或表达不确定' : 'samples with search miss or hedging'}
1876
- </div>
1877
- <div style="font-size:var(--fs-detail);color:var(--text-secondary);margin-bottom:10px">${weightedHint}</div>
1878
- ${typeBadges ? `<div style="margin-bottom:10px">${typeBadges}</div>` : ''}
1879
- ${inventory ? `<div style="margin-top:10px;padding-top:10px;border-top:1px solid var(--border)">
1880
- <div style="font-size:var(--fs-micro);color:var(--text-muted);margin-bottom:4px">${lang === 'zh' ? '具体哪些知识未命中' : 'Missed knowledge inventory'}</div>
1881
- ${inventory}
1882
- ${overflowHint}
1883
- </div>` : ''}
1884
- </div>`;
1885
- }).join('');
1886
- return `
1887
- <section style="margin-top:24px">
1888
- <h2>${title}</h2>
1889
- <p style="font-size:var(--fs-detail);color:var(--text-muted);margin-bottom:12px">${desc}</p>
1890
- <div style="display:flex;gap:16px;flex-wrap:wrap">
1891
- ${variantSections}
1892
- </div>
1893
- </section>
1894
- `;
1895
- }
1896
- //# sourceMappingURL=summary.js.map