oh-my-knowledge 0.47.0 → 0.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. package/README.md +50 -18
  2. package/README.zh.md +55 -23
  3. package/dist/analysis/coverage-analyzer.d.ts +1 -0
  4. package/dist/analysis/coverage-analyzer.js +125 -62
  5. package/dist/analysis/failure-clusterer.js +2 -1
  6. package/dist/analysis/gap-analyzer.d.ts +2 -2
  7. package/dist/analysis/gap-analyzer.js +13 -3
  8. package/dist/analysis/hedging-classifier.d.ts +2 -2
  9. package/dist/analysis/hedging-classifier.js +3 -4
  10. package/dist/analysis/report-diagnostics.js +9 -7
  11. package/dist/analysis/sample-diagnostics.js +6 -6
  12. package/dist/artifact-graph/doctor.js +15 -7
  13. package/dist/assets/agent-skills/omk/SKILL.md +27 -7
  14. package/dist/assets/agent-skills/omk/references/commands.md +19 -17
  15. package/dist/authoring/evolver.d.ts +10 -6
  16. package/dist/authoring/evolver.js +496 -83
  17. package/dist/authoring/generator.d.ts +3 -3
  18. package/dist/authoring/generator.js +5 -10
  19. package/dist/authoring/sample-fixer.d.ts +8 -6
  20. package/dist/authoring/sample-fixer.js +76 -5
  21. package/dist/cli/commands/doctor.js +31 -14
  22. package/dist/cli/commands/eval/index.d.ts +3 -0
  23. package/dist/cli/commands/eval/index.js +163 -17
  24. package/dist/cli/commands/evolve.d.ts +4 -4
  25. package/dist/cli/commands/evolve.js +27 -13
  26. package/dist/cli/commands/init.js +16 -3
  27. package/dist/cli/commands/observe/inbox.js +44 -21
  28. package/dist/cli/commands/observe/index.js +20 -11
  29. package/dist/cli/commands/observe/ingest.d.ts +3 -0
  30. package/dist/cli/commands/observe/ingest.js +35 -4
  31. package/dist/cli/commands/sample.d.ts +9 -3
  32. package/dist/cli/commands/sample.js +91 -74
  33. package/dist/cli/lib/cmd-flags.d.ts +1 -0
  34. package/dist/cli/lib/codex-model-hint.d.ts +9 -0
  35. package/dist/cli/lib/codex-model-hint.js +45 -0
  36. package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
  37. package/dist/cli/lib/generation-failure-hint.js +61 -0
  38. package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
  39. package/dist/cli/lib/i18n-dict/common.js +4 -0
  40. package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
  41. package/dist/cli/lib/i18n-dict/gen.js +38 -6
  42. package/dist/cli/lib/i18n-dict/help.js +6 -6
  43. package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
  44. package/dist/cli/lib/i18n-dict/init.js +13 -9
  45. package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
  46. package/dist/cli/lib/i18n-dict/run.js +34 -2
  47. package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
  48. package/dist/cli/lib/llm-failure-classifier.js +8 -0
  49. package/dist/cli/lib/parse-run-config.d.ts +6 -5
  50. package/dist/cli/lib/parse-run-config.js +16 -9
  51. package/dist/cli/lib/runtime-defaults.d.ts +21 -0
  52. package/dist/cli/lib/runtime-defaults.js +79 -0
  53. package/dist/diagnosis/observe-mapper.js +14 -15
  54. package/dist/diagnosis/observe-producer.js +3 -1
  55. package/dist/diagnosis/studio-projection.js +14 -7
  56. package/dist/diagnosis/types.d.ts +2 -0
  57. package/dist/diagnosis/types.js +12 -0
  58. package/dist/doctor/endpoint-rule.js +2 -1
  59. package/dist/eval-core/artifact-file-names.js +18 -1
  60. package/dist/eval-core/artifact-index.d.ts +7 -11
  61. package/dist/eval-core/artifact-index.js +139 -80
  62. package/dist/eval-core/cache.d.ts +12 -3
  63. package/dist/eval-core/cache.js +89 -29
  64. package/dist/eval-core/comparability.js +10 -6
  65. package/dist/eval-core/evaluation-execution.d.ts +2 -1
  66. package/dist/eval-core/evaluation-execution.js +122 -37
  67. package/dist/eval-core/evaluation-job.d.ts +4 -1
  68. package/dist/eval-core/evaluation-job.js +4 -1
  69. package/dist/eval-core/evaluation-reporting.d.ts +15 -13
  70. package/dist/eval-core/evaluation-reporting.js +54 -52
  71. package/dist/eval-core/execution-strategy.d.ts +2 -0
  72. package/dist/eval-core/execution-strategy.js +11 -9
  73. package/dist/eval-core/fact-checker.js +15 -7
  74. package/dist/eval-core/holdout.js +3 -2
  75. package/dist/eval-core/judge-independence.d.ts +2 -2
  76. package/dist/eval-core/mock-hook.cjs +23 -6
  77. package/dist/eval-core/mocks-runtime.js +30 -8
  78. package/dist/eval-core/report-document.d.ts +12 -0
  79. package/dist/eval-core/report-document.js +1151 -0
  80. package/dist/eval-core/report-extensions.d.ts +4 -0
  81. package/dist/eval-core/report-extensions.js +500 -0
  82. package/dist/eval-core/report-file-migration.js +7 -2
  83. package/dist/eval-core/resume-compatibility.d.ts +31 -0
  84. package/dist/eval-core/resume-compatibility.js +141 -0
  85. package/dist/eval-core/sample-fingerprint.d.ts +12 -0
  86. package/dist/eval-core/sample-fingerprint.js +193 -0
  87. package/dist/eval-core/schema.js +86 -31
  88. package/dist/eval-core/verdict.d.ts +8 -4
  89. package/dist/eval-core/verdict.js +24 -10
  90. package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
  91. package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
  92. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
  93. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
  94. package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
  95. package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
  96. package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
  97. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
  98. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
  99. package/dist/eval-workflows/evaluation-pipeline.js +12 -7
  100. package/dist/eval-workflows/run-evaluation.d.ts +9 -7
  101. package/dist/eval-workflows/run-evaluation.js +79 -51
  102. package/dist/executors/anthropic-api.js +65 -9
  103. package/dist/executors/claude-cli.js +16 -79
  104. package/dist/executors/claude-protocol.d.ts +28 -0
  105. package/dist/executors/claude-protocol.js +180 -0
  106. package/dist/executors/claude-sdk-trace.js +56 -28
  107. package/dist/executors/claude-sdk.d.ts +1 -0
  108. package/dist/executors/claude-sdk.js +39 -93
  109. package/dist/executors/codex-cli-trace.js +166 -31
  110. package/dist/executors/codex-cli.d.ts +6 -8
  111. package/dist/executors/codex-cli.js +49 -151
  112. package/dist/executors/codex-protocol.d.ts +24 -0
  113. package/dist/executors/codex-protocol.js +234 -0
  114. package/dist/executors/codex-sdk.js +68 -120
  115. package/dist/executors/gemini.js +88 -13
  116. package/dist/executors/index.d.ts +2 -3
  117. package/dist/executors/index.js +5 -3
  118. package/dist/executors/openai-api.js +70 -9
  119. package/dist/executors/runtime-fingerprint.js +88 -11
  120. package/dist/executors/script-command.d.ts +8 -0
  121. package/dist/executors/script-command.js +87 -0
  122. package/dist/executors/script.js +202 -29
  123. package/dist/executors/shared.d.ts +35 -3
  124. package/dist/executors/shared.js +113 -15
  125. package/dist/grading/assertions.d.ts +1 -1
  126. package/dist/grading/assertions.js +19 -9
  127. package/dist/grading/diagnostic.d.ts +9 -2
  128. package/dist/grading/diagnostic.js +25 -2
  129. package/dist/grading/index.js +10 -4
  130. package/dist/grading/judge.js +19 -6
  131. package/dist/grading/layered-scores.d.ts +2 -3
  132. package/dist/grading/layered-scores.js +2 -3
  133. package/dist/inputs/load-samples.d.ts +1 -2
  134. package/dist/inputs/load-samples.js +23 -1
  135. package/dist/inputs/mcp-resolver.js +6 -3
  136. package/dist/inputs/sample-document.d.ts +11 -0
  137. package/dist/inputs/sample-document.js +96 -0
  138. package/dist/managed/evidence.d.ts +1 -0
  139. package/dist/managed/evidence.js +1 -1
  140. package/dist/managed/store.js +200 -91
  141. package/dist/observability/codex-trace-adapter.d.ts +5 -0
  142. package/dist/observability/codex-trace-adapter.js +850 -0
  143. package/dist/observability/experience.d.ts +32 -6
  144. package/dist/observability/experience.js +2695 -459
  145. package/dist/observability/feedback-matchers.js +16 -1
  146. package/dist/observability/inbox-view-model.d.ts +2 -1
  147. package/dist/observability/inbox-view-model.js +20 -14
  148. package/dist/observability/inbox.d.ts +7 -1
  149. package/dist/observability/inbox.js +632 -124
  150. package/dist/observability/problem-patterns.js +2 -0
  151. package/dist/observability/review-state.d.ts +6 -0
  152. package/dist/observability/review-state.js +235 -63
  153. package/dist/observability/skill-chain-advisories.js +1 -1
  154. package/dist/observability/skill-chain.js +17 -4
  155. package/dist/observability/skill-health-analyzer.d.ts +32 -7
  156. package/dist/observability/skill-health-analyzer.js +194 -121
  157. package/dist/observability/skill-health-report.d.ts +10 -0
  158. package/dist/observability/skill-health-report.js +620 -0
  159. package/dist/observability/soft-standards/constants.d.ts +0 -1
  160. package/dist/observability/soft-standards/constants.js +0 -1
  161. package/dist/observability/soft-standards/index.d.ts +1 -1
  162. package/dist/observability/soft-standards/index.js +1 -1
  163. package/dist/observability/soft-standards/llm-extractor.js +8 -10
  164. package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
  165. package/dist/observability/soft-standards/skill-standards-store.js +59 -18
  166. package/dist/observability/soft-standards/types.d.ts +2 -2
  167. package/dist/observability/trace-adapter.d.ts +12 -7
  168. package/dist/observability/trace-adapter.js +11 -9
  169. package/dist/observability/trace-attribution.d.ts +13 -5
  170. package/dist/observability/trace-attribution.js +315 -21
  171. package/dist/observability/trace-ingestion.d.ts +9 -0
  172. package/dist/observability/trace-ingestion.js +80 -0
  173. package/dist/observability/trace-ir.d.ts +113 -0
  174. package/dist/observability/trace-ir.js +87 -0
  175. package/dist/observability/trace-segmenter.d.ts +19 -6
  176. package/dist/observability/trace-segmenter.js +377 -196
  177. package/dist/observability/trace-session-index.d.ts +19 -0
  178. package/dist/observability/trace-session-index.js +68 -0
  179. package/dist/observability/trace-source.d.ts +12 -4
  180. package/dist/observability/trace-source.js +939 -215
  181. package/dist/renderer/html-renderer.js +37 -6
  182. package/dist/renderer/icons.js +3 -0
  183. package/dist/renderer/observation-inbox-renderer.js +227 -91
  184. package/dist/renderer/skill-detail-renderer.js +452 -109
  185. package/dist/renderer/skill-health-renderer.js +69 -12
  186. package/dist/renderer/summary.js +28 -7
  187. package/dist/renderer/table.js +21 -4
  188. package/dist/renderer/test-view.d.ts +1 -0
  189. package/dist/renderer/test-view.js +44 -9
  190. package/dist/server/indexed-report-store.js +14 -18
  191. package/dist/server/job-store.js +64 -26
  192. package/dist/server/report-server.js +190 -78
  193. package/dist/server/report-store.js +57 -80
  194. package/dist/server/skill-index.js +143 -49
  195. package/dist/server/skill-insights.js +44 -5
  196. package/dist/shared/artifact-graph.d.ts +3 -0
  197. package/dist/shared/artifact-graph.js +224 -0
  198. package/dist/shared/assertion-types.d.ts +8 -0
  199. package/dist/shared/assertion-types.js +46 -0
  200. package/dist/shared/atomic-json.d.ts +8 -0
  201. package/dist/shared/atomic-json.js +33 -0
  202. package/dist/shared/diagnosis-schema.d.ts +9 -0
  203. package/dist/shared/diagnosis-schema.js +181 -0
  204. package/dist/shared/doctor-report.d.ts +3 -0
  205. package/dist/shared/doctor-report.js +103 -0
  206. package/dist/shared/evaluation-job.d.ts +6 -0
  207. package/dist/shared/evaluation-job.js +217 -0
  208. package/dist/shared/executor-result.d.ts +17 -0
  209. package/dist/shared/executor-result.js +221 -0
  210. package/dist/shared/file-lock.d.ts +12 -0
  211. package/dist/shared/file-lock.js +129 -0
  212. package/dist/shared/json-value.d.ts +5 -0
  213. package/dist/shared/json-value.js +36 -0
  214. package/dist/shared/keyed-mutex.d.ts +7 -0
  215. package/dist/shared/keyed-mutex.js +24 -0
  216. package/dist/shared/record-count.d.ts +8 -0
  217. package/dist/shared/record-count.js +43 -0
  218. package/dist/shared/sample-contract.d.ts +3 -0
  219. package/dist/shared/sample-contract.js +332 -0
  220. package/dist/shared/shell-quote.d.ts +2 -0
  221. package/dist/shared/shell-quote.js +7 -0
  222. package/dist/shared/timestamp.d.ts +6 -0
  223. package/dist/shared/timestamp.js +64 -0
  224. package/dist/shared/token-usage.d.ts +19 -0
  225. package/dist/shared/token-usage.js +50 -0
  226. package/dist/shared/tool-call-status.d.ts +8 -0
  227. package/dist/shared/tool-call-status.js +28 -0
  228. package/dist/shared/tool-identity.d.ts +21 -0
  229. package/dist/shared/tool-identity.js +84 -0
  230. package/dist/shared/tool-search.js +73 -16
  231. package/dist/shared/trace-projection.d.ts +5 -0
  232. package/dist/shared/trace-projection.js +20 -0
  233. package/dist/shared/trace-source-kind.d.ts +3 -0
  234. package/dist/shared/trace-source-kind.js +12 -0
  235. package/dist/types/diagnosis.d.ts +2 -0
  236. package/dist/types/eval.d.ts +4 -0
  237. package/dist/types/executor.d.ts +32 -5
  238. package/dist/types/index.d.ts +1 -0
  239. package/dist/types/index.js +1 -0
  240. package/dist/types/judge.d.ts +2 -0
  241. package/dist/types/observability.d.ts +116 -9
  242. package/dist/types/report.d.ts +58 -6
  243. package/dist/types/skill-index.d.ts +7 -0
  244. package/dist/types/trace.d.ts +2 -0
  245. package/dist/types/trace.js +1 -0
  246. package/package.json +9 -5
@@ -0,0 +1,84 @@
1
+ const BUILTIN_ALIASES = new Map([
2
+ ['bash', 'Bash'],
3
+ ['shell', 'Bash'],
4
+ ['exec_command', 'Bash'],
5
+ ['command_execution', 'Bash'],
6
+ ['read', 'Read'],
7
+ ['file_read', 'Read'],
8
+ ['grep', 'Grep'],
9
+ ['edit', 'Edit'],
10
+ ['apply_patch', 'Edit'],
11
+ ['file_change', 'Edit'],
12
+ ['write', 'Write'],
13
+ ['file_write', 'Write'],
14
+ ['view_image', 'ViewImage'],
15
+ ['viewimage', 'ViewImage'],
16
+ ['write_stdin', 'WriteStdin'],
17
+ ['writestdin', 'WriteStdin'],
18
+ ['web_search', 'WebSearch'],
19
+ ['websearch', 'WebSearch'],
20
+ ]);
21
+ /**
22
+ * Normalize runtime-specific tool labels into one comparison namespace while
23
+ * retaining source identity for audit and future protocol migrations.
24
+ */
25
+ export function normalizeToolIdentity(input) {
26
+ const sourceName = input.sourceName.trim() || 'unknown';
27
+ const namespace = nonEmpty(input.namespace) ?? inferredMcpNamespace(sourceName);
28
+ const authoritativeName = nonEmpty(input.authoritativeName);
29
+ const isMcp = Boolean(authoritativeName
30
+ || sourceName.startsWith('mcp__')
31
+ || namespace?.startsWith('mcp__'));
32
+ if (isMcp) {
33
+ const sourceLeaf = sourceName.split('__').filter(Boolean).at(-1) ?? sourceName;
34
+ const leaf = authoritativeName ?? sourceLeaf;
35
+ const provider = nonEmpty(input.provider) ?? providerFromNamespace(namespace);
36
+ const name = provider && !leaf.startsWith(`${provider}.`)
37
+ ? `${provider}.${leaf}`
38
+ : leaf;
39
+ return {
40
+ name,
41
+ ...(sourceName !== name ? { sourceName } : {}),
42
+ ...(namespace ? { namespace } : {}),
43
+ ...(provider ? { provider } : {}),
44
+ displayName: name,
45
+ };
46
+ }
47
+ if (namespace) {
48
+ const name = sourceName.startsWith(`${namespace}.`)
49
+ ? sourceName
50
+ : `${namespace}.${sourceName}`;
51
+ return {
52
+ name,
53
+ ...(sourceName !== name ? { sourceName } : {}),
54
+ namespace,
55
+ provider: nonEmpty(input.provider) ?? namespace,
56
+ displayName: name,
57
+ };
58
+ }
59
+ const name = BUILTIN_ALIASES.get(sourceName.toLowerCase()) ?? sourceName;
60
+ return {
61
+ name,
62
+ ...(sourceName !== name ? { sourceName } : {}),
63
+ };
64
+ }
65
+ function inferredMcpNamespace(sourceName) {
66
+ const parts = sourceName.split('__').filter(Boolean);
67
+ return parts[0] === 'mcp' && parts.length > 2
68
+ ? parts.slice(0, -1).join('__')
69
+ : undefined;
70
+ }
71
+ function providerFromNamespace(namespace) {
72
+ if (!namespace)
73
+ return undefined;
74
+ const parts = namespace.replace(/^mcp__/, '').split('__').filter(Boolean);
75
+ // codex_apps is a transport namespace; the following component is the
76
+ // actual connector/provider and matches Codex MCP end-event metadata.
77
+ if (parts[0] === 'codex_apps' && parts.length > 1)
78
+ parts.shift();
79
+ return parts.length > 0 ? parts.join('.') : undefined;
80
+ }
81
+ function nonEmpty(value) {
82
+ const normalized = value?.trim();
83
+ return normalized ? normalized : undefined;
84
+ }
@@ -1,11 +1,19 @@
1
+ import { isToolCallCancelled, isToolCallFailure, isToolCallSuccess } from './tool-call-status.js';
1
2
  export function toolCallQuery(tc) {
2
3
  const input = (tc.input && typeof tc.input === 'object') ? tc.input : {};
4
+ const legacyInput = typeof tc.input === 'string' ? tc.input : undefined;
3
5
  if (tc.tool === 'Grep')
4
6
  return { query: String(input.pattern ?? ''), path: String(input.path ?? '') };
5
7
  if (tc.tool === 'Read')
6
- return { path: String(input.file_path ?? '') };
8
+ return { path: String(input.file_path ?? legacyInput ?? '') };
7
9
  if (tc.tool === 'Bash')
8
- return bashStructuredQuery(String(input.command ?? ''));
10
+ return bashStructuredQuery(String(input.command ?? legacyInput ?? ''));
11
+ if (isWebSearchTool(tc.tool)) {
12
+ return {
13
+ query: typeof input.query === 'string' ? input.query : legacyInput,
14
+ path: typeof input.url === 'string' ? input.url : undefined,
15
+ };
16
+ }
9
17
  return {};
10
18
  }
11
19
  /**
@@ -57,15 +65,46 @@ function bashStructuredQuery(command) {
57
65
  const findName = /\bfind\b[^|;&]*?-(?:i?name|path|wholename)\s+['"]?([^'"\s]+)/.exec(command);
58
66
  if (findName)
59
67
  return { query: findName[1] };
60
- // ls / cat / test / head / tail / wc / file / stat / du / tree → path-only。
61
- // 路径首字符限制 [\w./~]:
62
- // - 排除 flag (`-` 开头) 防 `npm run test --watch` 把 `--watch` 当 path
63
- // - 允许字母/数字/`.`/`/`/`~` 开头的 path token (foo / ./foo / /foo / ~/foo)
64
- const pathOnly = /\b(?:ls|cat|test|head|tail|wc|file|stat|du|tree)\b\s+(?:-\w+\s+)*([\w./~][^\s|;&]*)/.exec(command);
65
- if (pathOnly)
66
- return { path: pathOnly[1] };
68
+ const path = shellReaderPath(command);
69
+ if (path)
70
+ return { path };
67
71
  return {};
68
72
  }
73
+ function shellReaderPath(command) {
74
+ const match = /\b(ls|cat|test|head|tail|wc|file|stat|du|tree|sed|awk|less|more|bat)\b\s+([^|;&]+)/.exec(command);
75
+ if (!match)
76
+ return undefined;
77
+ const tool = match[1];
78
+ const tokens = shellWords(match[2]);
79
+ const positional = [];
80
+ for (let index = 0; index < tokens.length; index += 1) {
81
+ const token = tokens[index];
82
+ if (!token.startsWith('-') || token === '-') {
83
+ positional.push(token);
84
+ continue;
85
+ }
86
+ if ((tool === 'head' || tool === 'tail')
87
+ && (token === '-n' || token === '-c')
88
+ && index + 1 < tokens.length) {
89
+ index += 1;
90
+ continue;
91
+ }
92
+ if (tool === 'sed' && (token === '-e' || token === '-f') && index + 1 < tokens.length) {
93
+ index += 1;
94
+ }
95
+ }
96
+ const candidate = tool === 'sed' || tool === 'awk'
97
+ ? positional.at(-1)
98
+ : positional[0];
99
+ if (!candidate || /^\d+$/.test(candidate))
100
+ return undefined;
101
+ return /^[\w./~]/.test(candidate) ? candidate : undefined;
102
+ }
103
+ function shellWords(value) {
104
+ return [...value.matchAll(/"((?:\\.|[^"])*)"|'((?:\\.|[^'])*)'|([^\s]+)/g)]
105
+ .map((match) => match[1] ?? match[2] ?? match[3])
106
+ .filter((token) => Boolean(token));
107
+ }
69
108
  export function bashSearchOrProbeCommand(command) {
70
109
  if (/\b(grep|rg|find)\b/.test(command))
71
110
  return true;
@@ -76,28 +115,46 @@ export function bashSearchOrProbeCommand(command) {
76
115
  export function isSearchToolCall(tc) {
77
116
  if (tc.tool === 'Read' || tc.tool === 'Grep')
78
117
  return true;
118
+ if (isWebSearchTool(tc.tool))
119
+ return true;
79
120
  if (tc.tool !== 'Bash')
80
121
  return false;
81
122
  const input = (tc.input && typeof tc.input === 'object') ? tc.input : {};
82
- return bashSearchOrProbeCommand(String(input.command ?? ''));
123
+ const command = input.command ?? (typeof tc.input === 'string' ? tc.input : '');
124
+ return bashSearchOrProbeCommand(String(command));
83
125
  }
84
126
  export function isFailedSearchToolCall(tc) {
85
127
  const output = typeof tc.output === 'string' ? tc.output : String(tc.output ?? '');
86
- const emptyOutput = output.trim() === '' || /No matches found/i.test(output);
128
+ // Runtime cancellation is authoritative. Some runtimes retain partial or
129
+ // stale output text on cancellation; it must not become knowledge-gap evidence.
130
+ if (isToolCallCancelled(tc))
131
+ return false;
132
+ const explicitlyEmpty = /No matches found/i.test(output);
133
+ // An unresolved call also has an empty output. That is missing evidence,
134
+ // not evidence that a search completed with zero matches.
135
+ const completedEmpty = output.trim() === '' && isToolCallSuccess(tc);
87
136
  if (tc.tool === 'Read') {
88
- return tc.success === false;
137
+ return isToolCallFailure(tc);
89
138
  }
90
139
  if (tc.tool === 'Grep') {
91
- if (tc.success === false)
140
+ if (isToolCallFailure(tc))
92
141
  return true;
93
- return emptyOutput;
142
+ return explicitlyEmpty || completedEmpty;
143
+ }
144
+ if (isWebSearchTool(tc.tool)) {
145
+ return isToolCallFailure(tc);
94
146
  }
95
147
  if (tc.tool === 'Bash') {
96
148
  if (!isSearchToolCall(tc))
97
149
  return false;
98
- if (tc.success === false)
150
+ if (isToolCallFailure(tc))
99
151
  return true;
100
- return emptyOutput;
152
+ return explicitlyEmpty || completedEmpty;
101
153
  }
102
154
  return false;
103
155
  }
156
+ function isWebSearchTool(tool) {
157
+ // `WebSearch` is the Trace IR canonical identity. Keep the old persisted spelling
158
+ // readable because report schemas before v5 could contain runtime-native names.
159
+ return tool === 'WebSearch' || tool === 'web_search';
160
+ }
@@ -0,0 +1,5 @@
1
+ import type { ToolCallInfo, TurnInfo } from '../types/index.js';
2
+ export declare const MAX_PERSISTED_TURN_CONTENT = 2000;
3
+ export declare const MAX_PERSISTED_TOOL_OUTPUT = 1000;
4
+ export declare function truncateToolCallsForPersistence(toolCalls: ToolCallInfo[]): ToolCallInfo[];
5
+ export declare function truncateTurnsForPersistence(turns: TurnInfo[]): TurnInfo[];
@@ -0,0 +1,20 @@
1
+ import { safeSliceForJson } from '../util/safe-slice.js';
2
+ export const MAX_PERSISTED_TURN_CONTENT = 2000;
3
+ export const MAX_PERSISTED_TOOL_OUTPUT = 1000;
4
+ export function truncateToolCallsForPersistence(toolCalls) {
5
+ return toolCalls.map((toolCall) => ({
6
+ ...toolCall,
7
+ output: typeof toolCall.output === 'string'
8
+ ? safeSliceForJson(toolCall.output, MAX_PERSISTED_TOOL_OUTPUT)
9
+ : toolCall.output,
10
+ }));
11
+ }
12
+ export function truncateTurnsForPersistence(turns) {
13
+ return turns.map((turn) => ({
14
+ ...turn,
15
+ content: safeSliceForJson(turn.content, MAX_PERSISTED_TURN_CONTENT),
16
+ ...(turn.toolCalls && {
17
+ toolCalls: truncateToolCallsForPersistence(turn.toolCalls),
18
+ }),
19
+ }));
20
+ }
@@ -0,0 +1,3 @@
1
+ import type { TraceSourceKind } from '../types/trace.js';
2
+ /** Runtime validator for the persisted Trace IR source identity protocol. */
3
+ export declare function isTraceSourceKind(value: unknown): value is TraceSourceKind;
@@ -0,0 +1,12 @@
1
+ const TRACE_SOURCE_KINDS = new Set([
2
+ 'claude',
3
+ 'codex',
4
+ 'openclaw',
5
+ 'markdown_log',
6
+ 'unknown',
7
+ ]);
8
+ /** Runtime validator for the persisted Trace IR source identity protocol. */
9
+ export function isTraceSourceKind(value) {
10
+ return typeof value === 'string'
11
+ && TRACE_SOURCE_KINDS.has(value);
12
+ }
@@ -12,6 +12,7 @@ export interface DiagnosisScope {
12
12
  primary: 'skill' | 'definition' | 'session' | 'sample';
13
13
  refs: {
14
14
  skillName: string;
15
+ traceId?: string;
15
16
  sessionId?: string;
16
17
  invocationId?: string;
17
18
  sampleId?: string;
@@ -23,6 +24,7 @@ export interface DiagnosisScope {
23
24
  export interface DiagnosisEvidenceRef {
24
25
  id: string;
25
26
  kind: string;
27
+ traceId?: string;
26
28
  sourceTrace?: string;
27
29
  sessionId?: string;
28
30
  messageIndex?: number;
@@ -305,6 +305,10 @@ export interface EvaluationRequest {
305
305
  strictBaseline?: boolean;
306
306
  /** Reasoning effort for executor LLM。透传到 ExecutorInput.effort。 */
307
307
  effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max';
308
+ /** 每个执行任务失败后的最大重试次数。缺省 / 0 表示不重试。 */
309
+ retry?: number;
310
+ /** true 时关闭 assertion 失败后的诊断 LLM 调用。缺省等价于 false。 */
311
+ noDiagnostic?: boolean;
308
312
  }
309
313
  export type EvaluationJobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled';
310
314
  export type EvaluationErrorCategory = 'user' | 'executor' | 'judge' | 'system';
@@ -1,19 +1,35 @@
1
+ import type { TraceSourceKind } from './trace.js';
2
+ export type ToolCallStatus = 'success' | 'failure' | 'cancelled' | 'unknown';
3
+ export type ToolCallStatusSource = 'runtime' | 'tool-output' | 'inferred' | 'unknown';
1
4
  export interface ToolCallInfo {
5
+ /** Source-neutral tool identity used by assertions and aggregate reports. */
2
6
  tool: string;
7
+ /** Runtime-native identity retained when `tool` was normalized. */
8
+ sourceTool?: string;
9
+ toolNamespace?: string;
10
+ toolProvider?: string;
3
11
  input: unknown;
4
12
  output: unknown;
13
+ /**
14
+ * Source-neutral completion state. Older reports only have `success`; readers
15
+ * must fall back to that boolean when this field is absent.
16
+ */
17
+ status?: ToolCallStatus;
18
+ statusSource?: ToolCallStatusSource;
5
19
  success: boolean;
6
20
  messageIndex?: number;
7
21
  messageUuid?: string;
22
+ /** Source-neutral identity for one concrete call occurrence. */
23
+ callInstanceId?: string;
8
24
  toolUseId?: string;
9
25
  timestamp?: string;
10
26
  sourceTrace?: string;
11
- sourceKind?: 'claude' | 'openclaw' | 'markdown_log' | 'unknown';
27
+ sourceKind?: TraceSourceKind;
12
28
  traceRole?: 'standalone' | 'main' | 'subagent';
13
29
  traceLabel?: string;
14
30
  }
15
31
  export interface TurnInfo {
16
- role: 'assistant' | 'tool';
32
+ role: 'user' | 'assistant' | 'tool';
17
33
  content: string;
18
34
  toolCalls?: ToolCallInfo[];
19
35
  durationMs?: number;
@@ -27,14 +43,23 @@ export interface ExecResult {
27
43
  outputTokens: number;
28
44
  cacheReadTokens: number;
29
45
  cacheCreationTokens: number;
46
+ /**
47
+ * Whether the token counters came from the underlying runtime.
48
+ * `false` means the numeric fields are compatibility placeholders and must
49
+ * not be treated as measured zero. Missing/true preserves legacy executors.
50
+ */
51
+ tokenUsageReportedByExecutor?: boolean;
30
52
  costUSD: number;
31
53
  /** Whether the underlying executor binary/SDK reported a USD cost figure.
32
- * - undefined / true : `costUSD` is authoritative (default — Anthropic SDK / OpenAI API / etc 都报)
33
- * - false : executor 不报 cost,`costUSD` 是占位 0,renderer 应显示「未报告」/「—」。
34
- * 目前只有 codex-cli executor 设 false (codex 0.125 binary 不输出 cost)。 */
54
+ * - undefined / true : `costUSD` is authoritative
55
+ * - false : executor 不报 cost,`costUSD` 是占位 0,renderer 应显示「未报告」/「—」。 */
35
56
  costReportedByExecutor?: boolean;
36
57
  stopReason: string;
37
58
  numTurns: number;
59
+ /** Number of executor attempts represented by this result. Missing means one.
60
+ * Output, token counters, and trace come from the final attempt; costUSD
61
+ * includes every attempt because all of them may have incurred spend. */
62
+ attemptCount?: number;
38
63
  fullNumTurns?: number;
39
64
  numSubAgents?: number;
40
65
  error?: string;
@@ -138,6 +163,8 @@ export interface ExecutorRuntimeBinary {
138
163
  source: 'path' | 'bundled' | 'none' | 'unknown';
139
164
  version?: string;
140
165
  path?: string;
166
+ /** Content identity for local script/custom executor inputs. */
167
+ contentHash?: string;
141
168
  package?: ExecutorRuntimePackage;
142
169
  error?: string;
143
170
  }
@@ -1,4 +1,5 @@
1
1
  export * from './shared.js';
2
+ export * from './trace.js';
2
3
  export * from './executor.js';
3
4
  export * from './judge.js';
4
5
  export * from './eval.js';
@@ -1,4 +1,5 @@
1
1
  export * from './shared.js';
2
+ export * from './trace.js';
2
3
  export * from './executor.js';
3
4
  export * from './judge.js';
4
5
  export * from './eval.js';
@@ -118,6 +118,8 @@ export interface DiagnosticResult {
118
118
  };
119
119
  /** Diagnostic 自身花费(USD)。 */
120
120
  costUSD?: number;
121
+ /** False 表示底层 executor 没有报告 diagnostic 的真实 USD 成本。 */
122
+ costReportedByExecutor?: boolean;
121
123
  /** Diagnostic 是否成功(LLM JSON parse 失败时为 false,fallback 显示原始文本)。 */
122
124
  ok: boolean;
123
125
  /** Diagnostic 失败时的错误信息(ok=false 时使用)。 */