oh-my-knowledge 0.48.0 → 0.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/README.md +50 -18
  2. package/README.zh.md +55 -23
  3. package/dist/analysis/coverage-analyzer.d.ts +1 -0
  4. package/dist/analysis/coverage-analyzer.js +125 -62
  5. package/dist/analysis/failure-clusterer.js +2 -1
  6. package/dist/analysis/gap-analyzer.d.ts +2 -2
  7. package/dist/analysis/gap-analyzer.js +13 -3
  8. package/dist/analysis/hedging-classifier.d.ts +2 -2
  9. package/dist/analysis/hedging-classifier.js +3 -4
  10. package/dist/analysis/report-diagnostics.js +9 -7
  11. package/dist/analysis/sample-diagnostics.js +6 -6
  12. package/dist/artifact-graph/doctor.js +15 -7
  13. package/dist/assets/agent-skills/omk/SKILL.md +27 -7
  14. package/dist/assets/agent-skills/omk/references/commands.md +18 -17
  15. package/dist/authoring/evolver.d.ts +10 -6
  16. package/dist/authoring/evolver.js +496 -83
  17. package/dist/authoring/generator.d.ts +3 -3
  18. package/dist/authoring/generator.js +5 -10
  19. package/dist/authoring/sample-fixer.d.ts +8 -6
  20. package/dist/authoring/sample-fixer.js +76 -5
  21. package/dist/cli/commands/doctor.js +31 -14
  22. package/dist/cli/commands/eval/index.d.ts +3 -0
  23. package/dist/cli/commands/eval/index.js +163 -17
  24. package/dist/cli/commands/evolve.d.ts +4 -4
  25. package/dist/cli/commands/evolve.js +27 -13
  26. package/dist/cli/commands/init.js +16 -3
  27. package/dist/cli/commands/observe/inbox.js +28 -21
  28. package/dist/cli/commands/observe/index.js +20 -11
  29. package/dist/cli/commands/observe/ingest.d.ts +3 -0
  30. package/dist/cli/commands/observe/ingest.js +30 -2
  31. package/dist/cli/commands/sample.d.ts +6 -3
  32. package/dist/cli/commands/sample.js +72 -68
  33. package/dist/cli/lib/codex-model-hint.d.ts +9 -0
  34. package/dist/cli/lib/codex-model-hint.js +45 -0
  35. package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
  36. package/dist/cli/lib/generation-failure-hint.js +61 -0
  37. package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
  38. package/dist/cli/lib/i18n-dict/common.js +4 -0
  39. package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
  40. package/dist/cli/lib/i18n-dict/gen.js +38 -6
  41. package/dist/cli/lib/i18n-dict/help.js +6 -6
  42. package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
  43. package/dist/cli/lib/i18n-dict/init.js +13 -9
  44. package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
  45. package/dist/cli/lib/i18n-dict/run.js +34 -2
  46. package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
  47. package/dist/cli/lib/llm-failure-classifier.js +8 -0
  48. package/dist/cli/lib/parse-run-config.d.ts +6 -5
  49. package/dist/cli/lib/parse-run-config.js +16 -9
  50. package/dist/cli/lib/runtime-defaults.d.ts +21 -0
  51. package/dist/cli/lib/runtime-defaults.js +79 -0
  52. package/dist/diagnosis/observe-mapper.js +14 -15
  53. package/dist/diagnosis/observe-producer.js +3 -1
  54. package/dist/diagnosis/studio-projection.js +14 -7
  55. package/dist/diagnosis/types.d.ts +2 -0
  56. package/dist/diagnosis/types.js +12 -0
  57. package/dist/doctor/endpoint-rule.js +2 -1
  58. package/dist/eval-core/artifact-file-names.js +18 -1
  59. package/dist/eval-core/artifact-index.d.ts +7 -11
  60. package/dist/eval-core/artifact-index.js +139 -80
  61. package/dist/eval-core/cache.d.ts +12 -3
  62. package/dist/eval-core/cache.js +89 -29
  63. package/dist/eval-core/comparability.js +10 -6
  64. package/dist/eval-core/evaluation-execution.d.ts +2 -1
  65. package/dist/eval-core/evaluation-execution.js +122 -37
  66. package/dist/eval-core/evaluation-job.d.ts +4 -1
  67. package/dist/eval-core/evaluation-job.js +4 -1
  68. package/dist/eval-core/evaluation-reporting.d.ts +15 -13
  69. package/dist/eval-core/evaluation-reporting.js +54 -52
  70. package/dist/eval-core/execution-strategy.d.ts +2 -0
  71. package/dist/eval-core/execution-strategy.js +11 -9
  72. package/dist/eval-core/fact-checker.js +15 -7
  73. package/dist/eval-core/holdout.js +3 -2
  74. package/dist/eval-core/judge-independence.d.ts +2 -2
  75. package/dist/eval-core/mock-hook.cjs +23 -6
  76. package/dist/eval-core/mocks-runtime.js +30 -8
  77. package/dist/eval-core/report-document.d.ts +12 -0
  78. package/dist/eval-core/report-document.js +1151 -0
  79. package/dist/eval-core/report-extensions.d.ts +4 -0
  80. package/dist/eval-core/report-extensions.js +500 -0
  81. package/dist/eval-core/report-file-migration.js +7 -2
  82. package/dist/eval-core/resume-compatibility.d.ts +31 -0
  83. package/dist/eval-core/resume-compatibility.js +141 -0
  84. package/dist/eval-core/sample-fingerprint.d.ts +12 -0
  85. package/dist/eval-core/sample-fingerprint.js +193 -0
  86. package/dist/eval-core/schema.js +86 -31
  87. package/dist/eval-core/verdict.d.ts +8 -4
  88. package/dist/eval-core/verdict.js +24 -10
  89. package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
  90. package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
  91. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
  92. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
  93. package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
  94. package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
  95. package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
  96. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
  97. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
  98. package/dist/eval-workflows/evaluation-pipeline.js +12 -7
  99. package/dist/eval-workflows/run-evaluation.d.ts +9 -7
  100. package/dist/eval-workflows/run-evaluation.js +79 -51
  101. package/dist/executors/anthropic-api.js +65 -9
  102. package/dist/executors/claude-cli.js +16 -79
  103. package/dist/executors/claude-protocol.d.ts +28 -0
  104. package/dist/executors/claude-protocol.js +180 -0
  105. package/dist/executors/claude-sdk-trace.js +56 -28
  106. package/dist/executors/claude-sdk.d.ts +1 -0
  107. package/dist/executors/claude-sdk.js +39 -93
  108. package/dist/executors/codex-cli-trace.js +166 -31
  109. package/dist/executors/codex-cli.d.ts +6 -8
  110. package/dist/executors/codex-cli.js +49 -151
  111. package/dist/executors/codex-protocol.d.ts +24 -0
  112. package/dist/executors/codex-protocol.js +234 -0
  113. package/dist/executors/codex-sdk.js +68 -120
  114. package/dist/executors/gemini.js +88 -13
  115. package/dist/executors/index.d.ts +2 -3
  116. package/dist/executors/index.js +5 -3
  117. package/dist/executors/openai-api.js +70 -9
  118. package/dist/executors/runtime-fingerprint.js +88 -11
  119. package/dist/executors/script-command.d.ts +8 -0
  120. package/dist/executors/script-command.js +87 -0
  121. package/dist/executors/script.js +202 -29
  122. package/dist/executors/shared.d.ts +35 -3
  123. package/dist/executors/shared.js +113 -15
  124. package/dist/grading/assertions.d.ts +1 -1
  125. package/dist/grading/assertions.js +19 -9
  126. package/dist/grading/diagnostic.d.ts +9 -2
  127. package/dist/grading/diagnostic.js +25 -2
  128. package/dist/grading/index.js +10 -4
  129. package/dist/grading/judge.js +19 -6
  130. package/dist/grading/layered-scores.d.ts +2 -3
  131. package/dist/grading/layered-scores.js +2 -3
  132. package/dist/inputs/load-samples.d.ts +1 -2
  133. package/dist/inputs/load-samples.js +23 -1
  134. package/dist/inputs/mcp-resolver.js +6 -3
  135. package/dist/inputs/sample-document.d.ts +11 -0
  136. package/dist/inputs/sample-document.js +96 -0
  137. package/dist/managed/evidence.d.ts +1 -0
  138. package/dist/managed/evidence.js +1 -1
  139. package/dist/managed/store.js +200 -91
  140. package/dist/observability/codex-trace-adapter.d.ts +5 -0
  141. package/dist/observability/codex-trace-adapter.js +850 -0
  142. package/dist/observability/experience.d.ts +32 -6
  143. package/dist/observability/experience.js +2695 -459
  144. package/dist/observability/feedback-matchers.js +16 -1
  145. package/dist/observability/inbox-view-model.d.ts +1 -1
  146. package/dist/observability/inbox-view-model.js +19 -14
  147. package/dist/observability/inbox.d.ts +7 -1
  148. package/dist/observability/inbox.js +632 -124
  149. package/dist/observability/problem-patterns.js +2 -0
  150. package/dist/observability/review-state.d.ts +6 -0
  151. package/dist/observability/review-state.js +235 -63
  152. package/dist/observability/skill-chain-advisories.js +1 -1
  153. package/dist/observability/skill-chain.js +17 -4
  154. package/dist/observability/skill-health-analyzer.d.ts +32 -7
  155. package/dist/observability/skill-health-analyzer.js +194 -121
  156. package/dist/observability/skill-health-report.d.ts +10 -0
  157. package/dist/observability/skill-health-report.js +620 -0
  158. package/dist/observability/soft-standards/constants.d.ts +0 -1
  159. package/dist/observability/soft-standards/constants.js +0 -1
  160. package/dist/observability/soft-standards/index.d.ts +1 -1
  161. package/dist/observability/soft-standards/index.js +1 -1
  162. package/dist/observability/soft-standards/llm-extractor.js +8 -10
  163. package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
  164. package/dist/observability/soft-standards/skill-standards-store.js +59 -18
  165. package/dist/observability/soft-standards/types.d.ts +2 -2
  166. package/dist/observability/trace-adapter.d.ts +12 -7
  167. package/dist/observability/trace-adapter.js +11 -9
  168. package/dist/observability/trace-attribution.d.ts +13 -5
  169. package/dist/observability/trace-attribution.js +315 -21
  170. package/dist/observability/trace-ingestion.d.ts +9 -0
  171. package/dist/observability/trace-ingestion.js +80 -0
  172. package/dist/observability/trace-ir.d.ts +113 -0
  173. package/dist/observability/trace-ir.js +87 -0
  174. package/dist/observability/trace-segmenter.d.ts +19 -6
  175. package/dist/observability/trace-segmenter.js +377 -196
  176. package/dist/observability/trace-session-index.d.ts +19 -0
  177. package/dist/observability/trace-session-index.js +68 -0
  178. package/dist/observability/trace-source.d.ts +12 -4
  179. package/dist/observability/trace-source.js +939 -215
  180. package/dist/renderer/html-renderer.js +37 -6
  181. package/dist/renderer/icons.js +3 -0
  182. package/dist/renderer/observation-inbox-renderer.js +208 -90
  183. package/dist/renderer/skill-detail-renderer.js +452 -109
  184. package/dist/renderer/skill-health-renderer.js +69 -12
  185. package/dist/renderer/summary.js +28 -7
  186. package/dist/renderer/table.js +21 -4
  187. package/dist/renderer/test-view.d.ts +1 -0
  188. package/dist/renderer/test-view.js +44 -9
  189. package/dist/server/indexed-report-store.js +14 -18
  190. package/dist/server/job-store.js +64 -26
  191. package/dist/server/report-server.js +190 -78
  192. package/dist/server/report-store.js +57 -80
  193. package/dist/server/skill-index.js +143 -49
  194. package/dist/server/skill-insights.js +44 -5
  195. package/dist/shared/artifact-graph.d.ts +3 -0
  196. package/dist/shared/artifact-graph.js +224 -0
  197. package/dist/shared/assertion-types.d.ts +8 -0
  198. package/dist/shared/assertion-types.js +46 -0
  199. package/dist/shared/atomic-json.d.ts +8 -0
  200. package/dist/shared/atomic-json.js +33 -0
  201. package/dist/shared/diagnosis-schema.d.ts +9 -0
  202. package/dist/shared/diagnosis-schema.js +181 -0
  203. package/dist/shared/doctor-report.d.ts +3 -0
  204. package/dist/shared/doctor-report.js +103 -0
  205. package/dist/shared/evaluation-job.d.ts +6 -0
  206. package/dist/shared/evaluation-job.js +217 -0
  207. package/dist/shared/executor-result.d.ts +17 -0
  208. package/dist/shared/executor-result.js +221 -0
  209. package/dist/shared/file-lock.d.ts +12 -0
  210. package/dist/shared/file-lock.js +129 -0
  211. package/dist/shared/json-value.d.ts +5 -0
  212. package/dist/shared/json-value.js +36 -0
  213. package/dist/shared/keyed-mutex.d.ts +7 -0
  214. package/dist/shared/keyed-mutex.js +24 -0
  215. package/dist/shared/record-count.d.ts +8 -0
  216. package/dist/shared/record-count.js +43 -0
  217. package/dist/shared/sample-contract.d.ts +3 -0
  218. package/dist/shared/sample-contract.js +332 -0
  219. package/dist/shared/timestamp.d.ts +6 -0
  220. package/dist/shared/timestamp.js +64 -0
  221. package/dist/shared/token-usage.d.ts +19 -0
  222. package/dist/shared/token-usage.js +50 -0
  223. package/dist/shared/tool-call-status.d.ts +8 -0
  224. package/dist/shared/tool-call-status.js +28 -0
  225. package/dist/shared/tool-identity.d.ts +21 -0
  226. package/dist/shared/tool-identity.js +84 -0
  227. package/dist/shared/tool-search.js +73 -16
  228. package/dist/shared/trace-projection.d.ts +5 -0
  229. package/dist/shared/trace-projection.js +20 -0
  230. package/dist/shared/trace-source-kind.d.ts +3 -0
  231. package/dist/shared/trace-source-kind.js +12 -0
  232. package/dist/types/diagnosis.d.ts +2 -0
  233. package/dist/types/eval.d.ts +4 -0
  234. package/dist/types/executor.d.ts +32 -5
  235. package/dist/types/index.d.ts +1 -0
  236. package/dist/types/index.js +1 -0
  237. package/dist/types/judge.d.ts +2 -0
  238. package/dist/types/observability.d.ts +116 -9
  239. package/dist/types/report.d.ts +58 -6
  240. package/dist/types/skill-index.d.ts +7 -0
  241. package/dist/types/trace.d.ts +2 -0
  242. package/dist/types/trace.js +1 -0
  243. package/package.json +9 -5
@@ -0,0 +1,234 @@
1
+ import { checkedSumTokenCounts, optionalTokenCount, splitInclusiveInputTokens, sumTokenCounts, } from '../shared/token-usage.js';
2
+ import { extractCodexTrace, isCodexResultEvent } from './codex-cli-trace.js';
3
+ const CODEX_EVENT_TYPES = new Set([
4
+ 'thread.started',
5
+ 'turn.started',
6
+ 'turn.completed',
7
+ 'turn.failed',
8
+ 'item.started',
9
+ 'item.updated',
10
+ 'item.completed',
11
+ 'error',
12
+ ]);
13
+ const CODEX_ITEM_TYPES = new Set([
14
+ 'agent_message',
15
+ 'reasoning',
16
+ 'command_execution',
17
+ 'file_change',
18
+ 'mcp_tool_call',
19
+ 'web_search',
20
+ 'todo_list',
21
+ 'error',
22
+ // Older CLI releases emitted these directly.
23
+ 'file_read',
24
+ 'file_write',
25
+ ]);
26
+ export function extractCodexUsage(events) {
27
+ const input = [];
28
+ const cached = [];
29
+ const output = [];
30
+ for (const event of events) {
31
+ if (!isCodexResultEvent(event) || !event.usage)
32
+ continue;
33
+ const normalized = splitInclusiveInputTokens(event.usage.input_tokens, event.usage.cached_input_tokens);
34
+ input.push(normalized.inputTokens);
35
+ cached.push(normalized.cacheReadTokens);
36
+ output.push(event.usage.output_tokens ?? 0);
37
+ }
38
+ return {
39
+ input: sumTokenCounts(...input),
40
+ cached: sumTokenCounts(...cached),
41
+ output: sumTokenCounts(...output),
42
+ };
43
+ }
44
+ /**
45
+ * Match @openai/codex-sdk's `finalResponse`: the latest completed
46
+ * agent_message is the answer. Earlier messages remain available in `turns`.
47
+ */
48
+ export function extractCodexFinalOutput(events) {
49
+ for (let index = events.length - 1; index >= 0; index -= 1) {
50
+ const event = events[index];
51
+ if (event.type === 'item.completed' && event.item?.type === 'agent_message') {
52
+ return typeof event.item.text === 'string' ? event.item.text : '';
53
+ }
54
+ }
55
+ return '';
56
+ }
57
+ export function extractCodexProtocolError(events) {
58
+ for (let index = events.length - 1; index >= 0; index -= 1) {
59
+ const event = events[index];
60
+ if (event.type === 'error') {
61
+ return event.message || event.error?.message || 'codex stream error';
62
+ }
63
+ }
64
+ return undefined;
65
+ }
66
+ export function validateCodexProtocol(events) {
67
+ const pendingItems = new Map();
68
+ let openTurnCount = 0;
69
+ const inputCounts = [];
70
+ const cacheCounts = [];
71
+ const outputCounts = [];
72
+ const elapsedCounts = [];
73
+ for (let eventIndex = 0; eventIndex < events.length; eventIndex += 1) {
74
+ const event = events[eventIndex];
75
+ const eventType = event.type;
76
+ if (!eventType || !CODEX_EVENT_TYPES.has(eventType)) {
77
+ return `unsupported codex event type: ${eventType || '<missing>'}`;
78
+ }
79
+ if (event.elapsed_ms !== undefined) {
80
+ const elapsed = optionalTokenCount(event.elapsed_ms);
81
+ if (elapsed === undefined)
82
+ return `invalid codex elapsed_ms at event ${eventIndex}`;
83
+ elapsedCounts.push(elapsed);
84
+ }
85
+ if (eventType === 'turn.started') {
86
+ openTurnCount += 1;
87
+ continue;
88
+ }
89
+ if (eventType.startsWith('item.')) {
90
+ const itemType = event.item?.type;
91
+ if (!itemType || !CODEX_ITEM_TYPES.has(itemType)) {
92
+ return `unsupported codex item type: ${itemType || '<missing>'}`;
93
+ }
94
+ const itemId = event.item?.id;
95
+ if (typeof itemId !== 'string' || itemId.trim() === '') {
96
+ return `missing codex item id at event ${eventIndex}`;
97
+ }
98
+ if (eventType === 'item.started') {
99
+ pendingItems.set(itemId, (pendingItems.get(itemId) ?? 0) + 1);
100
+ }
101
+ else if (eventType === 'item.updated' && !pendingItems.has(itemId)) {
102
+ return `codex item.updated without item.started: ${itemId}`;
103
+ }
104
+ else if (eventType === 'item.completed') {
105
+ const pendingCount = pendingItems.get(itemId) ?? 0;
106
+ if (pendingCount === 1)
107
+ pendingItems.delete(itemId);
108
+ else if (pendingCount > 1)
109
+ pendingItems.set(itemId, pendingCount - 1);
110
+ }
111
+ continue;
112
+ }
113
+ if (eventType !== 'turn.completed' && eventType !== 'turn.failed')
114
+ continue;
115
+ if (pendingItems.size > 0) {
116
+ return `codex turn ended with incomplete item(s): ${Array.from(pendingItems.keys()).join(', ')}`;
117
+ }
118
+ if (openTurnCount > 0)
119
+ openTurnCount -= 1;
120
+ if (event.usage !== undefined || eventType === 'turn.completed') {
121
+ const usageError = validateCodexUsage(event.usage, eventIndex);
122
+ if (usageError)
123
+ return usageError;
124
+ const normalized = splitInclusiveInputTokens(event.usage?.input_tokens, event.usage?.cached_input_tokens);
125
+ inputCounts.push(normalized.inputTokens);
126
+ cacheCounts.push(normalized.cacheReadTokens);
127
+ outputCounts.push(event.usage?.output_tokens ?? 0);
128
+ }
129
+ }
130
+ if (openTurnCount > 0)
131
+ return 'codex stream ended before turn completion';
132
+ if (pendingItems.size > 0) {
133
+ return `codex stream ended with incomplete item(s): ${Array.from(pendingItems.keys()).join(', ')}`;
134
+ }
135
+ if (!hasSafeAggregate(inputCounts)
136
+ || !hasSafeAggregate(cacheCounts)
137
+ || !hasSafeAggregate(outputCounts)
138
+ || !hasSafeAggregate([...inputCounts, ...cacheCounts, ...outputCounts])) {
139
+ return 'codex token usage aggregate exceeds safe integer';
140
+ }
141
+ if (!hasSafeAggregate(elapsedCounts)) {
142
+ return 'codex elapsed_ms aggregate exceeds safe integer';
143
+ }
144
+ return undefined;
145
+ }
146
+ function validateCodexUsage(usage, eventIndex) {
147
+ if (!usage)
148
+ return `missing codex usage at event ${eventIndex}`;
149
+ const input = optionalTokenCount(usage.input_tokens);
150
+ const output = optionalTokenCount(usage.output_tokens);
151
+ const cached = usage.cached_input_tokens === undefined
152
+ ? 0
153
+ : optionalTokenCount(usage.cached_input_tokens);
154
+ const reasoning = usage.reasoning_output_tokens === undefined
155
+ ? 0
156
+ : optionalTokenCount(usage.reasoning_output_tokens);
157
+ if (input === undefined || output === undefined || cached === undefined || reasoning === undefined) {
158
+ return `invalid codex usage at event ${eventIndex}`;
159
+ }
160
+ if (cached > input)
161
+ return `codex cached_input_tokens exceeds input_tokens at event ${eventIndex}`;
162
+ return undefined;
163
+ }
164
+ function hasSafeAggregate(values) {
165
+ return checkedSumTokenCounts(...values) !== undefined;
166
+ }
167
+ export function extractCodexStopReason(events) {
168
+ if (extractCodexProtocolError(events) || validateCodexProtocol(events))
169
+ return 'error';
170
+ const last = [...events].reverse().find((event) => isCodexResultEvent(event));
171
+ if (!last)
172
+ return 'unknown';
173
+ if (last.type === 'turn.failed')
174
+ return 'error';
175
+ return last.stop_reason || 'end_turn';
176
+ }
177
+ export function sumCodexElapsed(resultEvents, wallClock) {
178
+ const total = sumTokenCounts(...resultEvents.map((event) => event.elapsed_ms ?? 0));
179
+ return total > 0 ? total : wallClock;
180
+ }
181
+ export function buildCodexResult({ events, wallClockDurationMs, source, malformedLineCount = 0, forcedError, }) {
182
+ const resultEvents = events.filter(isCodexResultEvent);
183
+ const last = resultEvents.at(-1);
184
+ const usage = extractCodexUsage(events);
185
+ const trace = extractCodexTrace(events);
186
+ const finalOutput = extractCodexFinalOutput(events);
187
+ const errors = [];
188
+ if (forcedError)
189
+ errors.push(forcedError);
190
+ if (malformedLineCount > 0) {
191
+ errors.push(`${source} output contained ${malformedLineCount} malformed line(s)`);
192
+ }
193
+ const schemaError = validateCodexProtocol(events);
194
+ if (schemaError)
195
+ errors.push(schemaError);
196
+ const streamError = extractCodexProtocolError(events);
197
+ if (streamError)
198
+ errors.push(streamError);
199
+ if (!last) {
200
+ errors.push(`no turn.completed/turn.failed event in ${source} output`);
201
+ }
202
+ else if (last.type === 'turn.failed' || last.error) {
203
+ errors.push(last.error?.message || `${source} turn.failed`);
204
+ }
205
+ else if (!finalOutput.trim()) {
206
+ errors.push(`${source} completed without an assistant response`);
207
+ }
208
+ const uniqueErrors = [...new Set(errors)];
209
+ const ok = uniqueErrors.length === 0;
210
+ const tokenUsageReported = !forcedError
211
+ && malformedLineCount === 0
212
+ && !schemaError
213
+ && Boolean(last);
214
+ return {
215
+ ok,
216
+ durationMs: sumCodexElapsed(resultEvents, wallClockDurationMs),
217
+ durationApiMs: 0,
218
+ inputTokens: usage.input,
219
+ outputTokens: usage.output,
220
+ cacheReadTokens: usage.cached,
221
+ cacheCreationTokens: 0,
222
+ ...(!tokenUsageReported && { tokenUsageReportedByExecutor: false }),
223
+ costUSD: 0,
224
+ costReportedByExecutor: false,
225
+ output: finalOutput.trim() ? finalOutput : null,
226
+ stopReason: ok ? extractCodexStopReason(events) : 'error',
227
+ ...(uniqueErrors.length > 0 && { error: uniqueErrors.join('; ') }),
228
+ numTurns: resultEvents.length,
229
+ fullNumTurns: trace.fullNumTurns,
230
+ numSubAgents: trace.numSubAgents,
231
+ ...(trace.turns.length > 0 && { turns: trace.turns }),
232
+ ...(trace.toolCalls.length > 0 && { toolCalls: trace.toolCalls }),
233
+ };
234
+ }
@@ -1,12 +1,11 @@
1
1
  import { existsSync } from 'node:fs';
2
- import { mkdtemp, symlink } from 'node:fs/promises';
2
+ import { copyFile, mkdtemp, rm } from 'node:fs/promises';
3
3
  import { homedir, tmpdir } from 'node:os';
4
4
  import { join } from 'node:path';
5
- import { extractCodexTrace, isCodexResultEvent } from './codex-cli-trace.js';
6
- import { asErrorLike, buildExecEnv, DEFAULT_TIMEOUT_MS, errorMessage, timeoutExecResult, } from './shared.js';
7
- import { extractCodexFinalOutput, extractCodexStopReason, extractCodexUsage, isolateCodexCwd, sumCodexElapsed, } from './codex-cli.js';
5
+ import { buildCodexResult } from './codex-protocol.js';
6
+ import { asErrorLike, buildExecEnv, DEFAULT_TIMEOUT_MS, errorMessage, interruptedExecResult, normalizeCodexProtocolEvent, registerSigintSubscriber, timeoutExecResult, } from './shared.js';
7
+ import { isolateCodexCwd } from './codex-cli.js';
8
8
  let CodexCtor = null;
9
- let codexHomePromise = null;
10
9
  let hasWarnedSystem = false;
11
10
  let hasWarnedCost = false;
12
11
  async function getCodexCtor() {
@@ -24,37 +23,31 @@ async function getCodexCtor() {
24
23
  // instructions, tool config)into the eval and pollute `~/.codex/sessions/`
25
24
  // across runs.
26
25
  //
27
- // Mitigation: redirect $CODEX_HOME to a per-process tmp dir for codex-sdk only.
28
- // auth.json is symlinked from the real CODEX_HOME so the SDK can still
29
- // authenticate; config.toml + sessions/ stay isolated. Tmp dir stays alive for
30
- // the OS to rotate(no explicit cleanup, matches DEFAULT_TIMEOUT_MS scope).
26
+ // Mitigation: redirect $CODEX_HOME to a fresh tmp dir for every codex-sdk run.
27
+ // auth.json is copied from the real CODEX_HOME so token refreshes cannot mutate
28
+ // the user's credential file through a symlink; config.toml + sessions/ stay
29
+ // isolated and the tmp dir is removed after the SDK child exits.
31
30
  //
32
- // Best-effort on Windows where symlink may need elevated privileges; on
33
- // failure, the codex-sdk executor still runs but lands in the user's real
34
- // CODEX_HOME with a stderr warning.
31
+ // The SDK still does not expose codex CLI's `--ignore-rules`. Project
32
+ // execpolicy is therefore part of codex-sdk's runtime context; use codex-cli
33
+ // when that final isolation boundary matters.
35
34
  export async function getIsolatedCodexHome() {
36
- if (!codexHomePromise) {
37
- codexHomePromise = (async () => {
38
- const tmpHome = await mkdtemp(join(tmpdir(), 'omk-codex-sdk-'));
39
- const realHome = process.env.CODEX_HOME || join(homedir(), '.codex');
40
- const realAuth = join(realHome, 'auth.json');
41
- if (existsSync(realAuth)) {
42
- try {
43
- await symlink(realAuth, join(tmpHome, 'auth.json'));
44
- }
45
- catch (err) {
46
- process.stderr.write(`[codex-sdk] CODEX_HOME isolation: auth.json symlink failed (${err.message}); SDK will use bare tmp home and may fail to authenticate\n`);
47
- }
48
- }
49
- return tmpHome;
50
- })();
35
+ const tmpHome = await mkdtemp(join(tmpdir(), 'omk-codex-sdk-'));
36
+ const realHome = process.env.CODEX_HOME || join(homedir(), '.codex');
37
+ const realAuth = join(realHome, 'auth.json');
38
+ if (existsSync(realAuth)) {
39
+ try {
40
+ await copyFile(realAuth, join(tmpHome, 'auth.json'));
41
+ }
42
+ catch (err) {
43
+ process.stderr.write(`[codex-sdk] CODEX_HOME isolation: auth.json copy failed (${err.message}); SDK will use bare tmp home and may fail to authenticate\n`);
44
+ }
51
45
  }
52
- return codexHomePromise;
46
+ return tmpHome;
53
47
  }
54
48
  // exported for test isolation
55
49
  export function __resetCodexSdkStateForTest() {
56
50
  CodexCtor = null;
57
- codexHomePromise = null;
58
51
  hasWarnedSystem = false;
59
52
  hasWarnedCost = false;
60
53
  }
@@ -79,7 +72,7 @@ export async function createCodexSdkClient(env) {
79
72
  return new CodexClient(buildCodexSdkClientOptions(env));
80
73
  }
81
74
  function normalizeSdkEvent(event) {
82
- return event;
75
+ return normalizeCodexProtocolEvent(event) ?? {};
83
76
  }
84
77
  export async function codexSdkExecutor({ model, system, prompt, cwd, skillDir, timeoutMs = DEFAULT_TIMEOUT_MS, allowedSkills, verbose }) {
85
78
  isolateCodexCwd(allowedSkills, cwd, 'codex-sdk');
@@ -95,7 +88,16 @@ export async function codexSdkExecutor({ model, system, prompt, cwd, skillDir, t
95
88
  const env = buildExecEnv(skillDir);
96
89
  const start = Date.now();
97
90
  const abortController = new AbortController();
98
- const timer = setTimeout(() => abortController.abort(), timeoutMs);
91
+ let abortReason;
92
+ const abort = (reason) => {
93
+ if (abortReason)
94
+ return;
95
+ abortReason = reason;
96
+ abortController.abort();
97
+ };
98
+ const timer = setTimeout(() => {
99
+ abort('timeout');
100
+ }, timeoutMs);
99
101
  // SIGINT propagation: PR #33 helper only tracks child process refs registered
100
102
  // via spawnWithSigintPropagation; @openai/codex-sdk does its own spawn()
101
103
  // internally and the SDK child is not in that registry. Hook the
@@ -103,111 +105,57 @@ export async function codexSdkExecutor({ model, system, prompt, cwd, skillDir, t
103
105
  // abortSignal pipeline (SDK passes `signal: args.signal` to spawn). Without
104
106
  // this, Ctrl+C in a nested host orphans the codex SDK child until DEFAULT
105
107
  // timeout elapses, regressing PR #33's nested-host fix.
106
- const sigintListener = () => abortController.abort();
107
- process.on('SIGINT', sigintListener);
108
+ const unregisterSigint = registerSigintSubscriber(() => abort('interrupted'));
108
109
  const events = [];
110
+ let isolatedCodexHome;
109
111
  try {
110
- env.CODEX_HOME = await getIsolatedCodexHome();
112
+ isolatedCodexHome = await getIsolatedCodexHome();
113
+ env.CODEX_HOME = isolatedCodexHome;
111
114
  const codex = await createCodexSdkClient(env);
112
115
  const thread = codex.startThread(buildCodexSdkThreadOptions({ model, cwd }));
113
116
  const streamed = await thread.runStreamed(finalPrompt, { signal: abortController.signal });
114
117
  for await (const event of streamed.events) {
115
118
  events.push(normalizeSdkEvent(event));
116
119
  }
117
- const durationMs = Date.now() - start;
118
- const resultEvents = events.filter(isCodexResultEvent);
119
- if (resultEvents.length === 0) {
120
- return {
121
- ok: false,
122
- error: 'no turn.completed/turn.failed event in codex-sdk output',
123
- durationMs,
124
- durationApiMs: 0,
125
- inputTokens: 0,
126
- outputTokens: 0,
127
- cacheReadTokens: 0,
128
- cacheCreationTokens: 0,
129
- costUSD: 0,
130
- costReportedByExecutor: false,
131
- output: null,
132
- stopReason: 'error',
133
- numTurns: 0,
134
- };
135
- }
136
- const last = resultEvents[resultEvents.length - 1];
137
- const usage = extractCodexUsage(events);
138
- const trace = extractCodexTrace(events);
139
- const stopReason = extractCodexStopReason(events);
140
- const finalOutput = extractCodexFinalOutput(events);
141
- const ok = last.type !== 'turn.failed' && !last.error;
142
- return {
143
- ok,
144
- durationMs: sumCodexElapsed(resultEvents, durationMs),
145
- durationApiMs: 0,
146
- inputTokens: usage.input,
147
- outputTokens: usage.output,
148
- cacheReadTokens: usage.cached,
149
- cacheCreationTokens: 0,
150
- costUSD: 0,
151
- costReportedByExecutor: false,
152
- output: finalOutput,
153
- stopReason,
154
- ...(!ok && { error: last.error?.message || 'codex-sdk turn.failed' }),
155
- numTurns: resultEvents.length,
156
- fullNumTurns: trace.fullNumTurns,
157
- numSubAgents: trace.numSubAgents,
158
- ...(trace.turns.length > 0 && { turns: trace.turns }),
159
- ...(trace.toolCalls.length > 0 && { toolCalls: trace.toolCalls }),
160
- };
120
+ return buildCodexResult({
121
+ events,
122
+ wallClockDurationMs: Date.now() - start,
123
+ source: 'codex-sdk',
124
+ });
161
125
  }
162
126
  catch (err) {
163
127
  const durationMs = Date.now() - start;
164
128
  const details = asErrorLike(err);
165
- const isAbort = details.name === 'AbortError' || abortController.signal.aborted;
166
- if (isAbort)
129
+ if (abortReason === 'interrupted') {
130
+ return { ...interruptedExecResult(durationMs), costReportedByExecutor: false };
131
+ }
132
+ if (abortReason === 'timeout') {
167
133
  return { ...timeoutExecResult(timeoutMs, durationMs), costReportedByExecutor: false };
168
- const resultEvents = events.filter(isCodexResultEvent);
169
- if (resultEvents.length > 0) {
170
- const last = resultEvents[resultEvents.length - 1];
171
- const usage = extractCodexUsage(events);
172
- const trace = extractCodexTrace(events);
173
- return {
174
- ok: false,
175
- error: last.error?.message || errorMessage(err),
176
- durationMs: sumCodexElapsed(resultEvents, durationMs),
177
- durationApiMs: 0,
178
- inputTokens: usage.input,
179
- outputTokens: usage.output,
180
- cacheReadTokens: usage.cached,
181
- cacheCreationTokens: 0,
182
- costUSD: 0,
183
- costReportedByExecutor: false,
184
- output: extractCodexFinalOutput(events) || null,
185
- stopReason: 'error',
186
- numTurns: resultEvents.length,
187
- fullNumTurns: trace.fullNumTurns,
188
- numSubAgents: trace.numSubAgents,
189
- ...(trace.turns.length > 0 && { turns: trace.turns }),
190
- ...(trace.toolCalls.length > 0 && { toolCalls: trace.toolCalls }),
191
- };
192
134
  }
193
- return {
194
- ok: false,
195
- error: errorMessage(err),
196
- durationMs,
197
- durationApiMs: 0,
198
- inputTokens: 0,
199
- outputTokens: 0,
200
- cacheReadTokens: 0,
201
- cacheCreationTokens: 0,
202
- costUSD: 0,
203
- costReportedByExecutor: false,
204
- output: null,
205
- stopReason: 'error',
206
- numTurns: 0,
207
- };
135
+ return buildCodexResult({
136
+ events,
137
+ wallClockDurationMs: durationMs,
138
+ source: 'codex-sdk',
139
+ forcedError: details.name === 'AbortError'
140
+ ? 'codex-sdk execution aborted'
141
+ : errorMessage(err),
142
+ });
208
143
  }
209
144
  finally {
210
145
  clearTimeout(timer);
211
- process.removeListener('SIGINT', sigintListener);
146
+ unregisterSigint();
147
+ if (isolatedCodexHome) {
148
+ try {
149
+ await rm(isolatedCodexHome, {
150
+ recursive: true,
151
+ force: true,
152
+ maxRetries: 3,
153
+ retryDelay: 50,
154
+ });
155
+ }
156
+ catch (err) {
157
+ process.stderr.write(`[codex-sdk] unable to remove isolated CODEX_HOME ${isolatedCodexHome}: ${errorMessage(err)}\n`);
158
+ }
159
+ }
212
160
  }
213
161
  }
@@ -1,4 +1,5 @@
1
1
  import { DEFAULT_TIMEOUT_MS, errorMessage, interruptedExecResult, parseJson, spawnWithSigintPropagation, timeoutExecResult, } from './shared.js';
2
+ import { optionalTokenCount } from '../shared/token-usage.js';
2
3
  export async function geminiExecutor({ model, system, prompt, timeoutMs = DEFAULT_TIMEOUT_MS }) {
3
4
  const fullPrompt = system ? `${system}\n\n${prompt}` : prompt;
4
5
  const start = Date.now();
@@ -10,6 +11,7 @@ export async function geminiExecutor({ model, system, prompt, timeoutMs = DEFAUL
10
11
  env: { ...process.env },
11
12
  timeoutMs,
12
13
  });
14
+ child.stdin?.on('error', () => undefined);
13
15
  child.stdin?.write(fullPrompt);
14
16
  child.stdin?.end();
15
17
  let output;
@@ -23,30 +25,81 @@ export async function geminiExecutor({ model, system, prompt, timeoutMs = DEFAUL
23
25
  return timeoutExecResult(timeoutMs, Date.now() - start);
24
26
  if (details.killedBySignal)
25
27
  return interruptedExecResult(Date.now() - start);
26
- // gemini 退出码非 0 但有 stdout 时仍按成功路径解析(原行为);只在 stdout 为空时才视为失败
27
- if (details.stdout && details.stdout.length > 0) {
28
- output = details.stdout;
29
- }
30
- else {
31
- throw err;
32
- }
28
+ const durationMs = Date.now() - start;
29
+ return {
30
+ ok: false,
31
+ error: details.stderr?.trim() || details.message || `gemini exited with code ${details.code ?? '?'}`,
32
+ durationMs,
33
+ durationApiMs: 0,
34
+ inputTokens: 0,
35
+ outputTokens: 0,
36
+ cacheReadTokens: 0,
37
+ cacheCreationTokens: 0,
38
+ tokenUsageReportedByExecutor: false,
39
+ costUSD: 0,
40
+ costReportedByExecutor: false,
41
+ output: details.stdout?.trim() || null,
42
+ stopReason: 'error',
43
+ numTurns: 0,
44
+ };
33
45
  }
34
46
  const durationMs = Date.now() - start;
35
47
  let text = output;
36
48
  let inputTokens = 0;
37
49
  let outputTokens = 0;
50
+ let tokenUsageReported = false;
38
51
  try {
39
- const data = parseJson(output);
40
- if (data.response)
41
- text = data.response;
42
- if (data.stats) {
43
- inputTokens = data.stats.inputTokens || 0;
44
- outputTokens = data.stats.outputTokens || 0;
52
+ const parsed = parseJson(output);
53
+ if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
54
+ const record = parsed;
55
+ const declaresProtocol = Object.hasOwn(record, 'response')
56
+ || Object.hasOwn(record, 'stats');
57
+ if (declaresProtocol) {
58
+ const data = parsed;
59
+ if (typeof data.response !== 'string') {
60
+ return invalidGeminiProtocolResult(durationMs, '"response" must be a string');
61
+ }
62
+ text = data.response;
63
+ if (data.stats !== undefined
64
+ && (!data.stats
65
+ || typeof data.stats !== 'object'
66
+ || Array.isArray(data.stats))) {
67
+ return invalidGeminiProtocolResult(durationMs, '"stats" must be an object when present');
68
+ }
69
+ if (data.stats !== undefined) {
70
+ const parsedInput = optionalTokenCount(data.stats.inputTokens);
71
+ const parsedOutput = optionalTokenCount(data.stats.outputTokens);
72
+ if (parsedInput === undefined || parsedOutput === undefined) {
73
+ return invalidGeminiProtocolResult(durationMs, 'invalid token usage');
74
+ }
75
+ inputTokens = parsedInput;
76
+ outputTokens = parsedOutput;
77
+ tokenUsageReported = true;
78
+ }
79
+ }
45
80
  }
46
81
  }
47
82
  catch {
48
83
  text = output.trim();
49
84
  }
85
+ if (!text.trim()) {
86
+ return {
87
+ ok: false,
88
+ error: 'gemini completed without model output',
89
+ durationMs,
90
+ durationApiMs: 0,
91
+ inputTokens,
92
+ outputTokens,
93
+ cacheReadTokens: 0,
94
+ cacheCreationTokens: 0,
95
+ ...(!tokenUsageReported && { tokenUsageReportedByExecutor: false }),
96
+ costUSD: 0,
97
+ costReportedByExecutor: false,
98
+ output: null,
99
+ stopReason: 'error',
100
+ numTurns: 1,
101
+ };
102
+ }
50
103
  return {
51
104
  ok: true,
52
105
  durationMs,
@@ -55,7 +108,9 @@ export async function geminiExecutor({ model, system, prompt, timeoutMs = DEFAUL
55
108
  outputTokens,
56
109
  cacheReadTokens: 0,
57
110
  cacheCreationTokens: 0,
111
+ ...(!tokenUsageReported && { tokenUsageReportedByExecutor: false }),
58
112
  costUSD: 0,
113
+ costReportedByExecutor: false,
59
114
  output: text,
60
115
  stopReason: 'end',
61
116
  numTurns: 1,
@@ -72,10 +127,30 @@ export async function geminiExecutor({ model, system, prompt, timeoutMs = DEFAUL
72
127
  outputTokens: 0,
73
128
  cacheReadTokens: 0,
74
129
  cacheCreationTokens: 0,
130
+ tokenUsageReportedByExecutor: false,
75
131
  costUSD: 0,
132
+ costReportedByExecutor: false,
76
133
  output: null,
77
134
  stopReason: 'error',
78
135
  numTurns: 0,
79
136
  };
80
137
  }
81
138
  }
139
+ function invalidGeminiProtocolResult(durationMs, message) {
140
+ return {
141
+ ok: false,
142
+ error: `gemini returned malformed protocol JSON: ${message}`,
143
+ durationMs,
144
+ durationApiMs: 0,
145
+ inputTokens: 0,
146
+ outputTokens: 0,
147
+ cacheReadTokens: 0,
148
+ cacheCreationTokens: 0,
149
+ tokenUsageReportedByExecutor: false,
150
+ costUSD: 0,
151
+ costReportedByExecutor: false,
152
+ output: null,
153
+ stopReason: 'error',
154
+ numTurns: 1,
155
+ };
156
+ }
@@ -1,6 +1,5 @@
1
1
  import type { ExecutorFn } from '../types/index.js';
2
2
  import { extractAgentTrace } from './claude-sdk-trace.js';
3
3
  import { createScriptExecutor } from './script.js';
4
- import { DEFAULT_MODEL, JUDGE_MODEL } from './shared.js';
5
- export { DEFAULT_MODEL, JUDGE_MODEL, extractAgentTrace, createScriptExecutor };
6
- export declare function createExecutor(name?: string): ExecutorFn;
4
+ export { extractAgentTrace, createScriptExecutor };
5
+ export declare function createExecutor(name: string): ExecutorFn;
@@ -7,7 +7,6 @@ import { codexSdkExecutor } from './codex-sdk.js';
7
7
  import { geminiExecutor } from './gemini.js';
8
8
  import { openAiApiExecutor } from './openai-api.js';
9
9
  import { createScriptExecutor } from './script.js';
10
- import { DEFAULT_MODEL, JUDGE_MODEL } from './shared.js';
11
10
  // 命名一致性:provider HTTP 路径统一用 `<vendor>-api`(`anthropic-api` / `openai-api`),
12
11
  // vendor coding agent CLI 用 vendor 名(`claude` / `codex`)。`openai` 这个不带 -api 后缀的
13
12
  // 旧 alias 历史上指 openai-cli 子进程实现,删除后不再设别名 — 用 `--executor openai-api`。
@@ -20,7 +19,10 @@ const EXECUTOR_REGISTRY = {
20
19
  'anthropic-api': anthropicApiExecutor,
21
20
  'openai-api': openAiApiExecutor,
22
21
  };
23
- export { DEFAULT_MODEL, JUDGE_MODEL, extractAgentTrace, createScriptExecutor };
24
- export function createExecutor(name = 'claude') {
22
+ export { extractAgentTrace, createScriptExecutor };
23
+ export function createExecutor(name) {
24
+ if (name.trim().length === 0) {
25
+ throw new Error('executor name or script command is required');
26
+ }
25
27
  return EXECUTOR_REGISTRY[name] || createScriptExecutor(name);
26
28
  }