oh-my-knowledge 0.0.1 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +537 -332
  3. package/dist/src/analysis/coverage-analyzer.d.ts +57 -0
  4. package/dist/src/analysis/coverage-analyzer.d.ts.map +1 -0
  5. package/dist/src/analysis/coverage-analyzer.js +262 -0
  6. package/dist/src/analysis/coverage-analyzer.js.map +1 -0
  7. package/dist/src/analysis/gap-analyzer.d.ts +114 -0
  8. package/dist/src/analysis/gap-analyzer.d.ts.map +1 -0
  9. package/dist/src/analysis/gap-analyzer.js +439 -0
  10. package/dist/src/analysis/gap-analyzer.js.map +1 -0
  11. package/dist/src/analysis/hedging-classifier.d.ts +34 -0
  12. package/dist/src/analysis/hedging-classifier.d.ts.map +1 -0
  13. package/dist/src/analysis/hedging-classifier.js +144 -0
  14. package/dist/src/analysis/hedging-classifier.js.map +1 -0
  15. package/dist/src/analysis/report-diagnostics.d.ts +9 -0
  16. package/dist/src/analysis/report-diagnostics.d.ts.map +1 -0
  17. package/dist/src/analysis/report-diagnostics.js +631 -0
  18. package/dist/src/analysis/report-diagnostics.js.map +1 -0
  19. package/dist/src/authoring/evolver.d.ts +64 -0
  20. package/dist/src/authoring/evolver.d.ts.map +1 -0
  21. package/dist/src/authoring/evolver.js +275 -0
  22. package/dist/src/authoring/evolver.js.map +1 -0
  23. package/dist/src/authoring/generator.d.ts +13 -0
  24. package/dist/src/authoring/generator.d.ts.map +1 -0
  25. package/dist/src/authoring/generator.js +60 -0
  26. package/dist/src/authoring/generator.js.map +1 -0
  27. package/dist/src/cli.d.ts +3 -0
  28. package/dist/src/cli.d.ts.map +1 -0
  29. package/dist/src/cli.js +1004 -0
  30. package/dist/src/cli.js.map +1 -0
  31. package/dist/src/eval-core/cache.d.ts +11 -0
  32. package/dist/src/eval-core/cache.d.ts.map +1 -0
  33. package/dist/src/eval-core/cache.js +53 -0
  34. package/dist/src/eval-core/cache.js.map +1 -0
  35. package/dist/src/eval-core/ci-gates.d.ts +17 -0
  36. package/dist/src/eval-core/ci-gates.d.ts.map +1 -0
  37. package/dist/src/eval-core/ci-gates.js +42 -0
  38. package/dist/src/eval-core/ci-gates.js.map +1 -0
  39. package/dist/src/eval-core/dependency-checker.d.ts +37 -0
  40. package/dist/src/eval-core/dependency-checker.d.ts.map +1 -0
  41. package/dist/src/eval-core/dependency-checker.js +280 -0
  42. package/dist/src/eval-core/dependency-checker.js.map +1 -0
  43. package/dist/src/eval-core/evaluation-execution.d.ts +26 -0
  44. package/dist/src/eval-core/evaluation-execution.d.ts.map +1 -0
  45. package/dist/src/eval-core/evaluation-execution.js +196 -0
  46. package/dist/src/eval-core/evaluation-execution.js.map +1 -0
  47. package/dist/src/eval-core/evaluation-job.d.ts +48 -0
  48. package/dist/src/eval-core/evaluation-job.d.ts.map +1 -0
  49. package/dist/src/eval-core/evaluation-job.js +112 -0
  50. package/dist/src/eval-core/evaluation-job.js.map +1 -0
  51. package/dist/src/eval-core/evaluation-reporting.d.ts +28 -0
  52. package/dist/src/eval-core/evaluation-reporting.d.ts.map +1 -0
  53. package/dist/src/eval-core/evaluation-reporting.js +123 -0
  54. package/dist/src/eval-core/evaluation-reporting.js.map +1 -0
  55. package/dist/src/eval-core/execution-strategy.d.ts +11 -0
  56. package/dist/src/eval-core/execution-strategy.d.ts.map +1 -0
  57. package/dist/src/eval-core/execution-strategy.js +121 -0
  58. package/dist/src/eval-core/execution-strategy.js.map +1 -0
  59. package/dist/src/eval-core/fact-checker.d.ts +25 -0
  60. package/dist/src/eval-core/fact-checker.d.ts.map +1 -0
  61. package/dist/src/eval-core/fact-checker.js +66 -0
  62. package/dist/src/eval-core/fact-checker.js.map +1 -0
  63. package/dist/src/eval-core/schema.d.ts +30 -0
  64. package/dist/src/eval-core/schema.d.ts.map +1 -0
  65. package/dist/src/eval-core/schema.js +195 -0
  66. package/dist/src/eval-core/schema.js.map +1 -0
  67. package/dist/src/eval-core/statistics.d.ts +86 -0
  68. package/dist/src/eval-core/statistics.d.ts.map +1 -0
  69. package/dist/src/eval-core/statistics.js +210 -0
  70. package/dist/src/eval-core/statistics.js.map +1 -0
  71. package/dist/src/eval-core/task-planner.d.ts +4 -0
  72. package/dist/src/eval-core/task-planner.d.ts.map +1 -0
  73. package/dist/src/eval-core/task-planner.js +33 -0
  74. package/dist/src/eval-core/task-planner.js.map +1 -0
  75. package/dist/src/eval-workflows/each-evaluation-workflow.d.ts +133 -0
  76. package/dist/src/eval-workflows/each-evaluation-workflow.d.ts.map +1 -0
  77. package/dist/src/eval-workflows/each-evaluation-workflow.js +169 -0
  78. package/dist/src/eval-workflows/each-evaluation-workflow.js.map +1 -0
  79. package/dist/src/eval-workflows/evaluation-pipeline.d.ts +38 -0
  80. package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -0
  81. package/dist/src/eval-workflows/evaluation-pipeline.js +214 -0
  82. package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -0
  83. package/dist/src/eval-workflows/evaluation-preparation.d.ts +64 -0
  84. package/dist/src/eval-workflows/evaluation-preparation.d.ts.map +1 -0
  85. package/dist/src/eval-workflows/evaluation-preparation.js +90 -0
  86. package/dist/src/eval-workflows/evaluation-preparation.js.map +1 -0
  87. package/dist/src/eval-workflows/run-evaluation.d.ts +105 -0
  88. package/dist/src/eval-workflows/run-evaluation.d.ts.map +1 -0
  89. package/dist/src/eval-workflows/run-evaluation.js +264 -0
  90. package/dist/src/eval-workflows/run-evaluation.js.map +1 -0
  91. package/dist/src/executors/anthropic-api.d.ts +3 -0
  92. package/dist/src/executors/anthropic-api.d.ts.map +1 -0
  93. package/dist/src/executors/anthropic-api.js +43 -0
  94. package/dist/src/executors/anthropic-api.js.map +1 -0
  95. package/dist/src/executors/claude-cli.d.ts +3 -0
  96. package/dist/src/executors/claude-cli.d.ts.map +1 -0
  97. package/dist/src/executors/claude-cli.js +103 -0
  98. package/dist/src/executors/claude-cli.js.map +1 -0
  99. package/dist/src/executors/claude-sdk-trace.d.ts +10 -0
  100. package/dist/src/executors/claude-sdk-trace.d.ts.map +1 -0
  101. package/dist/src/executors/claude-sdk-trace.js +100 -0
  102. package/dist/src/executors/claude-sdk-trace.js.map +1 -0
  103. package/dist/src/executors/claude-sdk.d.ts +3 -0
  104. package/dist/src/executors/claude-sdk.d.ts.map +1 -0
  105. package/dist/src/executors/claude-sdk.js +160 -0
  106. package/dist/src/executors/claude-sdk.js.map +1 -0
  107. package/dist/src/executors/gemini.d.ts +3 -0
  108. package/dist/src/executors/gemini.d.ts.map +1 -0
  109. package/dist/src/executors/gemini.js +78 -0
  110. package/dist/src/executors/gemini.js.map +1 -0
  111. package/dist/src/executors/index.d.ts +7 -0
  112. package/dist/src/executors/index.d.ts.map +1 -0
  113. package/dist/src/executors/index.js +22 -0
  114. package/dist/src/executors/index.js.map +1 -0
  115. package/dist/src/executors/openai-api.d.ts +3 -0
  116. package/dist/src/executors/openai-api.d.ts.map +1 -0
  117. package/dist/src/executors/openai-api.js +40 -0
  118. package/dist/src/executors/openai-api.js.map +1 -0
  119. package/dist/src/executors/openai-cli.d.ts +3 -0
  120. package/dist/src/executors/openai-cli.d.ts.map +1 -0
  121. package/dist/src/executors/openai-cli.js +60 -0
  122. package/dist/src/executors/openai-cli.js.map +1 -0
  123. package/dist/src/executors/script.d.ts +3 -0
  124. package/dist/src/executors/script.d.ts.map +1 -0
  125. package/dist/src/executors/script.js +63 -0
  126. package/dist/src/executors/script.js.map +1 -0
  127. package/dist/src/executors/shared.d.ts +117 -0
  128. package/dist/src/executors/shared.d.ts.map +1 -0
  129. package/dist/src/executors/shared.js +49 -0
  130. package/dist/src/executors/shared.js.map +1 -0
  131. package/dist/src/grading/assertions.d.ts +18 -0
  132. package/dist/src/grading/assertions.d.ts.map +1 -0
  133. package/dist/src/grading/assertions.js +239 -0
  134. package/dist/src/grading/assertions.js.map +1 -0
  135. package/dist/src/grading/index.d.ts +26 -0
  136. package/dist/src/grading/index.d.ts.map +1 -0
  137. package/dist/src/grading/index.js +98 -0
  138. package/dist/src/grading/index.js.map +1 -0
  139. package/dist/src/grading/judge.d.ts +13 -0
  140. package/dist/src/grading/judge.d.ts.map +1 -0
  141. package/dist/src/grading/judge.js +98 -0
  142. package/dist/src/grading/judge.js.map +1 -0
  143. package/dist/src/grading/layered-scores.d.ts +13 -0
  144. package/dist/src/grading/layered-scores.d.ts.map +1 -0
  145. package/dist/src/grading/layered-scores.js +62 -0
  146. package/dist/src/grading/layered-scores.js.map +1 -0
  147. package/dist/src/inputs/eval-config.d.ts +13 -0
  148. package/dist/src/inputs/eval-config.d.ts.map +1 -0
  149. package/dist/src/inputs/eval-config.js +136 -0
  150. package/dist/src/inputs/eval-config.js.map +1 -0
  151. package/dist/src/inputs/load-samples.d.ts +16 -0
  152. package/dist/src/inputs/load-samples.d.ts.map +1 -0
  153. package/dist/src/inputs/load-samples.js +51 -0
  154. package/dist/src/inputs/load-samples.js.map +1 -0
  155. package/dist/src/inputs/mcp-resolver.d.ts +50 -0
  156. package/dist/src/inputs/mcp-resolver.d.ts.map +1 -0
  157. package/dist/src/inputs/mcp-resolver.js +307 -0
  158. package/dist/src/inputs/mcp-resolver.js.map +1 -0
  159. package/dist/src/inputs/skill-loader.d.ts +18 -0
  160. package/dist/src/inputs/skill-loader.d.ts.map +1 -0
  161. package/dist/src/inputs/skill-loader.js +222 -0
  162. package/dist/src/inputs/skill-loader.js.map +1 -0
  163. package/dist/src/inputs/url-fetcher.d.ts +19 -0
  164. package/dist/src/inputs/url-fetcher.d.ts.map +1 -0
  165. package/dist/src/inputs/url-fetcher.js +241 -0
  166. package/dist/src/inputs/url-fetcher.js.map +1 -0
  167. package/dist/src/observability/production-analyzer.d.ts +61 -0
  168. package/dist/src/observability/production-analyzer.d.ts.map +1 -0
  169. package/dist/src/observability/production-analyzer.js +183 -0
  170. package/dist/src/observability/production-analyzer.js.map +1 -0
  171. package/dist/src/observability/trace-adapter.d.ts +75 -0
  172. package/dist/src/observability/trace-adapter.d.ts.map +1 -0
  173. package/dist/src/observability/trace-adapter.js +341 -0
  174. package/dist/src/observability/trace-adapter.js.map +1 -0
  175. package/dist/src/renderer/html-renderer.d.ts +9 -0
  176. package/dist/src/renderer/html-renderer.d.ts.map +1 -0
  177. package/dist/src/renderer/html-renderer.js +338 -0
  178. package/dist/src/renderer/html-renderer.js.map +1 -0
  179. package/dist/src/renderer/layout.d.ts +13 -0
  180. package/dist/src/renderer/layout.d.ts.map +1 -0
  181. package/dist/src/renderer/layout.js +476 -0
  182. package/dist/src/renderer/layout.js.map +1 -0
  183. package/dist/src/renderer/skill-health-renderer.d.ts +20 -0
  184. package/dist/src/renderer/skill-health-renderer.d.ts.map +1 -0
  185. package/dist/src/renderer/skill-health-renderer.js +218 -0
  186. package/dist/src/renderer/skill-health-renderer.js.map +1 -0
  187. package/dist/src/renderer/summary.d.ts +21 -0
  188. package/dist/src/renderer/summary.d.ts.map +1 -0
  189. package/dist/src/renderer/summary.js +1072 -0
  190. package/dist/src/renderer/summary.js.map +1 -0
  191. package/dist/src/renderer/table.d.ts +3 -0
  192. package/dist/src/renderer/table.d.ts.map +1 -0
  193. package/dist/src/renderer/table.js +134 -0
  194. package/dist/src/renderer/table.js.map +1 -0
  195. package/dist/src/renderer/trends.d.ts +6 -0
  196. package/dist/src/renderer/trends.d.ts.map +1 -0
  197. package/dist/src/renderer/trends.js +127 -0
  198. package/dist/src/renderer/trends.js.map +1 -0
  199. package/dist/src/server/job-store.d.ts +4 -0
  200. package/dist/src/server/job-store.d.ts.map +1 -0
  201. package/dist/src/server/job-store.js +68 -0
  202. package/dist/src/server/job-store.js.map +1 -0
  203. package/dist/src/server/report-server.d.ts +16 -0
  204. package/dist/src/server/report-server.d.ts.map +1 -0
  205. package/dist/src/server/report-server.js +196 -0
  206. package/dist/src/server/report-server.js.map +1 -0
  207. package/dist/src/server/report-store.d.ts +44 -0
  208. package/dist/src/server/report-store.d.ts.map +1 -0
  209. package/dist/src/server/report-store.js +190 -0
  210. package/dist/src/server/report-store.js.map +1 -0
  211. package/dist/src/types.d.ts +553 -0
  212. package/dist/src/types.d.ts.map +1 -0
  213. package/dist/src/types.js +2 -0
  214. package/dist/src/types.js.map +1 -0
  215. package/package.json +44 -3
@@ -0,0 +1,264 @@
1
+ import { resolve } from 'node:path';
2
+ import { DEFAULT_OUTPUT_DIR, persistReport } from '../eval-core/evaluation-reporting.js';
3
+ import { createExecutor, DEFAULT_MODEL, JUDGE_MODEL } from '../executors/index.js';
4
+ import { discoverEachSkills } from '../inputs/skill-loader.js';
5
+ import { confidenceInterval, tTest, effectSize } from '../eval-core/statistics.js';
6
+ import { executeEachEvaluationRuns } from './each-evaluation-workflow.js';
7
+ import { buildDryRunEachArtifacts, buildDryRunTaskReport, prepareEvaluationRun, } from './evaluation-preparation.js';
8
+ import { executeEvaluationPipeline } from './evaluation-pipeline.js';
9
+ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], artifacts, model = DEFAULT_MODEL, judgeModel = JUDGE_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, blind = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', judgeExecutorName, jobStore = null, persistJob = true, onProgress = null, skipPreflight = false, mcpConfig, verbose = false, retry = 0, resume, layeredStats = false, }) {
10
+ const { samples, artifacts: resolvedArtifacts, tasks, variantNames, requires } = await prepareEvaluationRun({
11
+ samplesPath,
12
+ skillDir,
13
+ variantSpecs,
14
+ artifacts,
15
+ dryRun,
16
+ mcpConfig,
17
+ });
18
+ if (dryRun) {
19
+ return {
20
+ report: buildDryRunTaskReport({
21
+ model,
22
+ judgeModel,
23
+ executorName,
24
+ samplesPath,
25
+ skillDir,
26
+ tasks,
27
+ variantNames,
28
+ }),
29
+ filePath: null,
30
+ };
31
+ }
32
+ // --resume: load existing report results to skip completed tasks
33
+ let existingResults;
34
+ if (resume) {
35
+ const { createFileStore } = await import('../server/report-store.js');
36
+ const store = createFileStore(resolve(outputDir || DEFAULT_OUTPUT_DIR));
37
+ const existing = await store.get(resume);
38
+ if (existing) {
39
+ existingResults = {};
40
+ for (const entry of existing.results || []) {
41
+ existingResults[entry.sample_id] = entry.variants;
42
+ }
43
+ if (onProgress) {
44
+ const count = Object.values(existingResults).reduce((sum, v) => sum + Object.values(v).filter((r) => r.ok).length, 0);
45
+ process.stderr.write(`\n📂 从报告 ${resume} 恢复了 ${count} 个已完成结果\n`);
46
+ }
47
+ }
48
+ else {
49
+ process.stderr.write(`\n⚠️ 报告 ${resume} 未找到,将从头执行\n`);
50
+ }
51
+ }
52
+ const executor = createExecutor(executorName);
53
+ const judgeExecutor = createExecutor(judgeExecutorName || executorName);
54
+ return executeEvaluationPipeline({
55
+ samplesPath,
56
+ skillDir,
57
+ samples,
58
+ tasks,
59
+ artifacts: resolvedArtifacts,
60
+ model,
61
+ judgeModel,
62
+ noJudge,
63
+ executorName,
64
+ judgeExecutorName: judgeExecutorName || executorName,
65
+ executor,
66
+ judgeExecutor,
67
+ outputDir,
68
+ project,
69
+ owner,
70
+ tags,
71
+ blind,
72
+ concurrency,
73
+ timeoutMs,
74
+ noCache,
75
+ jobStore,
76
+ persistJob,
77
+ onProgress,
78
+ skipPreflight,
79
+ verbose,
80
+ retry,
81
+ existingResults,
82
+ requires,
83
+ layeredStats,
84
+ });
85
+ }
86
+ // Top-level (composite) extractor: feeds the legacy flat-field variance on
87
+ // VariantVariance / VarianceComparison. Composite lives here for backward
88
+ // compatibility with historical reports; layer-independent stats are attached
89
+ // via byLayer (see LAYER_EXTRACTORS) starting v0.16 work item B / PR-2.
90
+ const COMPOSITE_EXTRACTOR = (s) => s?.avgCompositeScore;
91
+ // Non-quality metric extractors tracked in byMetric (cost + efficiency).
92
+ const METRIC_EXTRACTORS = {
93
+ cost: (s) => s?.totalExecCostUSD,
94
+ efficiency: (s) => s?.avgDurationMs,
95
+ };
96
+ // Three-layer extractors (PR-2): fact / behavior / judge each get their own
97
+ // independent variance + t-test + effect size. Lets "judge ↑ 0.8, fact ↑ 0.1"
98
+ // structural signals surface instead of getting diluted by the composite average.
99
+ // `judge` is the rubric-based LLM judge score (UI 中文: "LLM 评价").
100
+ const LAYER_EXTRACTORS = {
101
+ fact: (s) => s?.avgFactScore,
102
+ behavior: (s) => s?.avgBehaviorScore,
103
+ judge: (s) => s?.avgJudgeScore,
104
+ };
105
+ function buildMetricStats(runs, variant, extractor) {
106
+ const scores = runs
107
+ .map((run) => extractor(run.summary?.[variant]))
108
+ .filter((x) => typeof x === 'number');
109
+ if (scores.length === 0)
110
+ return null;
111
+ return { scores, ...confidenceInterval(scores) };
112
+ }
113
+ function buildComparisonMetric(scoresA, scoresB, meanA, meanB) {
114
+ const t = tTest(scoresA, scoresB);
115
+ const es = effectSize(scoresA, scoresB);
116
+ const meanDiff = Number((meanA - meanB).toFixed(4));
117
+ return {
118
+ meanDiff,
119
+ tStatistic: t.tStatistic,
120
+ df: t.df,
121
+ significant: t.significant,
122
+ effectSize: es,
123
+ };
124
+ }
125
+ export function buildVarianceData(runs) {
126
+ if (runs.length <= 1) {
127
+ return null;
128
+ }
129
+ const variants = runs[0].meta.variants || [];
130
+ const perVariant = {};
131
+ for (const variant of variants) {
132
+ // Composite lives on the legacy flat fields.
133
+ const composite = buildMetricStats(runs, variant, COMPOSITE_EXTRACTOR);
134
+ if (!composite)
135
+ continue;
136
+ const byMetric = {};
137
+ const cost = buildMetricStats(runs, variant, METRIC_EXTRACTORS.cost);
138
+ if (cost)
139
+ byMetric.cost = cost;
140
+ const efficiency = buildMetricStats(runs, variant, METRIC_EXTRACTORS.efficiency);
141
+ if (efficiency)
142
+ byMetric.efficiency = efficiency;
143
+ // Three-layer independent stats (PR-2). Any layer with no data across runs is omitted.
144
+ const byLayer = {};
145
+ for (const key of ['fact', 'behavior', 'judge']) {
146
+ const layerStats = buildMetricStats(runs, variant, LAYER_EXTRACTORS[key]);
147
+ if (layerStats)
148
+ byLayer[key] = layerStats;
149
+ }
150
+ perVariant[variant] = {
151
+ ...composite,
152
+ ...(Object.keys(byMetric).length > 0 ? { byMetric } : {}),
153
+ ...(Object.keys(byLayer).length > 0 ? { byLayer } : {}),
154
+ };
155
+ }
156
+ const comparisons = [];
157
+ for (let i = 0; i < variants.length; i++) {
158
+ for (let j = i + 1; j < variants.length; j++) {
159
+ const vA = perVariant[variants[i]];
160
+ const vB = perVariant[variants[j]];
161
+ if (!vA || !vB)
162
+ continue;
163
+ const compositeComp = buildComparisonMetric(vA.scores, vB.scores, vA.mean, vB.mean);
164
+ const byMetricComp = {};
165
+ for (const key of ['cost', 'efficiency']) {
166
+ const mA = vA.byMetric?.[key];
167
+ const mB = vB.byMetric?.[key];
168
+ if (mA && mB) {
169
+ byMetricComp[key] = buildComparisonMetric(mA.scores, mB.scores, mA.mean, mB.mean);
170
+ }
171
+ }
172
+ // Three-layer comparison (PR-2). Each layer pair runs its own Welch's t + Cohen's d.
173
+ const byLayerComp = {};
174
+ for (const key of ['fact', 'behavior', 'judge']) {
175
+ const lA = vA.byLayer?.[key];
176
+ const lB = vB.byLayer?.[key];
177
+ if (lA && lB) {
178
+ byLayerComp[key] = buildComparisonMetric(lA.scores, lB.scores, lA.mean, lB.mean);
179
+ }
180
+ }
181
+ comparisons.push({
182
+ a: variants[i],
183
+ b: variants[j],
184
+ ...compositeComp,
185
+ ...(Object.keys(byMetricComp).length > 0 ? { byMetric: byMetricComp } : {}),
186
+ ...(Object.keys(byLayerComp).length > 0 ? { byLayer: byLayerComp } : {}),
187
+ });
188
+ }
189
+ }
190
+ return { runs: runs.length, perVariant, comparisons };
191
+ }
192
+ export async function runEachEvaluation({ skillDir, model = DEFAULT_MODEL, judgeModel = JUDGE_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, executorName = 'claude', judgeExecutorName, jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipPreflight = false, mcpConfig, verbose = false, }) {
193
+ const skillEntries = discoverEachSkills(resolve(skillDir));
194
+ if (skillEntries.length === 0) {
195
+ throw new Error(`未发现带配对 eval-samples 的 skill:${skillDir}`);
196
+ }
197
+ if (dryRun) {
198
+ const { artifacts: dryArtifacts, totalTasks } = buildDryRunEachArtifacts(skillEntries);
199
+ return {
200
+ report: {
201
+ dryRun: true,
202
+ each: true,
203
+ model,
204
+ judgeModel,
205
+ executor: executorName,
206
+ skillDir,
207
+ totalArtifacts: dryArtifacts.length,
208
+ totalTasks,
209
+ artifacts: dryArtifacts,
210
+ },
211
+ filePath: null,
212
+ };
213
+ }
214
+ return executeEachEvaluationRuns({
215
+ skillDir,
216
+ skillEntries,
217
+ model,
218
+ judgeModel,
219
+ outputDir,
220
+ project,
221
+ owner,
222
+ tags,
223
+ noJudge,
224
+ concurrency,
225
+ timeoutMs,
226
+ executorName,
227
+ judgeExecutorName,
228
+ jobStore,
229
+ persistJob,
230
+ onProgress,
231
+ onSkillProgress,
232
+ skipPreflight,
233
+ mcpConfig,
234
+ verbose,
235
+ runSingleEvaluation: async (options) => {
236
+ const result = await runEvaluation(options);
237
+ return { report: result.report, filePath: result.filePath };
238
+ },
239
+ });
240
+ }
241
+ export async function runMultiple({ repeat = 1, onRepeatProgress, ...config }) {
242
+ const runs = [];
243
+ const savedOutputDir = config.outputDir;
244
+ for (let i = 0; i < repeat; i++) {
245
+ onRepeatProgress?.({ run: i + 1, total: repeat });
246
+ const isLast = i === repeat - 1;
247
+ const { report } = await runEvaluation({
248
+ ...config,
249
+ outputDir: isLast ? savedOutputDir : null,
250
+ persistJob: isLast,
251
+ });
252
+ runs.push(report);
253
+ }
254
+ const report = runs[runs.length - 1];
255
+ const aggregated = buildVarianceData(runs);
256
+ let filePath = null;
257
+ if (aggregated) {
258
+ report.variance = aggregated;
259
+ // pipeline 内部在 variance 赋值前已写入报告,需要重新写一次以包含方差数据
260
+ filePath = persistReport(report, savedOutputDir || DEFAULT_OUTPUT_DIR);
261
+ }
262
+ return { report, aggregated, filePath };
263
+ }
264
+ //# sourceMappingURL=run-evaluation.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run-evaluation.js","sourceRoot":"","sources":["../../../src/eval-workflows/run-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACzF,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACnF,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AAuGrE,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,EAClC,WAAW,EACX,QAAQ,EACR,YAAY,GAAG,EAAE,EACjB,SAAS,EACT,KAAK,GAAG,aAAa,EACrB,UAAU,GAAG,WAAW,EACxB,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,KAAK,EACf,MAAM,GAAG,KAAK,EACd,KAAK,GAAG,KAAK,EACb,WAAW,GAAG,CAAC,EACf,SAAS,EACT,OAAO,GAAG,KAAK,EACf,YAAY,GAAG,QAAQ,EACvB,iBAAiB,EACjB,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,aAAa,GAAG,KAAK,EACrB,SAAS,EACT,OAAO,GAAG,KAAK,EACf,KAAK,GAAG,CAAC,EACT,MAAM,EACN,YAAY,GAAG,KAAK,GACC;IACrB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,MAAM,oBAAoB,CAAC;QAC1G,WAAW;QACX,QAAQ;QACR,YAAY;QACZ,SAAS;QACT,MAAM;QACN,SAAS;KACV,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,MAAM,EAAE,qBAAqB,CAAC;gBAC5B,KAAK;gBACL,UAAU;gBACV,YAAY;gBACZ,WAAW;gBACX,QAAQ;gBACR,KAAK;gBACL,YAAY;aACb,CAAC;YACF,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,IAAI,eAAgG,CAAC;IACrG,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,GAAG,EAAE,CAAC;YACrB,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;gBAC3C,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC;YACpD,CAAC;YACD,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACtH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,QAAQ,KAAK,WAAW,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,cAAc,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAe,cAAc,CAAC,YAAY,CAAC,CAAC;IAC1D,MAAM,aAAa,GAAe,cAAc,CAAC,iBAAiB,IAAI,YAAY,CAAC,CAAC;IACpF,OAAO,yBAAyB,CAAC;QAC/B,WAAW;QACX,QAAQ;QACR,OAAO;QACP,KAAK;QACL,SAAS,EAAE,iBAAiB;QAC5B,KAAK;QACL,UAAU;QACV,OAAO;QACP,YAAY;QACZ,iBAAiB,EAAE,iBAAiB,IAAI,YAAY;QACpD,QAAQ;QACR,aAAa;QACb,SAAS;QACT,OAAO;QACP,KAAK;QACL,IAAI;QACJ,KAAK;QACL,WAAW;QACX,SAAS;QACT,OAAO;QACP,QAAQ;QACR,UAAU;QACV,UAAU;QACV,aAAa;QACb,OAAO;QACP,KAAK;QACL,eAAe;QACf,QAAQ;QACR,YAAY;KACb,CAAC,CAAC;AACL,CAAC;AAeD,2EAA2E;AAC3E,0EAA0E;AAC1E,8EAA8E;AAC9E,wEAAwE;AACxE,MAAM,mBAAmB,GAAG,CAAC,CAA6B,EAAsB,EAAE,CAAC,CAAC,EAAE,iBAAiB,CAAC;AAExG,yEAAyE;AACzE,MAAM,iBAAiB,GAAyF;IAC9G,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,gBAAgB;IAChC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,aAAa;CACpC,CAAC;AAEF,4EAA4E;AAC5E,8EAA8E;AAC9E,kFAAkF;AAClF,iEAAiE;AACjE,MAAM,gBAAgB,GAAoF;IACxG,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,YAAY;IAC5B,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,gBAAgB;IACpC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,aAAa;CAC/B,CAAC;AAEF,SAAS,gBAAgB,CAAC,IAAc,EAAE,OAAe,EAAE,SAAgE;IACzH,MAAM,MAAM,GAAG,IAAI;SAChB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;SAC/C,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAiB,EAAE,OAAiB,EAAE,KAAa,EAAE,KAAa;IAC/F,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,OAAO;QACL,QAAQ;QACR,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,UAAU,EAAE,EAAE;KACf,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAc;IAC9C,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;IAC7C,MAAM,UAAU,GAAoC,EAAE,CAAC;IACvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,6CAA6C;QAC7C,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC;QACvE,IAAI,CAAC,SAAS;YAAE,SAAS;QAEzB,MAAM,QAAQ,GAA2D,EAAE,CAAC;QAC5E,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACrE,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC;QAC/B,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,CAAC;QACjF,IAAI,UAAU;YAAE,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;QAEjD,uFAAuF;QACvF,MAAM,OAAO,GAAsD,EAAE,CAAC;QACtE,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE,CAAC;YACzD,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1E,IAAI,UAAU;gBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;QAC5C,CAAC;QAED,UAAU,CAAC,OAAO,CAAC,GAAG;YACpB,GAAG,SAAS;YACZ,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAyB,EAAE,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE;gBAAE,SAAS;YAEzB,MAAM,aAAa,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YAEpF,MAAM,YAAY,GAAqE,EAAE,CAAC;YAC1F,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,YAAY,CAAU,EAAE,CAAC;gBAClD,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;oBACb,YAAY,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBACpF,CAAC;YACH,CAAC;YAED,qFAAqF;YACrF,MAAM,WAAW,GAAgE,EAAE,CAAC;YACpF,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE,CAAC;gBACzD,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC7B,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;oBACb,WAAW,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBACnF,CAAC;YACH,CAAC;YAED,WAAW,CAAC,IAAI,CAAC;gBACf,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACd,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACd,GAAG,aAAa;gBAChB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3E,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACzE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EACtC,QAAQ,EACR,KAAK,GAAG,aAAa,EACrB,UAAU,GAAG,WAAW,EACxB,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,KAAK,EACf,MAAM,GAAG,KAAK,EACd,WAAW,GAAG,CAAC,EACf,SAAS,EACT,YAAY,GAAG,QAAQ,EACvB,iBAAiB,EACjB,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,eAAe,GAAG,IAAI,EACtB,aAAa,GAAG,KAAK,EACrB,SAAS,EACT,OAAO,GAAG,KAAK,GACU;IACzB,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3D,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,wBAAwB,CAAC,YAAY,CAAC,CAAC;QACvF,OAAO;YACL,MAAM,EAAE;gBACN,MAAM,EAAE,IAAI;gBACZ,IAAI,EAAE,IAAI;gBACV,KAAK;gBACL,UAAU;gBACV,QAAQ,EAAE,YAAY;gBACtB,QAAQ;gBACR,cAAc,EAAE,YAAY,CAAC,MAAM;gBACnC,UAAU;gBACV,SAAS,EAAE,YAAY;aACxB;YACD,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IACD,OAAO,yBAAyB,CAAC;QAC/B,QAAQ;QACR,YAAY;QACZ,KAAK;QACL,UAAU;QACV,SAAS;QACT,OAAO;QACP,KAAK;QACL,IAAI;QACJ,OAAO;QACP,WAAW;QACX,SAAS;QACT,YAAY;QACZ,iBAAiB;QACjB,QAAQ;QACR,UAAU;QACV,UAAU;QACV,eAAe;QACf,aAAa;QACb,SAAS;QACT,OAAO;QACP,mBAAmB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACrC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;YAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAgB,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxE,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,gBAAgB,EAAE,GAAG,MAAM,EAAsB;IAC/F,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC;IAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,gBAAgB,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC;QAChC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC;YACrC,GAAG,MAAM;YACT,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI;YACzC,UAAU,EAAE,MAAM;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,MAAgB,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;QAC7B,gDAAgD;QAChD,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,cAAc,IAAI,kBAAkB,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
@@ -0,0 +1,3 @@
1
+ import type { ExecResult, ExecutorInput } from '../types.js';
2
+ export declare function anthropicApiExecutor({ model, system, prompt, timeoutMs }: ExecutorInput): Promise<ExecResult>;
3
+ //# sourceMappingURL=anthropic-api.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"anthropic-api.d.ts","sourceRoot":"","sources":["../../../src/executors/anthropic-api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG7D,wBAAsB,oBAAoB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAA8B,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,CA8CxI"}
@@ -0,0 +1,43 @@
1
+ import { asErrorLike, DEFAULT_TIMEOUT_MS, errorMessage } from './shared.js';
2
+ export async function anthropicApiExecutor({ model, system, prompt, timeoutMs = DEFAULT_TIMEOUT_MS }) {
3
+ const apiKey = process.env.ANTHROPIC_API_KEY;
4
+ if (!apiKey)
5
+ throw new Error('ANTHROPIC_API_KEY 环境变量未设置');
6
+ const baseUrl = process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com';
7
+ const reqBody = {
8
+ model: model || 'claude-sonnet-4-5-20250514',
9
+ max_tokens: 8192,
10
+ messages: [{ role: 'user', content: prompt }],
11
+ };
12
+ if (system)
13
+ reqBody.system = system;
14
+ const start = Date.now();
15
+ try {
16
+ const res = await fetch(`${baseUrl}/v1/messages`, {
17
+ method: 'POST',
18
+ headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey, 'anthropic-version': '2023-06-01' },
19
+ body: JSON.stringify(reqBody),
20
+ signal: AbortSignal.timeout(timeoutMs),
21
+ });
22
+ const data = await res.json();
23
+ const durationMs = Date.now() - start;
24
+ if (!res.ok) {
25
+ return { ok: false, error: data.error?.message || `API error ${res.status}`, durationMs, durationApiMs: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0, costUSD: 0, output: null, stopReason: 'error', numTurns: 0 };
26
+ }
27
+ const usage = data.usage || {};
28
+ return {
29
+ ok: true, output: data.content?.map((c) => c.text || '').join('') || '', durationMs, durationApiMs: 0,
30
+ inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0,
31
+ cacheReadTokens: usage.cache_read_input_tokens || 0, cacheCreationTokens: usage.cache_creation_input_tokens || 0,
32
+ costUSD: 0, stopReason: data.stop_reason || 'end_turn', numTurns: 1,
33
+ };
34
+ }
35
+ catch (err) {
36
+ const durationMs = Date.now() - start;
37
+ const details = asErrorLike(err);
38
+ const stopReason = details.name === 'TimeoutError' ? 'timeout' : 'error';
39
+ const error = details.name === 'TimeoutError' ? `API request timed out after ${timeoutMs / 1000}s` : errorMessage(err);
40
+ return { ok: false, error, durationMs, durationApiMs: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0, costUSD: 0, output: null, stopReason, numTurns: 0 };
41
+ }
42
+ }
43
+ //# sourceMappingURL=anthropic-api.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"anthropic-api.js","sourceRoot":"","sources":["../../../src/executors/anthropic-api.ts"],"names":[],"mappings":"AACA,OAAO,EAAqB,WAAW,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE/F,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,kBAAkB,EAAiB;IACjH,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAE1D,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,2BAA2B,CAAC;IAC9E,MAAM,OAAO,GAKT;QACF,KAAK,EAAE,KAAK,IAAI,4BAA4B;QAC5C,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;KAC9C,CAAC;IACF,IAAI,MAAM;QAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;IAEpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,cAAc,EAAE;YAChD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,EAAE,mBAAmB,EAAE,YAAY,EAAE;YACvG,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;YAC7B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;SACvC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAuB,CAAC;QACnD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEtC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,aAAa,GAAG,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QACvP,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/B,OAAO;YACL,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;YACrG,WAAW,EAAE,KAAK,CAAC,YAAY,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC;YAC5E,eAAe,EAAE,KAAK,CAAC,uBAAuB,IAAI,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,2BAA2B,IAAI,CAAC;YAChH,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,WAAW,IAAI,UAAU,EAAE,QAAQ,EAAE,CAAC;SACpE,CAAC;IACJ,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;QACzE,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,+BAA+B,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACvH,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IAC5L,CAAC;AACH,CAAC"}
@@ -0,0 +1,3 @@
1
+ import type { ExecResult, ExecutorInput } from '../types.js';
2
+ export declare function claudeCliExecutor({ model, system, prompt, cwd, skillDir, timeoutMs }: ExecutorInput): Promise<ExecResult>;
3
+ //# sourceMappingURL=claude-cli.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"claude-cli.d.ts","sourceRoot":"","sources":["../../../src/executors/claude-cli.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAyB7D,wBAAsB,iBAAiB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,SAA8B,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,CA2FpJ"}
@@ -0,0 +1,103 @@
1
+ import { extractAgentTrace, isClaudeSdkResultMessage } from './claude-sdk-trace.js';
2
+ import { asErrorLike, buildExecEnv, DEFAULT_TIMEOUT_MS, errorMessage, execFileAsync, MAX_BUFFER, timeoutExecResult, } from './shared.js';
3
+ function parseStreamJson(stdout) {
4
+ const messages = [];
5
+ for (const line of stdout.split('\n')) {
6
+ const trimmed = line.trim();
7
+ if (!trimmed)
8
+ continue;
9
+ try {
10
+ messages.push(JSON.parse(trimmed));
11
+ }
12
+ catch { /* skip non-JSON lines */ }
13
+ }
14
+ return messages;
15
+ }
16
+ export async function claudeCliExecutor({ model, system, prompt, cwd, skillDir, timeoutMs = DEFAULT_TIMEOUT_MS }) {
17
+ const args = ['-p', prompt, '--output-format', 'stream-json', '--verbose', '--model', model];
18
+ if (system)
19
+ args.push('--system-prompt', system);
20
+ const env = buildExecEnv(skillDir);
21
+ const start = Date.now();
22
+ try {
23
+ const { stdout } = await execFileAsync('claude', args, {
24
+ maxBuffer: MAX_BUFFER,
25
+ timeout: timeoutMs,
26
+ env,
27
+ ...(cwd && { cwd }),
28
+ });
29
+ const durationMs = Date.now() - start;
30
+ const messages = parseStreamJson(stdout);
31
+ // 提取 result 消息
32
+ const resultMsgs = messages.filter(isClaudeSdkResultMessage);
33
+ if (resultMsgs.length === 0) {
34
+ return {
35
+ ok: false, error: 'no result message in stream-json output',
36
+ durationMs, durationApiMs: 0,
37
+ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0,
38
+ costUSD: 0, output: null, stopReason: 'error', numTurns: 0,
39
+ };
40
+ }
41
+ const last = resultMsgs[resultMsgs.length - 1];
42
+ const usage = last.usage || {};
43
+ // 提取 trace
44
+ const trace = extractAgentTrace(messages);
45
+ return {
46
+ ok: !last.errors?.length && last.subtype !== 'error',
47
+ durationMs: last.duration_ms || durationMs,
48
+ durationApiMs: last.duration_api_ms || 0,
49
+ inputTokens: usage.input_tokens || 0,
50
+ outputTokens: usage.output_tokens || 0,
51
+ cacheReadTokens: usage.cache_read_input_tokens || 0,
52
+ cacheCreationTokens: usage.cache_creation_input_tokens || 0,
53
+ costUSD: last.total_cost_usd || 0,
54
+ output: last.result || '',
55
+ stopReason: last.subtype || 'unknown',
56
+ numTurns: last.num_turns || 1,
57
+ fullNumTurns: trace.fullNumTurns,
58
+ numSubAgents: trace.numSubAgents,
59
+ ...(trace.turns.length > 0 && { turns: trace.turns }),
60
+ ...(trace.toolCalls.length > 0 && { toolCalls: trace.toolCalls }),
61
+ };
62
+ }
63
+ catch (err) {
64
+ const durationMs = Date.now() - start;
65
+ const details = asErrorLike(err);
66
+ if (details.killed)
67
+ return timeoutExecResult(timeoutMs, durationMs);
68
+ // 尝试从 stdout 解析 stream-json(即使进程退出码非 0 也可能有 result)
69
+ const messages = parseStreamJson(details.stdout || '');
70
+ const resultMsgs = messages.filter(isClaudeSdkResultMessage);
71
+ if (resultMsgs.length > 0) {
72
+ const last = resultMsgs[resultMsgs.length - 1];
73
+ const usage = last.usage || {};
74
+ const trace = extractAgentTrace(messages);
75
+ return {
76
+ ok: false,
77
+ error: last.errors?.join('; ') || last.result || errorMessage(err),
78
+ durationMs: last.duration_ms || durationMs,
79
+ durationApiMs: last.duration_api_ms || 0,
80
+ inputTokens: usage.input_tokens || 0,
81
+ outputTokens: usage.output_tokens || 0,
82
+ cacheReadTokens: usage.cache_read_input_tokens || 0,
83
+ cacheCreationTokens: usage.cache_creation_input_tokens || 0,
84
+ costUSD: last.total_cost_usd || 0,
85
+ output: last.result || null,
86
+ stopReason: 'error',
87
+ numTurns: last.num_turns || 0,
88
+ fullNumTurns: trace.fullNumTurns,
89
+ numSubAgents: trace.numSubAgents,
90
+ ...(trace.turns.length > 0 && { turns: trace.turns }),
91
+ ...(trace.toolCalls.length > 0 && { toolCalls: trace.toolCalls }),
92
+ };
93
+ }
94
+ return {
95
+ ok: false,
96
+ error: errorMessage(err),
97
+ durationMs, durationApiMs: 0,
98
+ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0,
99
+ costUSD: 0, output: null, stopReason: 'error', numTurns: 0,
100
+ };
101
+ }
102
+ }
103
+ //# sourceMappingURL=claude-cli.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"claude-cli.js","sourceRoot":"","sources":["../../../src/executors/claude-cli.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AAEpF,OAAO,EACL,WAAW,EACX,YAAY,EACZ,kBAAkB,EAClB,YAAY,EACZ,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,aAAa,CAAC;AAErB,SAAS,eAAe,CAAC,MAAc;IACrC,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,IAAI,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAyB,CAAC,CAAC;QAC7D,CAAC;QAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,GAAG,kBAAkB,EAAiB;IAC7H,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,EAAE,aAAa,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC7F,IAAI,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IAEjD,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEnC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE;YACrD,SAAS,EAAE,UAAU;YACrB,OAAO,EAAE,SAAS;YAClB,GAAG;YACH,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;SACpB,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QAEzC,eAAe;QACf,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,wBAAwB,CAA6B,CAAC;QACzF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO;gBACL,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,yCAAyC;gBAC3D,UAAU,EAAE,aAAa,EAAE,CAAC;gBAC5B,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC;gBAC3E,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;aAC3D,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAE/B,WAAW;QACX,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAE1C,OAAO;YACL,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO;YACpD,UAAU,EAAE,IAAI,CAAC,WAAW,IAAI,UAAU;YAC1C,aAAa,EAAE,IAAI,CAAC,eAAe,IAAI,CAAC;YACxC,WAAW,EAAE,KAAK,CAAC,YAAY,IAAI,CAAC;YACpC,YAAY,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC;YACtC,eAAe,EAAE,KAAK,CAAC,uBAAuB,IAAI,CAAC;YACnD,mBAAmB,EAAE,KAAK,CAAC,2BAA2B,IAAI,CAAC;YAC3D,OAAO,EAAE,IAAI,CAAC,cAAc,IAAI,CAAC;YACjC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;YACzB,UAAU,EAAE,IAAI,CAAC,OAAO,IAAI,SAAS;YACrC,QAAQ,EAAE,IAAI,CAAC,SAAS,IAAI,CAAC;YAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;YACrD,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC;SAClE,CAAC;IACJ,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,OAAO,CAAC,MAAM;YAAE,OAAO,iBAAiB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAEpE,oDAAoD;QACpD,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACvD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,wBAAwB,CAA6B,CAAC;QACzF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAC1C,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,GAAG,CAAC;gBAClE,UAAU,EAAE,IAAI,CAAC,WAAW,IAAI,UAAU;gBAC1C,aAAa,EAAE,IAAI,CAAC,eAAe,IAAI,CAAC;gBACxC,WAAW,EAAE,KAAK,CAAC,YAAY,IAAI,CAAC;gBACpC,YAAY,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC;gBACtC,eAAe,EAAE,KAAK,CAAC,uBAAuB,IAAI,CAAC;gBACnD,mBAAmB,EAAE,KAAK,CAAC,2BAA2B,IAAI,CAAC;gBAC3D,OAAO,EAAE,IAAI,CAAC,cAAc,IAAI,CAAC;gBACjC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;gBAC3B,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,IAAI,CAAC,SAAS,IAAI,CAAC;gBAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBACrD,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC;aAClE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC;YACxB,UAAU,EAAE,aAAa,EAAE,CAAC;YAC5B,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC;YAC3E,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;SAC3D,CAAC;IACJ,CAAC;AACH,CAAC"}
@@ -0,0 +1,10 @@
1
+ import type { ToolCallInfo, TurnInfo } from '../types.js';
2
+ import type { ClaudeSdkBaseMessage } from './shared.js';
3
+ export declare function isClaudeSdkResultMessage(message: ClaudeSdkBaseMessage): boolean;
4
+ export declare function extractAgentTrace(messages: ClaudeSdkBaseMessage[], timestamps?: number[]): {
5
+ turns: TurnInfo[];
6
+ toolCalls: ToolCallInfo[];
7
+ fullNumTurns: number;
8
+ numSubAgents: number;
9
+ };
10
+ //# sourceMappingURL=claude-sdk-trace.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"claude-sdk-trace.d.ts","sourceRoot":"","sources":["../../../src/executors/claude-sdk-trace.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC1D,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAExD,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAE/E;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG;IAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IAAC,SAAS,EAAE,YAAY,EAAE,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAgGvL"}
@@ -0,0 +1,100 @@
1
+ export function isClaudeSdkResultMessage(message) {
2
+ return message.type === 'result';
3
+ }
4
+ export function extractAgentTrace(messages, timestamps) {
5
+ const turns = [];
6
+ const toolCalls = [];
7
+ const pendingToolUse = new Map();
8
+ let lastTurnTs = timestamps?.[0] || 0;
9
+ let fullNumTurns = 0;
10
+ let numSubAgents = 0;
11
+ for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) {
12
+ const msg = messages[msgIdx];
13
+ const msgTs = timestamps?.[msgIdx] || 0;
14
+ if (msg.type === 'result' || msg.type === 'system' || msg.type === 'rate_limit_event')
15
+ continue;
16
+ const content = msg.message?.content;
17
+ if (!Array.isArray(content))
18
+ continue;
19
+ if (msg.type === 'assistant') {
20
+ const textParts = [];
21
+ const turnToolCalls = [];
22
+ let hasNonThinking = false;
23
+ for (const block of content) {
24
+ if (block.type === 'text' && block.text) {
25
+ textParts.push(block.text);
26
+ hasNonThinking = true;
27
+ }
28
+ else if (block.type === 'tool_use' && block.name) {
29
+ pendingToolUse.set(block.id || '', { tool: block.name, input: block.input });
30
+ turnToolCalls.push({ tool: block.name, input: block.input, output: null, success: true });
31
+ hasNonThinking = true;
32
+ if (block.name === 'Agent')
33
+ numSubAgents++;
34
+ }
35
+ else if (block.type !== 'thinking') {
36
+ hasNonThinking = true;
37
+ }
38
+ }
39
+ if (hasNonThinking)
40
+ fullNumTurns++;
41
+ if (textParts.length > 0 || turnToolCalls.length > 0) {
42
+ const dur = msgTs && lastTurnTs ? msgTs - lastTurnTs : undefined;
43
+ turns.push({
44
+ role: 'assistant',
45
+ content: textParts.join('\n'),
46
+ ...(turnToolCalls.length > 0 && { toolCalls: turnToolCalls }),
47
+ ...(dur != null && dur > 0 && { durationMs: dur }),
48
+ });
49
+ if (msgTs)
50
+ lastTurnTs = msgTs;
51
+ }
52
+ }
53
+ if (msg.type === 'user') {
54
+ for (const block of content) {
55
+ if (block.type !== 'tool_result')
56
+ continue;
57
+ const toolUseId = block.tool_use_id || '';
58
+ const pending = pendingToolUse.get(toolUseId);
59
+ const isError = block.is_error || false;
60
+ const resultContent = block.content;
61
+ const outputText = typeof resultContent === 'string'
62
+ ? resultContent
63
+ : Array.isArray(resultContent)
64
+ ? resultContent.map((c) => c.text || '').join('')
65
+ : '';
66
+ const tc = {
67
+ tool: pending?.tool || 'unknown',
68
+ input: pending?.input || null,
69
+ output: outputText.slice(0, 500),
70
+ success: !isError,
71
+ };
72
+ toolCalls.push(tc);
73
+ if (pending) {
74
+ for (let i = turns.length - 1; i >= 0; i--) {
75
+ const turn = turns[i];
76
+ if (turn.role === 'assistant' && turn.toolCalls) {
77
+ const placeholder = turn.toolCalls.find((t) => t.tool === pending.tool && t.output === null);
78
+ if (placeholder) {
79
+ placeholder.output = tc.output;
80
+ placeholder.success = !isError;
81
+ break;
82
+ }
83
+ }
84
+ }
85
+ }
86
+ const toolDur = msgTs && lastTurnTs ? msgTs - lastTurnTs : undefined;
87
+ turns.push({
88
+ role: 'tool',
89
+ content: outputText.slice(0, 500),
90
+ ...(toolDur != null && toolDur > 0 && { durationMs: toolDur }),
91
+ });
92
+ if (msgTs)
93
+ lastTurnTs = msgTs;
94
+ pendingToolUse.delete(toolUseId);
95
+ }
96
+ }
97
+ }
98
+ return { turns, toolCalls, fullNumTurns, numSubAgents };
99
+ }
100
+ //# sourceMappingURL=claude-sdk-trace.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"claude-sdk-trace.js","sourceRoot":"","sources":["../../../src/executors/claude-sdk-trace.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,wBAAwB,CAAC,OAA6B;IACpE,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAAgC,EAAE,UAAqB;IACvF,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,MAAM,cAAc,GAAG,IAAI,GAAG,EAA4C,CAAC;IAC3E,IAAI,UAAU,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QACxD,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,KAAK,kBAAkB;YAAE,SAAS;QAEhG,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,SAAS;QAEtC,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,MAAM,aAAa,GAAmB,EAAE,CAAC;YACzC,IAAI,cAAc,GAAG,KAAK,CAAC;YAE3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACxC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC3B,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACnD,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;oBAC7E,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC1F,cAAc,GAAG,IAAI,CAAC;oBACtB,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;wBAAE,YAAY,EAAE,CAAC;gBAC7C,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;oBACrC,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,IAAI,cAAc;gBAAE,YAAY,EAAE,CAAC;YAEnC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrD,MAAM,GAAG,GAAG,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;gBACjE,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAC7B,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC;oBAC7D,GAAG,CAAC,GAAG,IAAI,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;iBACnD,CAAC,CAAC;gBACH,IAAI,KAAK;oBAAE,UAAU,GAAG,KAAK,CAAC;YAChC,CAAC;QACH,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa;oBAAE,SAAS;gBAC3C,MAAM,SAAS,GAAI,KAA6C,CAAC,WAAW,IAAI,EAAE,CAAC;gBACnF,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,OAAO,GAAI,KAA2C,CAAC,QAAQ,IAAI,KAAK,CAAC;gBAC/E,MAAM,aAAa,GAAI,KAAkF,CAAC,OAAO,CAAC;gBAClH,MAAM,UAAU,GAAG,OAAO,aAAa,KAAK,QAAQ;oBAClD,CAAC,CAAC,aAAa;oBACf,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC;wBAC5B,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;wBACjD,CAAC,CAAC,EAAE,CAAC;gBAET,MAAM,EAAE,GAAiB;oBACvB,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,SAAS;oBAChC,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,IAAI;oBAC7B,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;oBAChC,OAAO,EAAE,CAAC,OAAO;iBAClB,CAAC;gBACF,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEnB,IAAI,OAAO,EAAE,CAAC;oBACZ,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;wBACtB,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;4BAChD,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;4BAC7F,IAAI,WAAW,EAAE,CAAC;gCAChB,WAAW,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;gCAC/B,WAAW,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC;gCAC/B,MAAM;4BACR,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,MAAM,OAAO,GAAG,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;gBACrE,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;oBACjC,GAAG,CAAC,OAAO,IAAI,IAAI,IAAI,OAAO,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;iBAC/D,CAAC,CAAC;gBACH,IAAI,KAAK;oBAAE,UAAU,GAAG,KAAK,CAAC;gBAC9B,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;AAC1D,CAAC"}
@@ -0,0 +1,3 @@
1
+ import type { ExecResult, ExecutorInput } from '../types.js';
2
+ export declare function claudeSdkExecutor({ model, system, prompt, cwd, skillDir, timeoutMs, verbose }: ExecutorInput): Promise<ExecResult>;
3
+ //# sourceMappingURL=claude-sdk.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"claude-sdk.d.ts","sourceRoot":"","sources":["../../../src/executors/claude-sdk.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAe7D,wBAAsB,iBAAiB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,SAA8B,EAAE,OAAe,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,CAqJrK"}