oh-my-knowledge 0.0.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +537 -332
- package/dist/src/analysis/coverage-analyzer.d.ts +57 -0
- package/dist/src/analysis/coverage-analyzer.d.ts.map +1 -0
- package/dist/src/analysis/coverage-analyzer.js +262 -0
- package/dist/src/analysis/coverage-analyzer.js.map +1 -0
- package/dist/src/analysis/gap-analyzer.d.ts +114 -0
- package/dist/src/analysis/gap-analyzer.d.ts.map +1 -0
- package/dist/src/analysis/gap-analyzer.js +439 -0
- package/dist/src/analysis/gap-analyzer.js.map +1 -0
- package/dist/src/analysis/hedging-classifier.d.ts +34 -0
- package/dist/src/analysis/hedging-classifier.d.ts.map +1 -0
- package/dist/src/analysis/hedging-classifier.js +144 -0
- package/dist/src/analysis/hedging-classifier.js.map +1 -0
- package/dist/src/analysis/report-diagnostics.d.ts +9 -0
- package/dist/src/analysis/report-diagnostics.d.ts.map +1 -0
- package/dist/src/analysis/report-diagnostics.js +631 -0
- package/dist/src/analysis/report-diagnostics.js.map +1 -0
- package/dist/src/authoring/evolver.d.ts +64 -0
- package/dist/src/authoring/evolver.d.ts.map +1 -0
- package/dist/src/authoring/evolver.js +275 -0
- package/dist/src/authoring/evolver.js.map +1 -0
- package/dist/src/authoring/generator.d.ts +13 -0
- package/dist/src/authoring/generator.d.ts.map +1 -0
- package/dist/src/authoring/generator.js +60 -0
- package/dist/src/authoring/generator.js.map +1 -0
- package/dist/src/cli.d.ts +3 -0
- package/dist/src/cli.d.ts.map +1 -0
- package/dist/src/cli.js +1004 -0
- package/dist/src/cli.js.map +1 -0
- package/dist/src/eval-core/cache.d.ts +11 -0
- package/dist/src/eval-core/cache.d.ts.map +1 -0
- package/dist/src/eval-core/cache.js +53 -0
- package/dist/src/eval-core/cache.js.map +1 -0
- package/dist/src/eval-core/ci-gates.d.ts +17 -0
- package/dist/src/eval-core/ci-gates.d.ts.map +1 -0
- package/dist/src/eval-core/ci-gates.js +42 -0
- package/dist/src/eval-core/ci-gates.js.map +1 -0
- package/dist/src/eval-core/dependency-checker.d.ts +37 -0
- package/dist/src/eval-core/dependency-checker.d.ts.map +1 -0
- package/dist/src/eval-core/dependency-checker.js +280 -0
- package/dist/src/eval-core/dependency-checker.js.map +1 -0
- package/dist/src/eval-core/evaluation-execution.d.ts +26 -0
- package/dist/src/eval-core/evaluation-execution.d.ts.map +1 -0
- package/dist/src/eval-core/evaluation-execution.js +196 -0
- package/dist/src/eval-core/evaluation-execution.js.map +1 -0
- package/dist/src/eval-core/evaluation-job.d.ts +48 -0
- package/dist/src/eval-core/evaluation-job.d.ts.map +1 -0
- package/dist/src/eval-core/evaluation-job.js +112 -0
- package/dist/src/eval-core/evaluation-job.js.map +1 -0
- package/dist/src/eval-core/evaluation-reporting.d.ts +28 -0
- package/dist/src/eval-core/evaluation-reporting.d.ts.map +1 -0
- package/dist/src/eval-core/evaluation-reporting.js +123 -0
- package/dist/src/eval-core/evaluation-reporting.js.map +1 -0
- package/dist/src/eval-core/execution-strategy.d.ts +11 -0
- package/dist/src/eval-core/execution-strategy.d.ts.map +1 -0
- package/dist/src/eval-core/execution-strategy.js +121 -0
- package/dist/src/eval-core/execution-strategy.js.map +1 -0
- package/dist/src/eval-core/fact-checker.d.ts +25 -0
- package/dist/src/eval-core/fact-checker.d.ts.map +1 -0
- package/dist/src/eval-core/fact-checker.js +66 -0
- package/dist/src/eval-core/fact-checker.js.map +1 -0
- package/dist/src/eval-core/schema.d.ts +30 -0
- package/dist/src/eval-core/schema.d.ts.map +1 -0
- package/dist/src/eval-core/schema.js +195 -0
- package/dist/src/eval-core/schema.js.map +1 -0
- package/dist/src/eval-core/statistics.d.ts +86 -0
- package/dist/src/eval-core/statistics.d.ts.map +1 -0
- package/dist/src/eval-core/statistics.js +210 -0
- package/dist/src/eval-core/statistics.js.map +1 -0
- package/dist/src/eval-core/task-planner.d.ts +4 -0
- package/dist/src/eval-core/task-planner.d.ts.map +1 -0
- package/dist/src/eval-core/task-planner.js +33 -0
- package/dist/src/eval-core/task-planner.js.map +1 -0
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts +133 -0
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts.map +1 -0
- package/dist/src/eval-workflows/each-evaluation-workflow.js +169 -0
- package/dist/src/eval-workflows/each-evaluation-workflow.js.map +1 -0
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts +38 -0
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -0
- package/dist/src/eval-workflows/evaluation-pipeline.js +214 -0
- package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -0
- package/dist/src/eval-workflows/evaluation-preparation.d.ts +64 -0
- package/dist/src/eval-workflows/evaluation-preparation.d.ts.map +1 -0
- package/dist/src/eval-workflows/evaluation-preparation.js +90 -0
- package/dist/src/eval-workflows/evaluation-preparation.js.map +1 -0
- package/dist/src/eval-workflows/run-evaluation.d.ts +105 -0
- package/dist/src/eval-workflows/run-evaluation.d.ts.map +1 -0
- package/dist/src/eval-workflows/run-evaluation.js +264 -0
- package/dist/src/eval-workflows/run-evaluation.js.map +1 -0
- package/dist/src/executors/anthropic-api.d.ts +3 -0
- package/dist/src/executors/anthropic-api.d.ts.map +1 -0
- package/dist/src/executors/anthropic-api.js +43 -0
- package/dist/src/executors/anthropic-api.js.map +1 -0
- package/dist/src/executors/claude-cli.d.ts +3 -0
- package/dist/src/executors/claude-cli.d.ts.map +1 -0
- package/dist/src/executors/claude-cli.js +103 -0
- package/dist/src/executors/claude-cli.js.map +1 -0
- package/dist/src/executors/claude-sdk-trace.d.ts +10 -0
- package/dist/src/executors/claude-sdk-trace.d.ts.map +1 -0
- package/dist/src/executors/claude-sdk-trace.js +100 -0
- package/dist/src/executors/claude-sdk-trace.js.map +1 -0
- package/dist/src/executors/claude-sdk.d.ts +3 -0
- package/dist/src/executors/claude-sdk.d.ts.map +1 -0
- package/dist/src/executors/claude-sdk.js +160 -0
- package/dist/src/executors/claude-sdk.js.map +1 -0
- package/dist/src/executors/gemini.d.ts +3 -0
- package/dist/src/executors/gemini.d.ts.map +1 -0
- package/dist/src/executors/gemini.js +78 -0
- package/dist/src/executors/gemini.js.map +1 -0
- package/dist/src/executors/index.d.ts +7 -0
- package/dist/src/executors/index.d.ts.map +1 -0
- package/dist/src/executors/index.js +22 -0
- package/dist/src/executors/index.js.map +1 -0
- package/dist/src/executors/openai-api.d.ts +3 -0
- package/dist/src/executors/openai-api.d.ts.map +1 -0
- package/dist/src/executors/openai-api.js +40 -0
- package/dist/src/executors/openai-api.js.map +1 -0
- package/dist/src/executors/openai-cli.d.ts +3 -0
- package/dist/src/executors/openai-cli.d.ts.map +1 -0
- package/dist/src/executors/openai-cli.js +60 -0
- package/dist/src/executors/openai-cli.js.map +1 -0
- package/dist/src/executors/script.d.ts +3 -0
- package/dist/src/executors/script.d.ts.map +1 -0
- package/dist/src/executors/script.js +63 -0
- package/dist/src/executors/script.js.map +1 -0
- package/dist/src/executors/shared.d.ts +117 -0
- package/dist/src/executors/shared.d.ts.map +1 -0
- package/dist/src/executors/shared.js +49 -0
- package/dist/src/executors/shared.js.map +1 -0
- package/dist/src/grading/assertions.d.ts +18 -0
- package/dist/src/grading/assertions.d.ts.map +1 -0
- package/dist/src/grading/assertions.js +239 -0
- package/dist/src/grading/assertions.js.map +1 -0
- package/dist/src/grading/index.d.ts +26 -0
- package/dist/src/grading/index.d.ts.map +1 -0
- package/dist/src/grading/index.js +98 -0
- package/dist/src/grading/index.js.map +1 -0
- package/dist/src/grading/judge.d.ts +13 -0
- package/dist/src/grading/judge.d.ts.map +1 -0
- package/dist/src/grading/judge.js +98 -0
- package/dist/src/grading/judge.js.map +1 -0
- package/dist/src/grading/layered-scores.d.ts +13 -0
- package/dist/src/grading/layered-scores.d.ts.map +1 -0
- package/dist/src/grading/layered-scores.js +62 -0
- package/dist/src/grading/layered-scores.js.map +1 -0
- package/dist/src/inputs/eval-config.d.ts +13 -0
- package/dist/src/inputs/eval-config.d.ts.map +1 -0
- package/dist/src/inputs/eval-config.js +136 -0
- package/dist/src/inputs/eval-config.js.map +1 -0
- package/dist/src/inputs/load-samples.d.ts +16 -0
- package/dist/src/inputs/load-samples.d.ts.map +1 -0
- package/dist/src/inputs/load-samples.js +51 -0
- package/dist/src/inputs/load-samples.js.map +1 -0
- package/dist/src/inputs/mcp-resolver.d.ts +50 -0
- package/dist/src/inputs/mcp-resolver.d.ts.map +1 -0
- package/dist/src/inputs/mcp-resolver.js +307 -0
- package/dist/src/inputs/mcp-resolver.js.map +1 -0
- package/dist/src/inputs/skill-loader.d.ts +18 -0
- package/dist/src/inputs/skill-loader.d.ts.map +1 -0
- package/dist/src/inputs/skill-loader.js +222 -0
- package/dist/src/inputs/skill-loader.js.map +1 -0
- package/dist/src/inputs/url-fetcher.d.ts +19 -0
- package/dist/src/inputs/url-fetcher.d.ts.map +1 -0
- package/dist/src/inputs/url-fetcher.js +241 -0
- package/dist/src/inputs/url-fetcher.js.map +1 -0
- package/dist/src/observability/production-analyzer.d.ts +61 -0
- package/dist/src/observability/production-analyzer.d.ts.map +1 -0
- package/dist/src/observability/production-analyzer.js +183 -0
- package/dist/src/observability/production-analyzer.js.map +1 -0
- package/dist/src/observability/trace-adapter.d.ts +75 -0
- package/dist/src/observability/trace-adapter.d.ts.map +1 -0
- package/dist/src/observability/trace-adapter.js +341 -0
- package/dist/src/observability/trace-adapter.js.map +1 -0
- package/dist/src/renderer/html-renderer.d.ts +9 -0
- package/dist/src/renderer/html-renderer.d.ts.map +1 -0
- package/dist/src/renderer/html-renderer.js +338 -0
- package/dist/src/renderer/html-renderer.js.map +1 -0
- package/dist/src/renderer/layout.d.ts +13 -0
- package/dist/src/renderer/layout.d.ts.map +1 -0
- package/dist/src/renderer/layout.js +476 -0
- package/dist/src/renderer/layout.js.map +1 -0
- package/dist/src/renderer/skill-health-renderer.d.ts +20 -0
- package/dist/src/renderer/skill-health-renderer.d.ts.map +1 -0
- package/dist/src/renderer/skill-health-renderer.js +218 -0
- package/dist/src/renderer/skill-health-renderer.js.map +1 -0
- package/dist/src/renderer/summary.d.ts +21 -0
- package/dist/src/renderer/summary.d.ts.map +1 -0
- package/dist/src/renderer/summary.js +1072 -0
- package/dist/src/renderer/summary.js.map +1 -0
- package/dist/src/renderer/table.d.ts +3 -0
- package/dist/src/renderer/table.d.ts.map +1 -0
- package/dist/src/renderer/table.js +134 -0
- package/dist/src/renderer/table.js.map +1 -0
- package/dist/src/renderer/trends.d.ts +6 -0
- package/dist/src/renderer/trends.d.ts.map +1 -0
- package/dist/src/renderer/trends.js +127 -0
- package/dist/src/renderer/trends.js.map +1 -0
- package/dist/src/server/job-store.d.ts +4 -0
- package/dist/src/server/job-store.d.ts.map +1 -0
- package/dist/src/server/job-store.js +68 -0
- package/dist/src/server/job-store.js.map +1 -0
- package/dist/src/server/report-server.d.ts +16 -0
- package/dist/src/server/report-server.d.ts.map +1 -0
- package/dist/src/server/report-server.js +196 -0
- package/dist/src/server/report-server.js.map +1 -0
- package/dist/src/server/report-store.d.ts +44 -0
- package/dist/src/server/report-store.d.ts.map +1 -0
- package/dist/src/server/report-store.js +190 -0
- package/dist/src/server/report-store.js.map +1 -0
- package/dist/src/types.d.ts +553 -0
- package/dist/src/types.d.ts.map +1 -0
- package/dist/src/types.js +2 -0
- package/dist/src/types.js.map +1 -0
- package/package.json +44 -3
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
2
|
+
import { DEFAULT_OUTPUT_DIR, persistReport } from '../eval-core/evaluation-reporting.js';
|
|
3
|
+
import { createExecutor, DEFAULT_MODEL, JUDGE_MODEL } from '../executors/index.js';
|
|
4
|
+
import { discoverEachSkills } from '../inputs/skill-loader.js';
|
|
5
|
+
import { confidenceInterval, tTest, effectSize } from '../eval-core/statistics.js';
|
|
6
|
+
import { executeEachEvaluationRuns } from './each-evaluation-workflow.js';
|
|
7
|
+
import { buildDryRunEachArtifacts, buildDryRunTaskReport, prepareEvaluationRun, } from './evaluation-preparation.js';
|
|
8
|
+
import { executeEvaluationPipeline } from './evaluation-pipeline.js';
|
|
9
|
+
export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], artifacts, model = DEFAULT_MODEL, judgeModel = JUDGE_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, blind = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', judgeExecutorName, jobStore = null, persistJob = true, onProgress = null, skipPreflight = false, mcpConfig, verbose = false, retry = 0, resume, layeredStats = false, }) {
|
|
10
|
+
const { samples, artifacts: resolvedArtifacts, tasks, variantNames, requires } = await prepareEvaluationRun({
|
|
11
|
+
samplesPath,
|
|
12
|
+
skillDir,
|
|
13
|
+
variantSpecs,
|
|
14
|
+
artifacts,
|
|
15
|
+
dryRun,
|
|
16
|
+
mcpConfig,
|
|
17
|
+
});
|
|
18
|
+
if (dryRun) {
|
|
19
|
+
return {
|
|
20
|
+
report: buildDryRunTaskReport({
|
|
21
|
+
model,
|
|
22
|
+
judgeModel,
|
|
23
|
+
executorName,
|
|
24
|
+
samplesPath,
|
|
25
|
+
skillDir,
|
|
26
|
+
tasks,
|
|
27
|
+
variantNames,
|
|
28
|
+
}),
|
|
29
|
+
filePath: null,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
// --resume: load existing report results to skip completed tasks
|
|
33
|
+
let existingResults;
|
|
34
|
+
if (resume) {
|
|
35
|
+
const { createFileStore } = await import('../server/report-store.js');
|
|
36
|
+
const store = createFileStore(resolve(outputDir || DEFAULT_OUTPUT_DIR));
|
|
37
|
+
const existing = await store.get(resume);
|
|
38
|
+
if (existing) {
|
|
39
|
+
existingResults = {};
|
|
40
|
+
for (const entry of existing.results || []) {
|
|
41
|
+
existingResults[entry.sample_id] = entry.variants;
|
|
42
|
+
}
|
|
43
|
+
if (onProgress) {
|
|
44
|
+
const count = Object.values(existingResults).reduce((sum, v) => sum + Object.values(v).filter((r) => r.ok).length, 0);
|
|
45
|
+
process.stderr.write(`\n📂 从报告 ${resume} 恢复了 ${count} 个已完成结果\n`);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
process.stderr.write(`\n⚠️ 报告 ${resume} 未找到,将从头执行\n`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const executor = createExecutor(executorName);
|
|
53
|
+
const judgeExecutor = createExecutor(judgeExecutorName || executorName);
|
|
54
|
+
return executeEvaluationPipeline({
|
|
55
|
+
samplesPath,
|
|
56
|
+
skillDir,
|
|
57
|
+
samples,
|
|
58
|
+
tasks,
|
|
59
|
+
artifacts: resolvedArtifacts,
|
|
60
|
+
model,
|
|
61
|
+
judgeModel,
|
|
62
|
+
noJudge,
|
|
63
|
+
executorName,
|
|
64
|
+
judgeExecutorName: judgeExecutorName || executorName,
|
|
65
|
+
executor,
|
|
66
|
+
judgeExecutor,
|
|
67
|
+
outputDir,
|
|
68
|
+
project,
|
|
69
|
+
owner,
|
|
70
|
+
tags,
|
|
71
|
+
blind,
|
|
72
|
+
concurrency,
|
|
73
|
+
timeoutMs,
|
|
74
|
+
noCache,
|
|
75
|
+
jobStore,
|
|
76
|
+
persistJob,
|
|
77
|
+
onProgress,
|
|
78
|
+
skipPreflight,
|
|
79
|
+
verbose,
|
|
80
|
+
retry,
|
|
81
|
+
existingResults,
|
|
82
|
+
requires,
|
|
83
|
+
layeredStats,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
// Top-level (composite) extractor: feeds the legacy flat-field variance on
|
|
87
|
+
// VariantVariance / VarianceComparison. Composite lives here for backward
|
|
88
|
+
// compatibility with historical reports; layer-independent stats are attached
|
|
89
|
+
// via byLayer (see LAYER_EXTRACTORS) starting v0.16 work item B / PR-2.
|
|
90
|
+
const COMPOSITE_EXTRACTOR = (s) => s?.avgCompositeScore;
|
|
91
|
+
// Non-quality metric extractors tracked in byMetric (cost + efficiency).
|
|
92
|
+
const METRIC_EXTRACTORS = {
|
|
93
|
+
cost: (s) => s?.totalExecCostUSD,
|
|
94
|
+
efficiency: (s) => s?.avgDurationMs,
|
|
95
|
+
};
|
|
96
|
+
// Three-layer extractors (PR-2): fact / behavior / judge each get their own
|
|
97
|
+
// independent variance + t-test + effect size. Lets "judge ↑ 0.8, fact ↑ 0.1"
|
|
98
|
+
// structural signals surface instead of getting diluted by the composite average.
|
|
99
|
+
// `judge` is the rubric-based LLM judge score (UI 中文: "LLM 评价").
|
|
100
|
+
const LAYER_EXTRACTORS = {
|
|
101
|
+
fact: (s) => s?.avgFactScore,
|
|
102
|
+
behavior: (s) => s?.avgBehaviorScore,
|
|
103
|
+
judge: (s) => s?.avgJudgeScore,
|
|
104
|
+
};
|
|
105
|
+
function buildMetricStats(runs, variant, extractor) {
|
|
106
|
+
const scores = runs
|
|
107
|
+
.map((run) => extractor(run.summary?.[variant]))
|
|
108
|
+
.filter((x) => typeof x === 'number');
|
|
109
|
+
if (scores.length === 0)
|
|
110
|
+
return null;
|
|
111
|
+
return { scores, ...confidenceInterval(scores) };
|
|
112
|
+
}
|
|
113
|
+
function buildComparisonMetric(scoresA, scoresB, meanA, meanB) {
|
|
114
|
+
const t = tTest(scoresA, scoresB);
|
|
115
|
+
const es = effectSize(scoresA, scoresB);
|
|
116
|
+
const meanDiff = Number((meanA - meanB).toFixed(4));
|
|
117
|
+
return {
|
|
118
|
+
meanDiff,
|
|
119
|
+
tStatistic: t.tStatistic,
|
|
120
|
+
df: t.df,
|
|
121
|
+
significant: t.significant,
|
|
122
|
+
effectSize: es,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
export function buildVarianceData(runs) {
|
|
126
|
+
if (runs.length <= 1) {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
const variants = runs[0].meta.variants || [];
|
|
130
|
+
const perVariant = {};
|
|
131
|
+
for (const variant of variants) {
|
|
132
|
+
// Composite lives on the legacy flat fields.
|
|
133
|
+
const composite = buildMetricStats(runs, variant, COMPOSITE_EXTRACTOR);
|
|
134
|
+
if (!composite)
|
|
135
|
+
continue;
|
|
136
|
+
const byMetric = {};
|
|
137
|
+
const cost = buildMetricStats(runs, variant, METRIC_EXTRACTORS.cost);
|
|
138
|
+
if (cost)
|
|
139
|
+
byMetric.cost = cost;
|
|
140
|
+
const efficiency = buildMetricStats(runs, variant, METRIC_EXTRACTORS.efficiency);
|
|
141
|
+
if (efficiency)
|
|
142
|
+
byMetric.efficiency = efficiency;
|
|
143
|
+
// Three-layer independent stats (PR-2). Any layer with no data across runs is omitted.
|
|
144
|
+
const byLayer = {};
|
|
145
|
+
for (const key of ['fact', 'behavior', 'judge']) {
|
|
146
|
+
const layerStats = buildMetricStats(runs, variant, LAYER_EXTRACTORS[key]);
|
|
147
|
+
if (layerStats)
|
|
148
|
+
byLayer[key] = layerStats;
|
|
149
|
+
}
|
|
150
|
+
perVariant[variant] = {
|
|
151
|
+
...composite,
|
|
152
|
+
...(Object.keys(byMetric).length > 0 ? { byMetric } : {}),
|
|
153
|
+
...(Object.keys(byLayer).length > 0 ? { byLayer } : {}),
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
const comparisons = [];
|
|
157
|
+
for (let i = 0; i < variants.length; i++) {
|
|
158
|
+
for (let j = i + 1; j < variants.length; j++) {
|
|
159
|
+
const vA = perVariant[variants[i]];
|
|
160
|
+
const vB = perVariant[variants[j]];
|
|
161
|
+
if (!vA || !vB)
|
|
162
|
+
continue;
|
|
163
|
+
const compositeComp = buildComparisonMetric(vA.scores, vB.scores, vA.mean, vB.mean);
|
|
164
|
+
const byMetricComp = {};
|
|
165
|
+
for (const key of ['cost', 'efficiency']) {
|
|
166
|
+
const mA = vA.byMetric?.[key];
|
|
167
|
+
const mB = vB.byMetric?.[key];
|
|
168
|
+
if (mA && mB) {
|
|
169
|
+
byMetricComp[key] = buildComparisonMetric(mA.scores, mB.scores, mA.mean, mB.mean);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
// Three-layer comparison (PR-2). Each layer pair runs its own Welch's t + Cohen's d.
|
|
173
|
+
const byLayerComp = {};
|
|
174
|
+
for (const key of ['fact', 'behavior', 'judge']) {
|
|
175
|
+
const lA = vA.byLayer?.[key];
|
|
176
|
+
const lB = vB.byLayer?.[key];
|
|
177
|
+
if (lA && lB) {
|
|
178
|
+
byLayerComp[key] = buildComparisonMetric(lA.scores, lB.scores, lA.mean, lB.mean);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
comparisons.push({
|
|
182
|
+
a: variants[i],
|
|
183
|
+
b: variants[j],
|
|
184
|
+
...compositeComp,
|
|
185
|
+
...(Object.keys(byMetricComp).length > 0 ? { byMetric: byMetricComp } : {}),
|
|
186
|
+
...(Object.keys(byLayerComp).length > 0 ? { byLayer: byLayerComp } : {}),
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
return { runs: runs.length, perVariant, comparisons };
|
|
191
|
+
}
|
|
192
|
+
export async function runEachEvaluation({ skillDir, model = DEFAULT_MODEL, judgeModel = JUDGE_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, executorName = 'claude', judgeExecutorName, jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipPreflight = false, mcpConfig, verbose = false, }) {
|
|
193
|
+
const skillEntries = discoverEachSkills(resolve(skillDir));
|
|
194
|
+
if (skillEntries.length === 0) {
|
|
195
|
+
throw new Error(`未发现带配对 eval-samples 的 skill:${skillDir}`);
|
|
196
|
+
}
|
|
197
|
+
if (dryRun) {
|
|
198
|
+
const { artifacts: dryArtifacts, totalTasks } = buildDryRunEachArtifacts(skillEntries);
|
|
199
|
+
return {
|
|
200
|
+
report: {
|
|
201
|
+
dryRun: true,
|
|
202
|
+
each: true,
|
|
203
|
+
model,
|
|
204
|
+
judgeModel,
|
|
205
|
+
executor: executorName,
|
|
206
|
+
skillDir,
|
|
207
|
+
totalArtifacts: dryArtifacts.length,
|
|
208
|
+
totalTasks,
|
|
209
|
+
artifacts: dryArtifacts,
|
|
210
|
+
},
|
|
211
|
+
filePath: null,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
return executeEachEvaluationRuns({
|
|
215
|
+
skillDir,
|
|
216
|
+
skillEntries,
|
|
217
|
+
model,
|
|
218
|
+
judgeModel,
|
|
219
|
+
outputDir,
|
|
220
|
+
project,
|
|
221
|
+
owner,
|
|
222
|
+
tags,
|
|
223
|
+
noJudge,
|
|
224
|
+
concurrency,
|
|
225
|
+
timeoutMs,
|
|
226
|
+
executorName,
|
|
227
|
+
judgeExecutorName,
|
|
228
|
+
jobStore,
|
|
229
|
+
persistJob,
|
|
230
|
+
onProgress,
|
|
231
|
+
onSkillProgress,
|
|
232
|
+
skipPreflight,
|
|
233
|
+
mcpConfig,
|
|
234
|
+
verbose,
|
|
235
|
+
runSingleEvaluation: async (options) => {
|
|
236
|
+
const result = await runEvaluation(options);
|
|
237
|
+
return { report: result.report, filePath: result.filePath };
|
|
238
|
+
},
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
export async function runMultiple({ repeat = 1, onRepeatProgress, ...config }) {
|
|
242
|
+
const runs = [];
|
|
243
|
+
const savedOutputDir = config.outputDir;
|
|
244
|
+
for (let i = 0; i < repeat; i++) {
|
|
245
|
+
onRepeatProgress?.({ run: i + 1, total: repeat });
|
|
246
|
+
const isLast = i === repeat - 1;
|
|
247
|
+
const { report } = await runEvaluation({
|
|
248
|
+
...config,
|
|
249
|
+
outputDir: isLast ? savedOutputDir : null,
|
|
250
|
+
persistJob: isLast,
|
|
251
|
+
});
|
|
252
|
+
runs.push(report);
|
|
253
|
+
}
|
|
254
|
+
const report = runs[runs.length - 1];
|
|
255
|
+
const aggregated = buildVarianceData(runs);
|
|
256
|
+
let filePath = null;
|
|
257
|
+
if (aggregated) {
|
|
258
|
+
report.variance = aggregated;
|
|
259
|
+
// pipeline 内部在 variance 赋值前已写入报告,需要重新写一次以包含方差数据
|
|
260
|
+
filePath = persistReport(report, savedOutputDir || DEFAULT_OUTPUT_DIR);
|
|
261
|
+
}
|
|
262
|
+
return { report, aggregated, filePath };
|
|
263
|
+
}
|
|
264
|
+
//# sourceMappingURL=run-evaluation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-evaluation.js","sourceRoot":"","sources":["../../../src/eval-workflows/run-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACzF,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACnF,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AAuGrE,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,EAClC,WAAW,EACX,QAAQ,EACR,YAAY,GAAG,EAAE,EACjB,SAAS,EACT,KAAK,GAAG,aAAa,EACrB,UAAU,GAAG,WAAW,EACxB,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,KAAK,EACf,MAAM,GAAG,KAAK,EACd,KAAK,GAAG,KAAK,EACb,WAAW,GAAG,CAAC,EACf,SAAS,EACT,OAAO,GAAG,KAAK,EACf,YAAY,GAAG,QAAQ,EACvB,iBAAiB,EACjB,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,aAAa,GAAG,KAAK,EACrB,SAAS,EACT,OAAO,GAAG,KAAK,EACf,KAAK,GAAG,CAAC,EACT,MAAM,EACN,YAAY,GAAG,KAAK,GACC;IACrB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,MAAM,oBAAoB,CAAC;QAC1G,WAAW;QACX,QAAQ;QACR,YAAY;QACZ,SAAS;QACT,MAAM;QACN,SAAS;KACV,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,MAAM,EAAE,qBAAqB,CAAC;gBAC5B,KAAK;gBACL,UAAU;gBACV,YAAY;gBACZ,WAAW;gBACX,QAAQ;gBACR,KAAK;gBACL,YAAY;aACb,CAAC;YACF,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,IAAI,eAAgG,CAAC;IACrG,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,GAAG,EAAE,CAAC;YACrB,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;gBAC3C,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC;YACpD,CAAC;YACD,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACtH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,QAAQ,KAAK,WAAW,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,cAAc,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAe,cAAc,CAAC,YAAY,CAAC,CAAC;IAC1D,MAAM,aAAa,GAAe,cAAc,CAAC,iBAAiB,IAAI,YAAY,CAAC,CAAC;IACpF,OAAO,yBAAyB,CAAC;QAC/B,WAAW;QACX,QAAQ;QACR,OAAO;QACP,KAAK;QACL,SAAS,EAAE,iBAAiB;QAC5B,KAAK;QACL,UAAU;QACV,OAAO;QACP,YAAY;QACZ,iBAAiB,EAAE,iBAAiB,IAAI,YAAY;QACpD,QAAQ;QACR,aAAa;QACb,SAAS;QACT,OAAO;QACP,KAAK;QACL,IAAI;QACJ,KAAK;QACL,WAAW;QACX,SAAS;QACT,OAAO;QACP,QAAQ;QACR,UAAU;QACV,UAAU;QACV,aAAa;QACb,OAAO;QACP,KAAK;QACL,eAAe;QACf,QAAQ;QACR,YAAY;KACb,CAAC,CAAC;AACL,CAAC;AAeD,2EAA2E;AAC3E,0EAA0E;AAC1E,8EAA8E;AAC9E,wEAAwE;AACxE,MAAM,mBAAmB,GAAG,CAAC,CAA6B,EAAsB,EAAE,CAAC,CAAC,EAAE,iBAAiB,CAAC;AAExG,yEAAyE;AACzE,MAAM,iBAAiB,GAAyF;IAC9G,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,gBAAgB;IAChC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,aAAa;CACpC,CAAC;AAEF,4EAA4E;AAC5E,8EAA8E;AAC9E,kFAAkF;AAClF,iEAAiE;AACjE,MAAM,gBAAgB,GAAoF;IACxG,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,YAAY;IAC5B,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,gBAAgB;IACpC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,aAAa;CAC/B,CAAC;AAEF,SAAS,gBAAgB,CAAC,IAAc,EAAE,OAAe,EAAE,SAAgE;IACzH,MAAM,MAAM,GAAG,IAAI;SAChB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;SAC/C,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAiB,EAAE,OAAiB,EAAE,KAAa,EAAE,KAAa;IAC/F,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,OAAO;QACL,QAAQ;QACR,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,UAAU,EAAE,EAAE;KACf,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAc;IAC9C,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;IAC7C,MAAM,UAAU,GAAoC,EAAE,CAAC;IACvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,6CAA6C;QAC7C,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC;QACvE,IAAI,CAAC,SAAS;YAAE,SAAS;QAEzB,MAAM,QAAQ,GAA2D,EAAE,CAAC;QAC5E,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACrE,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC;QAC/B,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,CAAC;QACjF,IAAI,UAAU;YAAE,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;QAEjD,uFAAuF;QACvF,MAAM,OAAO,GAAsD,EAAE,CAAC;QACtE,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE,CAAC;YACzD,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1E,IAAI,UAAU;gBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;QAC5C,CAAC;QAED,UAAU,CAAC,OAAO,CAAC,GAAG;YACpB,GAAG,SAAS;YACZ,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAyB,EAAE,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE;gBAAE,SAAS;YAEzB,MAAM,aAAa,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YAEpF,MAAM,YAAY,GAAqE,EAAE,CAAC;YAC1F,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,YAAY,CAAU,EAAE,CAAC;gBAClD,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;oBACb,YAAY,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBACpF,CAAC;YACH,CAAC;YAED,qFAAqF;YACrF,MAAM,WAAW,GAAgE,EAAE,CAAC;YACpF,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE,CAAC;gBACzD,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC7B,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;oBACb,WAAW,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBACnF,CAAC;YACH,CAAC;YAED,WAAW,CAAC,IAAI,CAAC;gBACf,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACd,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACd,GAAG,aAAa;gBAChB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3E,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACzE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EACtC,QAAQ,EACR,KAAK,GAAG,aAAa,EACrB,UAAU,GAAG,WAAW,EACxB,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,KAAK,EACf,MAAM,GAAG,KAAK,EACd,WAAW,GAAG,CAAC,EACf,SAAS,EACT,YAAY,GAAG,QAAQ,EACvB,iBAAiB,EACjB,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,eAAe,GAAG,IAAI,EACtB,aAAa,GAAG,KAAK,EACrB,SAAS,EACT,OAAO,GAAG,KAAK,GACU;IACzB,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3D,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,wBAAwB,CAAC,YAAY,CAAC,CAAC;QACvF,OAAO;YACL,MAAM,EAAE;gBACN,MAAM,EAAE,IAAI;gBACZ,IAAI,EAAE,IAAI;gBACV,KAAK;gBACL,UAAU;gBACV,QAAQ,EAAE,YAAY;gBACtB,QAAQ;gBACR,cAAc,EAAE,YAAY,CAAC,MAAM;gBACnC,UAAU;gBACV,SAAS,EAAE,YAAY;aACxB;YACD,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IACD,OAAO,yBAAyB,CAAC;QAC/B,QAAQ;QACR,YAAY;QACZ,KAAK;QACL,UAAU;QACV,SAAS;QACT,OAAO;QACP,KAAK;QACL,IAAI;QACJ,OAAO;QACP,WAAW;QACX,SAAS;QACT,YAAY;QACZ,iBAAiB;QACjB,QAAQ;QACR,UAAU;QACV,UAAU;QACV,eAAe;QACf,aAAa;QACb,SAAS;QACT,OAAO;QACP,mBAAmB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACrC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;YAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAgB,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxE,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,gBAAgB,EAAE,GAAG,MAAM,EAAsB;IAC/F,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC;IAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,gBAAgB,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC;QAChC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC;YACrC,GAAG,MAAM;YACT,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI;YACzC,UAAU,EAAE,MAAM;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,MAAgB,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;QAC7B,gDAAgD;QAChD,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,cAAc,IAAI,kBAAkB,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"anthropic-api.d.ts","sourceRoot":"","sources":["../../../src/executors/anthropic-api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG7D,wBAAsB,oBAAoB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAA8B,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,CA8CxI"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { asErrorLike, DEFAULT_TIMEOUT_MS, errorMessage } from './shared.js';
|
|
2
|
+
export async function anthropicApiExecutor({ model, system, prompt, timeoutMs = DEFAULT_TIMEOUT_MS }) {
|
|
3
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
4
|
+
if (!apiKey)
|
|
5
|
+
throw new Error('ANTHROPIC_API_KEY 环境变量未设置');
|
|
6
|
+
const baseUrl = process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com';
|
|
7
|
+
const reqBody = {
|
|
8
|
+
model: model || 'claude-sonnet-4-5-20250514',
|
|
9
|
+
max_tokens: 8192,
|
|
10
|
+
messages: [{ role: 'user', content: prompt }],
|
|
11
|
+
};
|
|
12
|
+
if (system)
|
|
13
|
+
reqBody.system = system;
|
|
14
|
+
const start = Date.now();
|
|
15
|
+
try {
|
|
16
|
+
const res = await fetch(`${baseUrl}/v1/messages`, {
|
|
17
|
+
method: 'POST',
|
|
18
|
+
headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey, 'anthropic-version': '2023-06-01' },
|
|
19
|
+
body: JSON.stringify(reqBody),
|
|
20
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
21
|
+
});
|
|
22
|
+
const data = await res.json();
|
|
23
|
+
const durationMs = Date.now() - start;
|
|
24
|
+
if (!res.ok) {
|
|
25
|
+
return { ok: false, error: data.error?.message || `API error ${res.status}`, durationMs, durationApiMs: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0, costUSD: 0, output: null, stopReason: 'error', numTurns: 0 };
|
|
26
|
+
}
|
|
27
|
+
const usage = data.usage || {};
|
|
28
|
+
return {
|
|
29
|
+
ok: true, output: data.content?.map((c) => c.text || '').join('') || '', durationMs, durationApiMs: 0,
|
|
30
|
+
inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0,
|
|
31
|
+
cacheReadTokens: usage.cache_read_input_tokens || 0, cacheCreationTokens: usage.cache_creation_input_tokens || 0,
|
|
32
|
+
costUSD: 0, stopReason: data.stop_reason || 'end_turn', numTurns: 1,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
catch (err) {
|
|
36
|
+
const durationMs = Date.now() - start;
|
|
37
|
+
const details = asErrorLike(err);
|
|
38
|
+
const stopReason = details.name === 'TimeoutError' ? 'timeout' : 'error';
|
|
39
|
+
const error = details.name === 'TimeoutError' ? `API request timed out after ${timeoutMs / 1000}s` : errorMessage(err);
|
|
40
|
+
return { ok: false, error, durationMs, durationApiMs: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0, costUSD: 0, output: null, stopReason, numTurns: 0 };
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=anthropic-api.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"anthropic-api.js","sourceRoot":"","sources":["../../../src/executors/anthropic-api.ts"],"names":[],"mappings":"AACA,OAAO,EAAqB,WAAW,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE/F,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,kBAAkB,EAAiB;IACjH,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAE1D,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,2BAA2B,CAAC;IAC9E,MAAM,OAAO,GAKT;QACF,KAAK,EAAE,KAAK,IAAI,4BAA4B;QAC5C,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;KAC9C,CAAC;IACF,IAAI,MAAM;QAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;IAEpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,cAAc,EAAE;YAChD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,EAAE,mBAAmB,EAAE,YAAY,EAAE;YACvG,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;YAC7B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;SACvC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAuB,CAAC;QACnD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEtC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,aAAa,GAAG,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QACvP,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/B,OAAO;YACL,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;YACrG,WAAW,EAAE,KAAK,CAAC,YAAY,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC;YAC5E,eAAe,EAAE,KAAK,CAAC,uBAAuB,IAAI,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,2BAA2B,IAAI,CAAC;YAChH,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,WAAW,IAAI,UAAU,EAAE,QAAQ,EAAE,CAAC;SACpE,CAAC;IACJ,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;QACzE,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,+BAA+B,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACvH,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IAC5L,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"claude-cli.d.ts","sourceRoot":"","sources":["../../../src/executors/claude-cli.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAyB7D,wBAAsB,iBAAiB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,SAA8B,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,CA2FpJ"}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { extractAgentTrace, isClaudeSdkResultMessage } from './claude-sdk-trace.js';
|
|
2
|
+
import { asErrorLike, buildExecEnv, DEFAULT_TIMEOUT_MS, errorMessage, execFileAsync, MAX_BUFFER, timeoutExecResult, } from './shared.js';
|
|
3
|
+
function parseStreamJson(stdout) {
|
|
4
|
+
const messages = [];
|
|
5
|
+
for (const line of stdout.split('\n')) {
|
|
6
|
+
const trimmed = line.trim();
|
|
7
|
+
if (!trimmed)
|
|
8
|
+
continue;
|
|
9
|
+
try {
|
|
10
|
+
messages.push(JSON.parse(trimmed));
|
|
11
|
+
}
|
|
12
|
+
catch { /* skip non-JSON lines */ }
|
|
13
|
+
}
|
|
14
|
+
return messages;
|
|
15
|
+
}
|
|
16
|
+
export async function claudeCliExecutor({ model, system, prompt, cwd, skillDir, timeoutMs = DEFAULT_TIMEOUT_MS }) {
|
|
17
|
+
const args = ['-p', prompt, '--output-format', 'stream-json', '--verbose', '--model', model];
|
|
18
|
+
if (system)
|
|
19
|
+
args.push('--system-prompt', system);
|
|
20
|
+
const env = buildExecEnv(skillDir);
|
|
21
|
+
const start = Date.now();
|
|
22
|
+
try {
|
|
23
|
+
const { stdout } = await execFileAsync('claude', args, {
|
|
24
|
+
maxBuffer: MAX_BUFFER,
|
|
25
|
+
timeout: timeoutMs,
|
|
26
|
+
env,
|
|
27
|
+
...(cwd && { cwd }),
|
|
28
|
+
});
|
|
29
|
+
const durationMs = Date.now() - start;
|
|
30
|
+
const messages = parseStreamJson(stdout);
|
|
31
|
+
// 提取 result 消息
|
|
32
|
+
const resultMsgs = messages.filter(isClaudeSdkResultMessage);
|
|
33
|
+
if (resultMsgs.length === 0) {
|
|
34
|
+
return {
|
|
35
|
+
ok: false, error: 'no result message in stream-json output',
|
|
36
|
+
durationMs, durationApiMs: 0,
|
|
37
|
+
inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0,
|
|
38
|
+
costUSD: 0, output: null, stopReason: 'error', numTurns: 0,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
const last = resultMsgs[resultMsgs.length - 1];
|
|
42
|
+
const usage = last.usage || {};
|
|
43
|
+
// 提取 trace
|
|
44
|
+
const trace = extractAgentTrace(messages);
|
|
45
|
+
return {
|
|
46
|
+
ok: !last.errors?.length && last.subtype !== 'error',
|
|
47
|
+
durationMs: last.duration_ms || durationMs,
|
|
48
|
+
durationApiMs: last.duration_api_ms || 0,
|
|
49
|
+
inputTokens: usage.input_tokens || 0,
|
|
50
|
+
outputTokens: usage.output_tokens || 0,
|
|
51
|
+
cacheReadTokens: usage.cache_read_input_tokens || 0,
|
|
52
|
+
cacheCreationTokens: usage.cache_creation_input_tokens || 0,
|
|
53
|
+
costUSD: last.total_cost_usd || 0,
|
|
54
|
+
output: last.result || '',
|
|
55
|
+
stopReason: last.subtype || 'unknown',
|
|
56
|
+
numTurns: last.num_turns || 1,
|
|
57
|
+
fullNumTurns: trace.fullNumTurns,
|
|
58
|
+
numSubAgents: trace.numSubAgents,
|
|
59
|
+
...(trace.turns.length > 0 && { turns: trace.turns }),
|
|
60
|
+
...(trace.toolCalls.length > 0 && { toolCalls: trace.toolCalls }),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
const durationMs = Date.now() - start;
|
|
65
|
+
const details = asErrorLike(err);
|
|
66
|
+
if (details.killed)
|
|
67
|
+
return timeoutExecResult(timeoutMs, durationMs);
|
|
68
|
+
// 尝试从 stdout 解析 stream-json(即使进程退出码非 0 也可能有 result)
|
|
69
|
+
const messages = parseStreamJson(details.stdout || '');
|
|
70
|
+
const resultMsgs = messages.filter(isClaudeSdkResultMessage);
|
|
71
|
+
if (resultMsgs.length > 0) {
|
|
72
|
+
const last = resultMsgs[resultMsgs.length - 1];
|
|
73
|
+
const usage = last.usage || {};
|
|
74
|
+
const trace = extractAgentTrace(messages);
|
|
75
|
+
return {
|
|
76
|
+
ok: false,
|
|
77
|
+
error: last.errors?.join('; ') || last.result || errorMessage(err),
|
|
78
|
+
durationMs: last.duration_ms || durationMs,
|
|
79
|
+
durationApiMs: last.duration_api_ms || 0,
|
|
80
|
+
inputTokens: usage.input_tokens || 0,
|
|
81
|
+
outputTokens: usage.output_tokens || 0,
|
|
82
|
+
cacheReadTokens: usage.cache_read_input_tokens || 0,
|
|
83
|
+
cacheCreationTokens: usage.cache_creation_input_tokens || 0,
|
|
84
|
+
costUSD: last.total_cost_usd || 0,
|
|
85
|
+
output: last.result || null,
|
|
86
|
+
stopReason: 'error',
|
|
87
|
+
numTurns: last.num_turns || 0,
|
|
88
|
+
fullNumTurns: trace.fullNumTurns,
|
|
89
|
+
numSubAgents: trace.numSubAgents,
|
|
90
|
+
...(trace.turns.length > 0 && { turns: trace.turns }),
|
|
91
|
+
...(trace.toolCalls.length > 0 && { toolCalls: trace.toolCalls }),
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
ok: false,
|
|
96
|
+
error: errorMessage(err),
|
|
97
|
+
durationMs, durationApiMs: 0,
|
|
98
|
+
inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0,
|
|
99
|
+
costUSD: 0, output: null, stopReason: 'error', numTurns: 0,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
//# sourceMappingURL=claude-cli.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"claude-cli.js","sourceRoot":"","sources":["../../../src/executors/claude-cli.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AAEpF,OAAO,EACL,WAAW,EACX,YAAY,EACZ,kBAAkB,EAClB,YAAY,EACZ,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,aAAa,CAAC;AAErB,SAAS,eAAe,CAAC,MAAc;IACrC,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,IAAI,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAyB,CAAC,CAAC;QAC7D,CAAC;QAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,GAAG,kBAAkB,EAAiB;IAC7H,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,EAAE,aAAa,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC7F,IAAI,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IAEjD,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEnC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE;YACrD,SAAS,EAAE,UAAU;YACrB,OAAO,EAAE,SAAS;YAClB,GAAG;YACH,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;SACpB,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QAEzC,eAAe;QACf,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,wBAAwB,CAA6B,CAAC;QACzF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO;gBACL,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,yCAAyC;gBAC3D,UAAU,EAAE,aAAa,EAAE,CAAC;gBAC5B,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC;gBAC3E,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;aAC3D,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAE/B,WAAW;QACX,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAE1C,OAAO;YACL,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO;YACpD,UAAU,EAAE,IAAI,CAAC,WAAW,IAAI,UAAU;YAC1C,aAAa,EAAE,IAAI,CAAC,eAAe,IAAI,CAAC;YACxC,WAAW,EAAE,KAAK,CAAC,YAAY,IAAI,CAAC;YACpC,YAAY,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC;YACtC,eAAe,EAAE,KAAK,CAAC,uBAAuB,IAAI,CAAC;YACnD,mBAAmB,EAAE,KAAK,CAAC,2BAA2B,IAAI,CAAC;YAC3D,OAAO,EAAE,IAAI,CAAC,cAAc,IAAI,CAAC;YACjC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;YACzB,UAAU,EAAE,IAAI,CAAC,OAAO,IAAI,SAAS;YACrC,QAAQ,EAAE,IAAI,CAAC,SAAS,IAAI,CAAC;YAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;YACrD,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC;SAClE,CAAC;IACJ,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,OAAO,CAAC,MAAM;YAAE,OAAO,iBAAiB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAEpE,oDAAoD;QACpD,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACvD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,wBAAwB,CAA6B,CAAC;QACzF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAC1C,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,GAAG,CAAC;gBAClE,UAAU,EAAE,IAAI,CAAC,WAAW,IAAI,UAAU;gBAC1C,aAAa,EAAE,IAAI,CAAC,eAAe,IAAI,CAAC;gBACxC,WAAW,EAAE,KAAK,CAAC,YAAY,IAAI,CAAC;gBACpC,YAAY,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC;gBACtC,eAAe,EAAE,KAAK,CAAC,uBAAuB,IAAI,CAAC;gBACnD,mBAAmB,EAAE,KAAK,CAAC,2BAA2B,IAAI,CAAC;gBAC3D,OAAO,EAAE,IAAI,CAAC,cAAc,IAAI,CAAC;gBACjC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;gBAC3B,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,IAAI,CAAC,SAAS,IAAI,CAAC;gBAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBACrD,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC;aAClE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC;YACxB,UAAU,EAAE,aAAa,EAAE,CAAC;YAC5B,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC;YAC3E,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;SAC3D,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { ToolCallInfo, TurnInfo } from '../types.js';
|
|
2
|
+
import type { ClaudeSdkBaseMessage } from './shared.js';
|
|
3
|
+
export declare function isClaudeSdkResultMessage(message: ClaudeSdkBaseMessage): boolean;
|
|
4
|
+
export declare function extractAgentTrace(messages: ClaudeSdkBaseMessage[], timestamps?: number[]): {
|
|
5
|
+
turns: TurnInfo[];
|
|
6
|
+
toolCalls: ToolCallInfo[];
|
|
7
|
+
fullNumTurns: number;
|
|
8
|
+
numSubAgents: number;
|
|
9
|
+
};
|
|
10
|
+
//# sourceMappingURL=claude-sdk-trace.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"claude-sdk-trace.d.ts","sourceRoot":"","sources":["../../../src/executors/claude-sdk-trace.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC1D,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAExD,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAE/E;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG;IAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IAAC,SAAS,EAAE,YAAY,EAAE,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAgGvL"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
export function isClaudeSdkResultMessage(message) {
|
|
2
|
+
return message.type === 'result';
|
|
3
|
+
}
|
|
4
|
+
export function extractAgentTrace(messages, timestamps) {
|
|
5
|
+
const turns = [];
|
|
6
|
+
const toolCalls = [];
|
|
7
|
+
const pendingToolUse = new Map();
|
|
8
|
+
let lastTurnTs = timestamps?.[0] || 0;
|
|
9
|
+
let fullNumTurns = 0;
|
|
10
|
+
let numSubAgents = 0;
|
|
11
|
+
for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) {
|
|
12
|
+
const msg = messages[msgIdx];
|
|
13
|
+
const msgTs = timestamps?.[msgIdx] || 0;
|
|
14
|
+
if (msg.type === 'result' || msg.type === 'system' || msg.type === 'rate_limit_event')
|
|
15
|
+
continue;
|
|
16
|
+
const content = msg.message?.content;
|
|
17
|
+
if (!Array.isArray(content))
|
|
18
|
+
continue;
|
|
19
|
+
if (msg.type === 'assistant') {
|
|
20
|
+
const textParts = [];
|
|
21
|
+
const turnToolCalls = [];
|
|
22
|
+
let hasNonThinking = false;
|
|
23
|
+
for (const block of content) {
|
|
24
|
+
if (block.type === 'text' && block.text) {
|
|
25
|
+
textParts.push(block.text);
|
|
26
|
+
hasNonThinking = true;
|
|
27
|
+
}
|
|
28
|
+
else if (block.type === 'tool_use' && block.name) {
|
|
29
|
+
pendingToolUse.set(block.id || '', { tool: block.name, input: block.input });
|
|
30
|
+
turnToolCalls.push({ tool: block.name, input: block.input, output: null, success: true });
|
|
31
|
+
hasNonThinking = true;
|
|
32
|
+
if (block.name === 'Agent')
|
|
33
|
+
numSubAgents++;
|
|
34
|
+
}
|
|
35
|
+
else if (block.type !== 'thinking') {
|
|
36
|
+
hasNonThinking = true;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
if (hasNonThinking)
|
|
40
|
+
fullNumTurns++;
|
|
41
|
+
if (textParts.length > 0 || turnToolCalls.length > 0) {
|
|
42
|
+
const dur = msgTs && lastTurnTs ? msgTs - lastTurnTs : undefined;
|
|
43
|
+
turns.push({
|
|
44
|
+
role: 'assistant',
|
|
45
|
+
content: textParts.join('\n'),
|
|
46
|
+
...(turnToolCalls.length > 0 && { toolCalls: turnToolCalls }),
|
|
47
|
+
...(dur != null && dur > 0 && { durationMs: dur }),
|
|
48
|
+
});
|
|
49
|
+
if (msgTs)
|
|
50
|
+
lastTurnTs = msgTs;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (msg.type === 'user') {
|
|
54
|
+
for (const block of content) {
|
|
55
|
+
if (block.type !== 'tool_result')
|
|
56
|
+
continue;
|
|
57
|
+
const toolUseId = block.tool_use_id || '';
|
|
58
|
+
const pending = pendingToolUse.get(toolUseId);
|
|
59
|
+
const isError = block.is_error || false;
|
|
60
|
+
const resultContent = block.content;
|
|
61
|
+
const outputText = typeof resultContent === 'string'
|
|
62
|
+
? resultContent
|
|
63
|
+
: Array.isArray(resultContent)
|
|
64
|
+
? resultContent.map((c) => c.text || '').join('')
|
|
65
|
+
: '';
|
|
66
|
+
const tc = {
|
|
67
|
+
tool: pending?.tool || 'unknown',
|
|
68
|
+
input: pending?.input || null,
|
|
69
|
+
output: outputText.slice(0, 500),
|
|
70
|
+
success: !isError,
|
|
71
|
+
};
|
|
72
|
+
toolCalls.push(tc);
|
|
73
|
+
if (pending) {
|
|
74
|
+
for (let i = turns.length - 1; i >= 0; i--) {
|
|
75
|
+
const turn = turns[i];
|
|
76
|
+
if (turn.role === 'assistant' && turn.toolCalls) {
|
|
77
|
+
const placeholder = turn.toolCalls.find((t) => t.tool === pending.tool && t.output === null);
|
|
78
|
+
if (placeholder) {
|
|
79
|
+
placeholder.output = tc.output;
|
|
80
|
+
placeholder.success = !isError;
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const toolDur = msgTs && lastTurnTs ? msgTs - lastTurnTs : undefined;
|
|
87
|
+
turns.push({
|
|
88
|
+
role: 'tool',
|
|
89
|
+
content: outputText.slice(0, 500),
|
|
90
|
+
...(toolDur != null && toolDur > 0 && { durationMs: toolDur }),
|
|
91
|
+
});
|
|
92
|
+
if (msgTs)
|
|
93
|
+
lastTurnTs = msgTs;
|
|
94
|
+
pendingToolUse.delete(toolUseId);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return { turns, toolCalls, fullNumTurns, numSubAgents };
|
|
99
|
+
}
|
|
100
|
+
//# sourceMappingURL=claude-sdk-trace.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"claude-sdk-trace.js","sourceRoot":"","sources":["../../../src/executors/claude-sdk-trace.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,wBAAwB,CAAC,OAA6B;IACpE,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAAgC,EAAE,UAAqB;IACvF,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,MAAM,cAAc,GAAG,IAAI,GAAG,EAA4C,CAAC;IAC3E,IAAI,UAAU,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QACxD,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,KAAK,kBAAkB;YAAE,SAAS;QAEhG,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,SAAS;QAEtC,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,MAAM,aAAa,GAAmB,EAAE,CAAC;YACzC,IAAI,cAAc,GAAG,KAAK,CAAC;YAE3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACxC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC3B,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACnD,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;oBAC7E,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC1F,cAAc,GAAG,IAAI,CAAC;oBACtB,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;wBAAE,YAAY,EAAE,CAAC;gBAC7C,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;oBACrC,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,IAAI,cAAc;gBAAE,YAAY,EAAE,CAAC;YAEnC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrD,MAAM,GAAG,GAAG,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;gBACjE,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAC7B,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC;oBAC7D,GAAG,CAAC,GAAG,IAAI,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;iBACnD,CAAC,CAAC;gBACH,IAAI,KAAK;oBAAE,UAAU,GAAG,KAAK,CAAC;YAChC,CAAC;QACH,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa;oBAAE,SAAS;gBAC3C,MAAM,SAAS,GAAI,KAA6C,CAAC,WAAW,IAAI,EAAE,CAAC;gBACnF,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,OAAO,GAAI,KAA2C,CAAC,QAAQ,IAAI,KAAK,CAAC;gBAC/E,MAAM,aAAa,GAAI,KAAkF,CAAC,OAAO,CAAC;gBAClH,MAAM,UAAU,GAAG,OAAO,aAAa,KAAK,QAAQ;oBAClD,CAAC,CAAC,aAAa;oBACf,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC;wBAC5B,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;wBACjD,CAAC,CAAC,EAAE,CAAC;gBAET,MAAM,EAAE,GAAiB;oBACvB,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,SAAS;oBAChC,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,IAAI;oBAC7B,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;oBAChC,OAAO,EAAE,CAAC,OAAO;iBAClB,CAAC;gBACF,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEnB,IAAI,OAAO,EAAE,CAAC;oBACZ,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;wBACtB,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;4BAChD,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;4BAC7F,IAAI,WAAW,EAAE,CAAC;gCAChB,WAAW,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;gCAC/B,WAAW,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC;gCAC/B,MAAM;4BACR,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,MAAM,OAAO,GAAG,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;gBACrE,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;oBACjC,GAAG,CAAC,OAAO,IAAI,IAAI,IAAI,OAAO,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;iBAC/D,CAAC,CAAC;gBACH,IAAI,KAAK;oBAAE,UAAU,GAAG,KAAK,CAAC;gBAC9B,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;AAC1D,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"claude-sdk.d.ts","sourceRoot":"","sources":["../../../src/executors/claude-sdk.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAe7D,wBAAsB,iBAAiB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,SAA8B,EAAE,OAAe,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,CAqJrK"}
|