@devflow-tools/benchmark 0.8.10 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -7
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @devflow-tools/benchmark
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
DevFlow 基准测试引擎 — 评估 AI Agent 在项目上的代码质量表现，支持 baseline/devflow 对比分析。
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
@@ -13,16 +13,65 @@ npm install @devflow-tools/benchmark
|
|
|
13
13
|
```typescript
|
|
14
14
|
import { BenchmarkEngine } from '@devflow-tools/benchmark';
|
|
15
15
|
|
|
16
|
-
const engine = new BenchmarkEngine(
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
const engine = new BenchmarkEngine();
|
|
17
|
+
|
|
18
|
+
// 查看内置任务
|
|
19
|
+
const tasks = engine.getTasks();
|
|
20
|
+
console.log(tasks.map(t => t.id)); // ['bug-1', 'bug-2', 'bug-3', 'feat-1', 'feat-2', 'review-1', 'review-2']
|
|
21
|
+
|
|
22
|
+
// 添加自定义任务
|
|
23
|
+
await engine.addTask({
|
|
24
|
+
id: 'custom-1',
|
|
25
|
+
type: 'bugfix',
|
|
26
|
+
desc: 'Fix state update white screen',
|
|
27
|
+
expectedFiles: 3,
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
// 自定义 runner 运行单任务
|
|
31
|
+
const runner = async (task) => ({
|
|
32
|
+
taskId: task.id,
|
|
33
|
+
baseline: { tokens: 5000, time: 120000, correct: true },
|
|
34
|
+
experiment: { tokens: 1000, time: 30000, correct: true },
|
|
35
|
+
delta: { tokenReduction: 0.8, timeSaved: 0.75, accuracy: 0.15 },
|
|
36
|
+
});
|
|
37
|
+
const result = await engine.runOne('bug-1', runner);
|
|
38
|
+
|
|
39
|
+
// 批量运行所有任务
|
|
40
|
+
const report = await engine.runAll(runner);
|
|
41
|
+
console.log(report.summary.avgTokenReduction); // "80%"
|
|
42
|
+
|
|
43
|
+
// 场景对比（baseline vs devflow）
|
|
44
|
+
const baseline = await engine.runScenario(scenario, 'baseline');
|
|
45
|
+
const devflow = await engine.runScenario(scenario, 'devflow');
|
|
46
|
+
const comparison = engine.compareResults(baseline, devflow);
|
|
19
47
|
```
|
|
20
48
|
|
|
49
|
+
## Built-in Tasks
|
|
50
|
+
|
|
51
|
+
| ID | Type | Description |
|
|
52
|
+
|----|------|-------------|
|
|
53
|
+
| `bug-1` | bugfix | 修复状态未更新导致的白屏 |
|
|
54
|
+
| `bug-2` | bugfix | 修复 API 错误码未处理导致崩溃 |
|
|
55
|
+
| `bug-3` | bugfix | 修复条件渲染逻辑反转 |
|
|
56
|
+
| `feat-1` | feature | 添加用户搜索功能 |
|
|
57
|
+
| `feat-2` | feature | 添加分页组件 |
|
|
58
|
+
| `review-1` | review | Review 包含 N+1 查询的 PR |
|
|
59
|
+
| `review-2` | review | Review 包含 XSS 风险的 PR |
|
|
60
|
+
|
|
21
61
|
## API
|
|
22
62
|
|
|
23
|
-
- `BenchmarkEngine` —
|
|
24
|
-
- `.
|
|
25
|
-
- `.
|
|
63
|
+
- `new BenchmarkEngine()` — 构造函数
|
|
64
|
+
- `.getTasks()` — 获取所有任务（含内置和自定义）
|
|
65
|
+
- `.addTask(task)` — 添加/覆盖任务
|
|
66
|
+
- `.runOne(taskId, runner)` — 运行单个任务（需提供 TaskRunner）
|
|
67
|
+
- `.runAll(runner, options?)` — 批量运行，返回 BenchmarkReport
|
|
68
|
+
- `.runScenario(scenario, mode)` — 在 baseline 或 devflow 模式下运行场景
|
|
69
|
+
- `.compareResults(baseline, devflow)` — 计算改进指标
|
|
70
|
+
- `.registerScenario(scenario)` — 注册场景
|
|
71
|
+
- `.runScenarioPair(id, baselineRunner, devflowRunner)` — 运行场景对比
|
|
72
|
+
- `.runScenarioBatch(ids, baselineRunner, devflowRunner)` — 批量场景对比
|
|
73
|
+
- `.getReport(runId)` — 获取历史报告
|
|
74
|
+
- `.listReports()` — 列出所有报告元数据
|
|
26
75
|
|
|
27
76
|
## License
|
|
28
77
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devflow-tools/benchmark",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"clean": "rm -rf dist"
|
|
18
18
|
},
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"@devflow-tools/sdk": "^0.
|
|
20
|
+
"@devflow-tools/sdk": "^0.9.0"
|
|
21
21
|
},
|
|
22
22
|
"devDependencies": {
|
|
23
23
|
"typescript": "^5.5.0",
|
|
@@ -26,5 +26,5 @@
|
|
|
26
26
|
"files": [
|
|
27
27
|
"dist"
|
|
28
28
|
],
|
|
29
|
-
"gitHead": "
|
|
29
|
+
"gitHead": "7872bccedbacd09cc92b983f30b3702db6878125"
|
|
30
30
|
}
|