@devflow-tools/benchmark 0.17.4 → 0.17.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -62
- package/package.json +5 -5
package/README.md
CHANGED
|
@@ -1,77 +1,43 @@
|
|
|
1
1
|
# @devflow-tools/benchmark
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
> 用于比较 baseline 与 DevFlow Agent 结果的评测和检索质量基准库。（v0.17.6）
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
[](https://www.npmjs.com/package/@devflow-tools/benchmark)
|
|
6
|
+
|
|
7
|
+
## 安装
|
|
6
8
|
|
|
7
9
|
```bash
|
|
8
10
|
npm install @devflow-tools/benchmark
|
|
9
11
|
```
|
|
10
12
|
|
|
11
|
-
##
|
|
13
|
+
## 快速开始
|
|
12
14
|
|
|
13
15
|
```typescript
|
|
14
|
-
import {
|
|
15
|
-
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
// 查看内置任务
|
|
19
|
-
const tasks = engine.getTasks();
|
|
20
|
-
console.log(tasks.map(t => t.id)); // ['bug-1', 'bug-2', 'bug-3', 'feat-1', 'feat-2', 'review-1', 'review-2']
|
|
21
|
-
|
|
22
|
-
// 添加自定义任务
|
|
23
|
-
await engine.addTask({
|
|
24
|
-
id: 'custom-1',
|
|
25
|
-
type: 'bugfix',
|
|
26
|
-
desc: 'Fix state update white screen',
|
|
27
|
-
expectedFiles: 3,
|
|
28
|
-
});
|
|
29
|
-
|
|
30
|
-
// 自定义 runner 运行单任务
|
|
31
|
-
const runner = async (task) => ({
|
|
32
|
-
taskId: task.id,
|
|
33
|
-
baseline: { tokens: 5000, time: 120000, correct: true },
|
|
34
|
-
experiment: { tokens: 1000, time: 30000, correct: true },
|
|
35
|
-
delta: { tokenReduction: 0.8, timeSaved: 0.75, accuracy: 0.15 },
|
|
36
|
-
});
|
|
37
|
-
const result = await engine.runOne('bug-1', runner);
|
|
38
|
-
|
|
39
|
-
// 批量运行所有任务
|
|
40
|
-
const report = await engine.runAll(runner);
|
|
41
|
-
console.log(report.summary.avgTokenReduction); // "80%"
|
|
42
|
-
|
|
43
|
-
// 场景对比（baseline vs devflow）
|
|
44
|
-
const baseline = await engine.runScenario(scenario, 'baseline');
|
|
45
|
-
const devflow = await engine.runScenario(scenario, 'devflow');
|
|
46
|
-
const comparison = engine.compareResults(baseline, devflow);
|
|
16
|
+
import { runBenchmarkSuite, DEFAULT_BENCHMARK_SUITE } from '@devflow-tools/benchmark';
|
|
17
|
+
|
|
18
|
+
const report = await runBenchmarkSuite(process.cwd(), DEFAULT_BENCHMARK_SUITE);
|
|
19
|
+
console.log(report.summary);
|
|
47
20
|
```
|
|
48
21
|
|
|
49
|
-
##
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
-
|
|
67
|
-
-
|
|
68
|
-
- `.runScenario(scenario, mode)` — 在 baseline 或 devflow 模式下运行场景
|
|
69
|
-
- `.compareResults(baseline, devflow)` — 计算改进指标
|
|
70
|
-
- `.registerScenario(scenario)` — 注册场景
|
|
71
|
-
- `.runScenarioPair(id, baselineRunner, devflowRunner)` — 运行场景对比
|
|
72
|
-
- `.runScenarioBatch(ids, baselineRunner, devflowRunner)` — 批量场景对比
|
|
73
|
-
- `.getReport(runId)` — 获取历史报告
|
|
74
|
-
- `.listReports()` — 列出所有报告元数据
|
|
22
|
+
## 公共 API / 命令
|
|
23
|
+
|
|
24
|
+
`BenchmarkEngine`、`runBenchmarkSuite`、`runRetrievalQualityBenchmark`、`evaluateSemanticFixtures`、`renderBenchmarkReport` 以及内置 fixture/类型。
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
## 配置与运行时
|
|
28
|
+
|
|
29
|
+
该包用于离线评测，不是 DevFlow 运行时必需依赖；自定义 runner 负责实际执行任务。
|
|
30
|
+
|
|
31
|
+
## 版本与兼容性
|
|
32
|
+
|
|
33
|
+
- 当前包版本：`0.17.6`
|
|
34
|
+
- Node.js：未在 package manifest 声明更高版本时，使用 Node.js 18+；原生 SQLite/Playwright 能力请按对应依赖安装
|
|
35
|
+
- workspace 内部包应使用同一 DevFlow minor/patch 版本，避免类型和数据库 schema 不一致
|
|
36
|
+
|
|
37
|
+
## 相关链接
|
|
38
|
+
|
|
39
|
+
- [DevFlow 仓库](https://github.com/shilongfeicool/dev-flow)
|
|
40
|
+
- [问题反馈](https://github.com/shilongfeicool/dev-flow/issues)
|
|
75
41
|
|
|
76
42
|
## License
|
|
77
43
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devflow-tools/benchmark",
|
|
3
|
-
"version": "0.17.
|
|
3
|
+
"version": "0.17.6",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -19,9 +19,9 @@
|
|
|
19
19
|
},
|
|
20
20
|
"dependencies": {
|
|
21
21
|
"@colbymchenry/codegraph": "^1.1.1",
|
|
22
|
-
"@devflow-tools/context-engine": "0.17.
|
|
23
|
-
"@devflow-tools/database": "0.17.
|
|
24
|
-
"@devflow-tools/sdk": "0.17.
|
|
22
|
+
"@devflow-tools/context-engine": "0.17.6",
|
|
23
|
+
"@devflow-tools/database": "0.17.6",
|
|
24
|
+
"@devflow-tools/sdk": "0.17.6"
|
|
25
25
|
},
|
|
26
26
|
"devDependencies": {
|
|
27
27
|
"typescript": "^5.5.0",
|
|
@@ -31,5 +31,5 @@
|
|
|
31
31
|
"dist",
|
|
32
32
|
"fixtures"
|
|
33
33
|
],
|
|
34
|
-
"gitHead": "
|
|
34
|
+
"gitHead": "669fea3aa72aa3fde14ed50d24c1fec4dfac088f"
|
|
35
35
|
}
|