@devflow-tools/benchmark 0.15.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/accuracy-queries.d.ts.map +1 -1
- package/dist/accuracy-queries.js +1 -0
- package/dist/accuracy-queries.js.map +1 -1
- package/dist/benchmark-engine.d.ts +3 -1
- package/dist/benchmark-engine.d.ts.map +1 -1
- package/dist/benchmark-engine.js +4 -0
- package/dist/benchmark-engine.js.map +1 -1
- package/dist/default-suite.d.ts +4 -0
- package/dist/default-suite.d.ts.map +1 -0
- package/dist/default-suite.js +43 -0
- package/dist/default-suite.js.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/lexical-runner.d.ts +14 -0
- package/dist/lexical-runner.d.ts.map +1 -0
- package/dist/lexical-runner.js +98 -0
- package/dist/lexical-runner.js.map +1 -0
- package/dist/report-renderer.d.ts +4 -0
- package/dist/report-renderer.d.ts.map +1 -0
- package/dist/report-renderer.js +78 -0
- package/dist/report-renderer.js.map +1 -0
- package/dist/suite-runner.d.ts +10 -0
- package/dist/suite-runner.d.ts.map +1 -0
- package/dist/suite-runner.js +189 -0
- package/dist/suite-runner.js.map +1 -0
- package/package.json +4 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"accuracy-queries.d.ts","sourceRoot":"","sources":["../src/accuracy-queries.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,WAAW,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC7C,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,cAAc,EAAE,gBAAgB,
|
|
1
|
+
{"version":3,"file":"accuracy-queries.d.ts","sourceRoot":"","sources":["../src/accuracy-queries.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,WAAW,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC7C,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,cAAc,EAAE,gBAAgB,EA4G5C,CAAC"}
|
package/dist/accuracy-queries.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"accuracy-queries.js","sourceRoot":"","sources":["../src/accuracy-queries.ts"],"names":[],"mappings":"AAUA,MAAM,CAAC,MAAM,cAAc,GAAuB;IAChD;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,WAAW;QACnB,aAAa,EAAE,CAAC,aAAa,EAAE,YAAY,CAAC;QAC5C,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,WAAW;QACnB,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,aAAa,EAAE,CAAC,mBAAmB,CAAC;QACpC,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,6BAA6B;QACpC,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,OAAO;QACrB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,6BAA6B;QACpC,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,KAAK;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,uCAAuC;QAC9C,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,MAAM;QACpB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,kCAAkC;QACzC,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,4BAA4B;QACnC,MAAM,EAAE,WAAW;QACnB,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,4CAA4C;QACnD,MAAM,EAAE,WAAW;QACnB,WAAW,EAAE,CAAC,oBAAoB,CAAC;QACnC,aAAa,EAAE,CAAC,eAAe,CAAC;QAChC,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,eAAe;QACtB,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,EAAE;QACT,UAAU,EAAE,CAAC;KACd;IACD,6DAA6D;IAC7D;QACE,KAAK,EAAE,UAAU;QACjB,MAAM,EAAE,WAAW;QACnB,aAAa,EAAE,CAAC,WAAW,EAAE,YAAY,CAAC;QAC1C,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,WAAW;QAClB,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,kBAAkB;QACzB,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,OAAO;QACrB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,qBAAqB;QAC5B,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,eAAe;QACtB,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,KAAK;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,YAAY;QAC1B,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,sBAAsB;QAC7B,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,UAAU;QACxB,UAAU,EAAE,CAAC;KACd;CACF,CAAC"}
|
|
1
|
+
{"version":3,"file":"accuracy-queries.js","sourceRoot":"","sources":["../src/accuracy-queries.ts"],"names":[],"mappings":"AAUA,MAAM,CAAC,MAAM,cAAc,GAAuB;IAChD;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,WAAW;QACnB,aAAa,EAAE,CAAC,aAAa,EAAE,YAAY,CAAC;QAC5C,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,WAAW;QACnB,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,aAAa,EAAE,CAAC,mBAAmB,CAAC;QACpC,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,6BAA6B;QACpC,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,OAAO;QACrB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,6BAA6B;QACpC,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,KAAK;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,uCAAuC;QAC9C,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,MAAM;QACpB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,kCAAkC;QACzC,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,4BAA4B;QACnC,MAAM,EAAE,WAAW;QACnB,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,4CAA4C;QACnD,MAAM,EAAE,WAAW;QACnB,WAAW,EAAE,CAAC,oBAAoB,CAAC;QACnC,aAAa,EAAE,CAAC,eAAe,CAAC;QAChC,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,eAAe;QACtB,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,EAAE;QACT,UAAU,EAAE,CAAC;KACd;IACD,6DAA6D;IAC7D;QACE,KAAK,EAAE,UAAU;QACjB,MAAM,EAAE,WAAW;QACnB,aAAa,EAAE,CAAC,WAAW,EAAE,YAAY,CAAC;QAC1C,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,WAAW;QAClB,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,kBAAkB;QACzB,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,OAAO;QACrB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,WAAW;QACnB,WAAW,EAAE,CAAC,aAAa,CAAC;QAC5B,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,qBAAqB;QAC5B,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,eAAe;QACtB,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,KAAK;QACnB,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,YAAY;QAC1B,UAAU,EAAE,CAAC;KACd;IACD;QACE,KAAK,EAAE,sBAAsB;QAC7B,MAAM,EAAE,WAAW;QACnB,YAAY,EAAE,UAAU;QACxB,UAAU,EAAE,CAAC;KACd;CACF,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type { BenchmarkTask, TaskResult, BenchmarkReport, ReportMeta, BenchmarkScenario, BenchmarkRunResult, ComparisonResult, BenchmarkComparisonResult } from "@devflow-tools/sdk";
|
|
1
|
+
import type { BenchmarkTask, TaskResult, BenchmarkReport, ReportMeta, BenchmarkScenario, BenchmarkRunResult, ComparisonResult, BenchmarkComparisonResult, BenchmarkSuite, BenchmarkSuiteReport } from "@devflow-tools/sdk";
|
|
2
|
+
import { type SuiteRunOptions } from "./suite-runner.js";
|
|
2
3
|
export interface BenchmarkOptions {
|
|
3
4
|
taskIds?: string[];
|
|
4
5
|
rounds?: number;
|
|
@@ -15,6 +16,7 @@ export declare class BenchmarkEngine {
|
|
|
15
16
|
getReport(runId: string): Promise<BenchmarkReport | null>;
|
|
16
17
|
listReports(): Promise<ReportMeta[]>;
|
|
17
18
|
getTasks(): BenchmarkTask[];
|
|
19
|
+
runSuite(projectRoot: string, suite: BenchmarkSuite, options?: SuiteRunOptions): Promise<BenchmarkSuiteReport>;
|
|
18
20
|
runScenario(scenario: BenchmarkScenario, mode: "baseline" | "devflow"): Promise<BenchmarkRunResult>;
|
|
19
21
|
private runBaseline;
|
|
20
22
|
private runDevFlow;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark-engine.d.ts","sourceRoot":"","sources":["../src/benchmark-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,eAAe,EACf,UAAU,EACV,iBAAiB,EACjB,kBAAkB,EAClB,gBAAgB,EAChB,yBAAyB,
|
|
1
|
+
{"version":3,"file":"benchmark-engine.d.ts","sourceRoot":"","sources":["../src/benchmark-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,eAAe,EACf,UAAU,EACV,iBAAiB,EACjB,kBAAkB,EAClB,gBAAgB,EAChB,yBAAyB,EAEzB,cAAc,EACd,oBAAoB,EACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAqB,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAY5E,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,aAAa,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;AAEtE,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,KAAK,CAAuC;IACpD,OAAO,CAAC,SAAS,CAAwC;IAEnD,OAAO,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAS3C,MAAM,CACV,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,UAAU,CAAC;IAMhB,MAAM,CACV,MAAM,EAAE,UAAU,EAClB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;IAwB3B,OAAO,CAAC,cAAc;IAuBhB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAIzD,WAAW,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;IAQ1C,QAAQ,IAAI,aAAa,EAAE;IAIrB,QAAQ,CACZ,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,cAAc,EACrB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,oBAAoB,CAAC;IAI1B,WAAW,CACf,QAAQ,EAAE,iBAAiB,EAC3B,IAAI,EAAE,UAAU,GAAG,SAAS,GAC3B,OAAO,CAAC,kBAAkB,CAAC;YAyBhB,WAAW;YAgBX,UAAU;IAgBxB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,EAAE,kBAAkB,GAC1B,gBAAgB;IAmBnB,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,IAAI;IAI7C,eAAe,CACnB,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,CACd,MAAM,EAAE,MAAM,KACX,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,YAAY,GAAG,MAAM,GAAG,WAAW,CAAC,CAAC,EAC3E,aAAa,EAAE,CACb,MAAM,EAAE,MAAM,KACX,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,YAAY,GAAG,MAAM,GAAG,WAAW,CAAC,CAAC,GAC1E,OAAO,CAAC,yBAAyB,CAAC;IAiD/B,gBAAgB,CACpB,WAAW,EAAE,MAAM,EAAE,EACrB,cAAc,EAAE,CACd,MAAM,EAAE,MAAM,KACX,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,YAAY,GAAG,MAAM,GAAG,WAAW,CAAC,CAAC,EAC3E,aAAa,EAAE,CACb,MAAM,EAAE,MAAM,KACX,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,YAAY,GAAG,MAAM,GAAG,WAAW,CAAC,CAAC,GAC1E,OAAO,CAAC,yBAAyB,EAAE,CAAC;CAQxC"}
|
package/dist/benchmark-engine.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { runBenchmarkSuite } from "./suite-runner.js";
|
|
1
2
|
const BUILTIN_TASKS = [
|
|
2
3
|
{ id: "bug-1", type: "bugfix", desc: "修复状态未更新导致的白屏", expectedFiles: 3 },
|
|
3
4
|
{ id: "bug-2", type: "bugfix", desc: "修复 API 错误码未处理导致崩溃", expectedFiles: 2 },
|
|
@@ -78,6 +79,9 @@ export class BenchmarkEngine {
|
|
|
78
79
|
getTasks() {
|
|
79
80
|
return [...this.tasks];
|
|
80
81
|
}
|
|
82
|
+
async runSuite(projectRoot, suite, options) {
|
|
83
|
+
return runBenchmarkSuite(projectRoot, suite, options);
|
|
84
|
+
}
|
|
81
85
|
async runScenario(scenario, mode) {
|
|
82
86
|
const startTime = Date.now();
|
|
83
87
|
const prepStart = startTime;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark-engine.js","sourceRoot":"","sources":["../src/benchmark-engine.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"benchmark-engine.js","sourceRoot":"","sources":["../src/benchmark-engine.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,iBAAiB,EAAwB,MAAM,mBAAmB,CAAC;AAE5E,MAAM,aAAa,GAAoB;IACrC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,EAAE;IACvE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,mBAAmB,EAAE,aAAa,EAAE,CAAC,EAAE;IAC5E,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,EAAE;IACrE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,iBAAiB,EAAE,CAAC,EAAE;IACzE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC,EAAE;IACxE,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,sBAAsB,EAAE,cAAc,EAAE,CAAC,EAAE;IACnF,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,sBAAsB,EAAE,cAAc,EAAE,CAAC,EAAE;CACpF,CAAC;AASF,MAAM,OAAO,eAAe;IAClB,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC7C,KAAK,GAAoB,CAAC,GAAG,aAAa,CAAC,CAAC;IAC5C,SAAS,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEzD,KAAK,CAAC,OAAO,CAAC,IAAmB;QAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;YAClB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;QAC9B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CACV,MAAc,EACd,MAAkB;QAElB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,aAAa,CAAC,CAAC;QACzD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,MAAM,CACV,MAAkB,EAClB,OAA0B;QAE1B,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO;YAC9B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,OAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3D,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;QAEf,MAAM,KAAK,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC9E,MAAM,WAAW,GAAiB,EAAE,CAAC;QAErC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YAClC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;QAED,MAAM,MAAM,GAAoB;YAC9B,KAAK;YACL,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YACzC,KAAK,EAAE,WAAW;SACnB,CAAC;QAEF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAChC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,cAAc,CAAC,OAAqB;QAC1C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;gBACL,iBAAiB,EAAE,IAAI;gBACvB,YAAY,EAAE,IAAI;gBAClB,eAAe,EAAE,IAAI;gBACrB,iBAAiB,EAAE,IAAI;aACxB,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAC/E,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QAEzB,OAAO;YACL,iBAAiB,EAAE,GAAG,CAAC,cAAc,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YAC9D,YAAY,EAAE,IAAI,CAAC,cAAc,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YAC1D,eAAe,EAAE,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YAC5D,iBAAiB,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,CAAC,GAAG,IAAI,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;SACtF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAa;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,WAAW;QACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACnD,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;IACN,CAAC;IAED,QAAQ;QACN,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,WAAmB,EACnB,KAAqB,EACrB,OAAyB;QAEzB,OAAO,iBAAiB,CAAC,WAAW,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,WAAW,CACf,QAA2B,EAC3B,IAA4B;QAE5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,SAAS,CAAC;QAE5B,MAAM,MAAM,GACV,IAAI,KAAK,UAAU;YACjB,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;YAClC,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAEtC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAE1C,OAAO;YACL,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,IAAI;YACJ,UAAU;YACV,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,UAAU;YACV,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,SAAS,EAAE,SAAS;SACrB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,SAA4B;QAO5B,OAAO;YACL,UAAU,EAAE,CAAC;YACb,gBAAgB,EAAE,KAAK;YACvB,WAAW,EAAE,CAAC;YACd,KAAK,EAAE,iDAAiD;SACzD,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,SAA4B;QAO5B,OAAO;YACL,UAAU,EAAE,CAAC;YACb,gBAAgB,EAAE,KAAK;YACvB,WAAW,EAAE,CAAC;YACd,KAAK,EAAE,iDAAiD;SACzD,CAAC;IACJ,CAAC;IAED,cAAc,CACZ,QAA4B,EAC5B,OAA2B;QAE3B,OAAO;YACL,aAAa,EACX,QAAQ,CAAC,UAAU,GAAG,CAAC;gBACrB,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU;gBAClE,CAAC,CAAC,CAAC;YACP,kBAAkB,EAChB,QAAQ,CAAC,UAAU,GAAG,CAAC;gBACrB,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU;gBAClE,CAAC,CAAC,CAAC;YACP,aAAa,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/C,mBAAmB,EACjB,QAAQ,CAAC,WAAW,GAAG,CAAC;gBACtB,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;oBAC5C,QAAQ,CAAC,WAAW;gBACtB,CAAC,CAAC,CAAC;SACR,CAAC;IACJ,CAAC;IAED,gBAAgB,CAAC,QAA2B;QAC1C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,UAAkB,EAClB,cAE2E,EAC3E,aAE2E;QAE3E,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,aAAa,UAAU,aAAa,CAAC,CAAC;QAErE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC7B,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC;QACpD,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QAElD,MAAM,QAAQ,GAAuB;YACnC,GAAG,cAAc;YACjB,UAAU;YACV,IAAI,EAAE,UAA2B;YACjC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QAEF,MAAM,OAAO,GAAuB;YAClC,GAAG,aAAa;YAChB,UAAU;YACV,IAAI,EAAE,SAA0B;YAChC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QAEF,MAAM,UAAU,GAAqB;YACnC,aAAa,EACX,QAAQ,CAAC,UAAU,GAAG,CAAC;gBACrB,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU;gBAClE,CAAC,CAAC,CAAC;YACP,kBAAkB,EAChB,QAAQ,CAAC,UAAU,GAAG,CAAC;gBACrB,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU;gBAClE,CAAC,CAAC,CAAC;YACP,aAAa,EACX,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAClC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrC,mBAAmB,EACjB,QAAQ,CAAC,WAAW,GAAG,CAAC;gBACtB,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;oBAC5C,QAAQ,CAAC,WAAW;gBACtB,CAAC,CAAC,CAAC;SACR,CAAC;QAEF,OAAO;YACL,UAAU;YACV,QAAQ;YACR,OAAO;YACP,UAAU;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,WAAqB,EACrB,cAE2E,EAC3E,aAE2E;QAE3E,MAAM,OAAO,GAAgC,EAAE,CAAC;QAChD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC;YAC7E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"default-suite.d.ts","sourceRoot":"","sources":["../src/default-suite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAsB,MAAM,oBAAoB,CAAC;AAqC7E,eAAO,MAAM,uBAAuB,EAAE,cAOrC,CAAC;AAEF,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CAEnE"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { ACCURACY_TESTS } from "./accuracy-queries.js";
|
|
2
|
+
const KEYWORDS_BY_QUERY = {
|
|
3
|
+
"api key auth guard": ["api", "key", "auth", "guard"],
|
|
4
|
+
"post-tool-use hook": ["post-tool-use", "hook", "telemetry"],
|
|
5
|
+
"memory extraction pipeline": ["memory", "extraction", "pipeline"],
|
|
6
|
+
"benchmark test and performance measurement": ["benchmark", "performance", "measurement"],
|
|
7
|
+
"数据库迁移和数据模型定义": ["database", "migration", "schema", "model"],
|
|
8
|
+
};
|
|
9
|
+
const TASK_IDS_BY_QUERY = {
|
|
10
|
+
"api key auth guard": "trace-api-key-auth",
|
|
11
|
+
"post-tool-use hook": "trace-hook-telemetry",
|
|
12
|
+
"memory extraction pipeline": "trace-memory-pipeline",
|
|
13
|
+
"benchmark test and performance measurement": "trace-benchmark-flow",
|
|
14
|
+
"数据库迁移和数据模型定义": "trace-database-schema",
|
|
15
|
+
};
|
|
16
|
+
const tasks = ACCURACY_TESTS
|
|
17
|
+
.filter((test) => test.engine === "codegraph" && test.expectFiles?.length)
|
|
18
|
+
.filter((test) => TASK_IDS_BY_QUERY[test.query])
|
|
19
|
+
.map((test) => ({
|
|
20
|
+
id: TASK_IDS_BY_QUERY[test.query],
|
|
21
|
+
prompt: test.query,
|
|
22
|
+
keywords: KEYWORDS_BY_QUERY[test.query],
|
|
23
|
+
expectedFiles: [...(test.expectFiles ?? [])],
|
|
24
|
+
expectedSymbols: test.expectSymbols ? [...test.expectSymbols] : undefined,
|
|
25
|
+
scoringDimensions: [
|
|
26
|
+
"tokensUsed",
|
|
27
|
+
"filesTouched",
|
|
28
|
+
"correctnessScore",
|
|
29
|
+
"durationMs",
|
|
30
|
+
],
|
|
31
|
+
sourceAccuracyQuery: test.query,
|
|
32
|
+
}));
|
|
33
|
+
export const DEFAULT_BENCHMARK_SUITE = {
|
|
34
|
+
id: "default",
|
|
35
|
+
name: "DevFlow Retrieval Baseline",
|
|
36
|
+
version: "2026.07.1",
|
|
37
|
+
description: "Compares deterministic lexical repository search with DevFlow task-context retrieval on fixed accuracy queries.",
|
|
38
|
+
tasks,
|
|
39
|
+
};
|
|
40
|
+
export function getBenchmarkSuite(id) {
|
|
41
|
+
return id === DEFAULT_BENCHMARK_SUITE.id ? DEFAULT_BENCHMARK_SUITE : null;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=default-suite.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"default-suite.js","sourceRoot":"","sources":["../src/default-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,MAAM,iBAAiB,GAA6B;IAClD,oBAAoB,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC;IACrD,oBAAoB,EAAE,CAAC,eAAe,EAAE,MAAM,EAAE,WAAW,CAAC;IAC5D,4BAA4B,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,CAAC;IAClE,4CAA4C,EAAE,CAAC,WAAW,EAAE,aAAa,EAAE,aAAa,CAAC;IACzF,cAAc,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC;CAC7D,CAAC;AAEF,MAAM,iBAAiB,GAA2B;IAChD,oBAAoB,EAAE,oBAAoB;IAC1C,oBAAoB,EAAE,sBAAsB;IAC5C,4BAA4B,EAAE,uBAAuB;IACrD,4CAA4C,EAAE,sBAAsB;IACpE,cAAc,EAAE,uBAAuB;CACxC,CAAC;AAEF,MAAM,KAAK,GAAyB,cAAc;KAC/C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC;KACzE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;KAC/C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACd,EAAE,EAAE,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;IACjC,MAAM,EAAE,IAAI,CAAC,KAAK;IAClB,QAAQ,EAAE,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;IACvC,aAAa,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IAC5C,eAAe,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;IACzE,iBAAiB,EAAE;QACjB,YAAY;QACZ,cAAc;QACd,kBAAkB;QAClB,YAAY;KACb;IACD,mBAAmB,EAAE,IAAI,CAAC,KAAK;CAChC,CAAC,CAAC,CAAC;AAEN,MAAM,CAAC,MAAM,uBAAuB,GAAmB;IACrD,EAAE,EAAE,SAAS;IACb,IAAI,EAAE,4BAA4B;IAClC,OAAO,EAAE,WAAW;IACpB,WAAW,EACT,iHAAiH;IACnH,KAAK;CACN,CAAC;AAEF,MAAM,UAAU,iBAAiB,CAAC,EAAU;IAC1C,OAAO,EAAE,KAAK,uBAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5E,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,12 @@
|
|
|
1
1
|
export { BenchmarkEngine } from "./benchmark-engine.js";
|
|
2
2
|
export type { BenchmarkOptions, TaskRunner } from "./benchmark-engine.js";
|
|
3
|
+
export { DEFAULT_BENCHMARK_SUITE, getBenchmarkSuite } from "./default-suite.js";
|
|
4
|
+
export { ACCURACY_TESTS } from "./accuracy-queries.js";
|
|
5
|
+
export type { AccuracyTestCase } from "./accuracy-queries.js";
|
|
6
|
+
export { LexicalBenchmarkRunner } from "./lexical-runner.js";
|
|
7
|
+
export type { BenchmarkRetrievalResult } from "./lexical-runner.js";
|
|
8
|
+
export { runBenchmarkSuite } from "./suite-runner.js";
|
|
9
|
+
export type { BenchmarkRetrievalRunner, SuiteRunOptions } from "./suite-runner.js";
|
|
10
|
+
export { renderBenchmarkReport } from "./report-renderer.js";
|
|
11
|
+
export type { BenchmarkReportFormat } from "./report-renderer.js";
|
|
3
12
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,2 +1,7 @@
|
|
|
1
1
|
export { BenchmarkEngine } from "./benchmark-engine.js";
|
|
2
|
+
export { DEFAULT_BENCHMARK_SUITE, getBenchmarkSuite } from "./default-suite.js";
|
|
3
|
+
export { ACCURACY_TESTS } from "./accuracy-queries.js";
|
|
4
|
+
export { LexicalBenchmarkRunner } from "./lexical-runner.js";
|
|
5
|
+
export { runBenchmarkSuite } from "./suite-runner.js";
|
|
6
|
+
export { renderBenchmarkReport } from "./report-renderer.js";
|
|
2
7
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { BenchmarkSuiteTask } from "@devflow-tools/sdk";
|
|
2
|
+
export interface BenchmarkRetrievalResult {
|
|
3
|
+
files: string[];
|
|
4
|
+
symbols: string[];
|
|
5
|
+
tokenCount: number;
|
|
6
|
+
durationMs: number;
|
|
7
|
+
}
|
|
8
|
+
export declare class LexicalBenchmarkRunner {
|
|
9
|
+
private readonly projectRoot;
|
|
10
|
+
private readonly files;
|
|
11
|
+
constructor(projectRoot: string);
|
|
12
|
+
run(task: BenchmarkSuiteTask): Promise<BenchmarkRetrievalResult>;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=lexical-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lexical-runner.d.ts","sourceRoot":"","sources":["../src/lexical-runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAM7D,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB;AASD,qBAAa,sBAAsB;IAGrB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAFxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAW;gBAEJ,WAAW,EAAE,MAAM;IAI1C,GAAG,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,wBAAwB,CAAC;CAqDvE"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { readFileSync, statSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
const MAX_FILE_BYTES = 512 * 1024;
|
|
5
|
+
const MAX_RESULTS = 20;
|
|
6
|
+
const BINARY_EXTENSIONS = /\.(?:png|jpe?g|gif|webp|ico|pdf|zip|gz|tar|woff2?|ttf|eot|sqlite|db|wasm)$/i;
|
|
7
|
+
export class LexicalBenchmarkRunner {
|
|
8
|
+
projectRoot;
|
|
9
|
+
files;
|
|
10
|
+
constructor(projectRoot) {
|
|
11
|
+
this.projectRoot = projectRoot;
|
|
12
|
+
this.files = listRepositoryFiles(projectRoot);
|
|
13
|
+
}
|
|
14
|
+
async run(task) {
|
|
15
|
+
const startedAt = performance.now();
|
|
16
|
+
const terms = task.keywords.map((term) => term.toLowerCase());
|
|
17
|
+
const candidates = [];
|
|
18
|
+
for (const path of this.files) {
|
|
19
|
+
if (BINARY_EXTENSIONS.test(path))
|
|
20
|
+
continue;
|
|
21
|
+
const absolutePath = join(this.projectRoot, path);
|
|
22
|
+
let content;
|
|
23
|
+
try {
|
|
24
|
+
if (statSync(absolutePath).size > MAX_FILE_BYTES)
|
|
25
|
+
continue;
|
|
26
|
+
content = readFileSync(absolutePath, "utf8");
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
const lowerPath = path.toLowerCase();
|
|
32
|
+
const lowerContent = content.toLowerCase();
|
|
33
|
+
let score = 0;
|
|
34
|
+
for (const term of terms) {
|
|
35
|
+
if (lowerPath.includes(term))
|
|
36
|
+
score += 12;
|
|
37
|
+
const first = lowerContent.indexOf(term);
|
|
38
|
+
if (first >= 0) {
|
|
39
|
+
score += 1 + Math.min(8, countOccurrences(lowerContent, term));
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
if (score === 0)
|
|
43
|
+
continue;
|
|
44
|
+
candidates.push({
|
|
45
|
+
path,
|
|
46
|
+
score,
|
|
47
|
+
excerpt: matchingExcerpt(content, terms),
|
|
48
|
+
content,
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
const selected = candidates
|
|
52
|
+
.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path))
|
|
53
|
+
.slice(0, MAX_RESULTS);
|
|
54
|
+
const symbols = (task.expectedSymbols ?? []).filter((symbol) => selected.some((candidate) => candidate.content.includes(symbol)));
|
|
55
|
+
const serializedContext = selected
|
|
56
|
+
.map((candidate) => `${candidate.path}\n${candidate.excerpt}`)
|
|
57
|
+
.join("\n\n");
|
|
58
|
+
return {
|
|
59
|
+
files: selected.map((candidate) => candidate.path),
|
|
60
|
+
symbols,
|
|
61
|
+
tokenCount: estimateTokens(serializedContext),
|
|
62
|
+
durationMs: performance.now() - startedAt,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
function listRepositoryFiles(projectRoot) {
|
|
67
|
+
try {
|
|
68
|
+
return execFileSync("git", ["ls-files", "--cached", "--others", "--exclude-standard"], { cwd: projectRoot, encoding: "utf8", maxBuffer: 16 * 1024 * 1024 })
|
|
69
|
+
.split("\n")
|
|
70
|
+
.map((path) => path.trim())
|
|
71
|
+
.filter(Boolean);
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
75
|
+
throw new Error(`Unable to enumerate benchmark files in ${projectRoot}: ${message}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
function countOccurrences(content, term) {
|
|
79
|
+
let count = 0;
|
|
80
|
+
let position = 0;
|
|
81
|
+
while ((position = content.indexOf(term, position)) >= 0) {
|
|
82
|
+
count += 1;
|
|
83
|
+
position += term.length;
|
|
84
|
+
}
|
|
85
|
+
return count;
|
|
86
|
+
}
|
|
87
|
+
function matchingExcerpt(content, terms) {
|
|
88
|
+
const lines = content.split("\n");
|
|
89
|
+
const matches = lines.filter((line) => {
|
|
90
|
+
const lower = line.toLowerCase();
|
|
91
|
+
return terms.some((term) => lower.includes(term));
|
|
92
|
+
});
|
|
93
|
+
return matches.slice(0, 12).join("\n").slice(0, 2400);
|
|
94
|
+
}
|
|
95
|
+
function estimateTokens(content) {
|
|
96
|
+
return Math.ceil(content.length / 4);
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=lexical-runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lexical-runner.js","sourceRoot":"","sources":["../src/lexical-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,MAAM,cAAc,GAAG,GAAG,GAAG,IAAI,CAAC;AAClC,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,iBAAiB,GAAG,6EAA6E,CAAC;AAgBxG,MAAM,OAAO,sBAAsB;IAGJ;IAFZ,KAAK,CAAW;IAEjC,YAA6B,WAAmB;QAAnB,gBAAW,GAAX,WAAW,CAAQ;QAC9C,IAAI,CAAC,KAAK,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAwB;QAChC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAC9D,MAAM,UAAU,GAAsB,EAAE,CAAC;QAEzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YAClD,IAAI,OAAe,CAAC;YACpB,IAAI,CAAC;gBACH,IAAI,QAAQ,CAAC,YAAY,CAAC,CAAC,IAAI,GAAG,cAAc;oBAAE,SAAS;gBAC3D,OAAO,GAAG,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YAC/C,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAC3C,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,KAAK,IAAI,EAAE,CAAC;gBAC1C,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;oBACf,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,gBAAgB,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC;gBACjE,CAAC;YACH,CAAC;YACD,IAAI,KAAK,KAAK,CAAC;gBAAE,SAAS;YAE1B,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI;gBACJ,KAAK;gBACL,OAAO,EAAE,eAAe,CAAC,OAAO,EAAE,KAAK,CAAC;gBACxC,OAAO;aACR,CAAC,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU;aACxB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aACjE,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QACzB,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAC7D,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CACjE,CAAC;QACF,MAAM,iBAAiB,GAAG,QAAQ;aAC/B,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC;aAC7D,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,OAAO;YACL,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC;YAClD,OAAO;YACP,UAAU,EAAE,cAAc,CAAC,iBAAiB,CAAC;YAC7C,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS;SAC1C,CAAC;IACJ,CAAC;CACF;AAED,SAAS,mBAAmB,CAAC,WAAmB;IAC9C,IAAI,CAAC;QACH,OAAO,YAAY,CACjB,KAAK,EACL,CAAC,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,oBAAoB,CAAC,EAC1D,EAAE,GAAG,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CACpE;aACE,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,0CAA0C,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe,EAAE,IAAY;IACrD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,OAAO,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;QACzD,KAAK,IAAI,CAAC,CAAC;QACX,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC;IAC1B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,OAAe,EAAE,KAAe;IACvD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,cAAc,CAAC,OAAe;IACrC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { BenchmarkSuiteReport } from "@devflow-tools/sdk";
|
|
2
|
+
export type BenchmarkReportFormat = "text" | "json" | "markdown";
|
|
3
|
+
export declare function renderBenchmarkReport(report: BenchmarkSuiteReport, format: BenchmarkReportFormat): string;
|
|
4
|
+
//# sourceMappingURL=report-renderer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report-renderer.d.ts","sourceRoot":"","sources":["../src/report-renderer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE/D,MAAM,MAAM,qBAAqB,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;AAEjE,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,oBAAoB,EAC5B,MAAM,EAAE,qBAAqB,GAC5B,MAAM,CAIR"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
export function renderBenchmarkReport(report, format) {
|
|
2
|
+
if (format === "json")
|
|
3
|
+
return JSON.stringify(report, null, 2);
|
|
4
|
+
if (format === "markdown")
|
|
5
|
+
return renderMarkdown(report);
|
|
6
|
+
return renderText(report);
|
|
7
|
+
}
|
|
8
|
+
function renderText(report) {
|
|
9
|
+
const { baseline, devflow, delta } = report.summary;
|
|
10
|
+
const lines = [
|
|
11
|
+
`DevFlow Benchmark: ${report.suiteName} (${report.suiteVersion})`,
|
|
12
|
+
`Run: ${report.runId}`,
|
|
13
|
+
`Project: ${report.projectRoot}`,
|
|
14
|
+
`Commit: ${report.commit ?? "unknown"}`,
|
|
15
|
+
`Tasks: ${report.tasks.length} x ${report.rounds} round(s)`,
|
|
16
|
+
"",
|
|
17
|
+
"Metric Baseline DevFlow Delta",
|
|
18
|
+
`Tokens ${number(baseline.avgTokensUsed, 1).padStart(8)} ${number(devflow.avgTokensUsed, 1).padStart(8)} ${percent(delta.tokenReductionRate)}`,
|
|
19
|
+
`Files ${number(baseline.avgFilesTouched, 1).padStart(8)} ${number(devflow.avgFilesTouched, 1).padStart(8)} ${percent(delta.fileReductionRate)}`,
|
|
20
|
+
`Correctness ${percent(baseline.avgCorrectnessScore).padStart(8)} ${percent(devflow.avgCorrectnessScore).padStart(8)} ${signedPercent(delta.correctnessLift)}`,
|
|
21
|
+
`Duration ${number(baseline.avgDurationMs, 1).padStart(8)}ms ${number(devflow.avgDurationMs, 1).padStart(8)}ms ${percent(delta.timeSavedRate)}`,
|
|
22
|
+
"",
|
|
23
|
+
report.regressions.length === 0
|
|
24
|
+
? "Regressions >10%: none"
|
|
25
|
+
: `Regressions >10%: ${report.regressions.map((item) => item.message).join("; ")}`,
|
|
26
|
+
];
|
|
27
|
+
return lines.join("\n");
|
|
28
|
+
}
|
|
29
|
+
function renderMarkdown(report) {
|
|
30
|
+
const { baseline, devflow, delta } = report.summary;
|
|
31
|
+
const taskRows = report.tasks.map((task) => `| ${task.taskId} | ${percent(task.baseline.correctnessScore)} | ${percent(task.devflow.correctnessScore)} | ${number(task.baseline.tokensUsed, 0)} | ${number(task.devflow.tokensUsed, 0)} | ${number(task.baseline.durationMs, 1)} | ${number(task.devflow.durationMs, 1)} |`);
|
|
32
|
+
const regressions = report.regressions.length === 0
|
|
33
|
+
? "None."
|
|
34
|
+
: report.regressions.map((item) => `- ${item.message}`).join("\n");
|
|
35
|
+
return [
|
|
36
|
+
`# DevFlow Benchmark Baseline ${new Date(report.createdAt).toISOString().slice(0, 7)}`,
|
|
37
|
+
"",
|
|
38
|
+
`- Suite: \`${report.suiteId}\` (${report.suiteVersion})`,
|
|
39
|
+
`- Run: \`${report.runId}\``,
|
|
40
|
+
`- Project: \`${report.projectRoot}\``,
|
|
41
|
+
`- Commit: \`${report.commit ?? "unknown"}\``,
|
|
42
|
+
`- Rounds: ${report.rounds}`,
|
|
43
|
+
`- Generated: ${new Date(report.createdAt).toISOString()}`,
|
|
44
|
+
"",
|
|
45
|
+
"## Summary",
|
|
46
|
+
"",
|
|
47
|
+
"| Metric | Lexical baseline | DevFlow | Delta |",
|
|
48
|
+
"| --- | ---: | ---: | ---: |",
|
|
49
|
+
`| Tokens used | ${number(baseline.avgTokensUsed, 1)} | ${number(devflow.avgTokensUsed, 1)} | ${percent(delta.tokenReductionRate)} reduction |`,
|
|
50
|
+
`| Files touched | ${number(baseline.avgFilesTouched, 1)} | ${number(devflow.avgFilesTouched, 1)} | ${percent(delta.fileReductionRate)} reduction |`,
|
|
51
|
+
`| Correctness score | ${percent(baseline.avgCorrectnessScore)} | ${percent(devflow.avgCorrectnessScore)} | ${signedPercent(delta.correctnessLift)} |`,
|
|
52
|
+
`| Duration | ${number(baseline.avgDurationMs, 1)} ms | ${number(devflow.avgDurationMs, 1)} ms | ${percent(delta.timeSavedRate)} |`,
|
|
53
|
+
"",
|
|
54
|
+
"## Tasks",
|
|
55
|
+
"",
|
|
56
|
+
"| Task | Baseline correctness | DevFlow correctness | Baseline tokens | DevFlow tokens | Baseline ms | DevFlow ms |",
|
|
57
|
+
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
|
58
|
+
...taskRows,
|
|
59
|
+
"",
|
|
60
|
+
"## Regressions Over 10%",
|
|
61
|
+
"",
|
|
62
|
+
regressions,
|
|
63
|
+
"",
|
|
64
|
+
"The baseline leg is deterministic lexical repository search. The DevFlow leg uses task-context retrieval from the initialized project CodeGraph. No estimated or simulated scores are included.",
|
|
65
|
+
"",
|
|
66
|
+
].join("\n");
|
|
67
|
+
}
|
|
68
|
+
function number(value, digits) {
|
|
69
|
+
return value.toFixed(digits);
|
|
70
|
+
}
|
|
71
|
+
function percent(value) {
|
|
72
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
73
|
+
}
|
|
74
|
+
function signedPercent(value) {
|
|
75
|
+
const rendered = percent(value);
|
|
76
|
+
return value > 0 ? `+${rendered}` : rendered;
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=report-renderer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report-renderer.js","sourceRoot":"","sources":["../src/report-renderer.ts"],"names":[],"mappings":"AAIA,MAAM,UAAU,qBAAqB,CACnC,MAA4B,EAC5B,MAA6B;IAE7B,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,IAAI,MAAM,KAAK,UAAU;QAAE,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;IACzD,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,UAAU,CAAC,MAA4B;IAC9C,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IACpD,MAAM,KAAK,GAAG;QACZ,sBAAsB,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,YAAY,GAAG;QACjE,QAAQ,MAAM,CAAC,KAAK,EAAE;QACtB,YAAY,MAAM,CAAC,WAAW,EAAE;QAChC,WAAW,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE;QACvC,UAAU,MAAM,CAAC,KAAK,CAAC,MAAM,MAAM,MAAM,CAAC,MAAM,WAAW;QAC3D,EAAE;QACF,wDAAwD;QACxD,wBAAwB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE;QACtK,wBAAwB,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE;QACzK,wBAAwB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE;QACjL,wBAAwB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE;QACjK,EAAE;QACF,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;YAC7B,CAAC,CAAC,wBAAwB;YAC1B,CAAC,CAAC,qBAAqB,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACrF,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,cAAc,CAAC,MAA4B;IAClD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACzC,KAAK,IAAI,CAAC,MAAM,MAAM,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAChR,CAAC;IACF,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;QACjD,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAErE,OAAO;QACL,gCAAgC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QACtF,EAAE;QACF,cAAc,MAAM,CAAC,OAAO,OAAO,MAAM,CAAC,YAAY,GAAG;QACzD,YAAY,MAAM,CAAC,KAAK,IAAI;QAC5B,gBAAgB,MAAM,CAAC,WAAW,IAAI;QACtC,eAAe,MAAM,CAAC,MAAM,IAAI,SAAS,IAAI;QAC7C,aAAa,MAAM,CAAC,MAAM,EAAE;QAC5B,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE;QAC1D,EAAE;QACF,YAAY;QACZ,EAAE;QACF,iDAAiD;QACjD,8BAA8B;QAC9B,mBAAmB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,cAAc;QAC/I,qBAAqB,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,cAAc;QACpJ,yBAAyB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC,MAAM,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI;QACtJ,gBAAgB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI;QACnI,EAAE;QACF,UAAU;QACV,EAAE;QACF,qHAAqH;QACrH,mDAAmD;QACnD,GAAG,QAAQ;QACX,EAAE;QACF,yBAAyB;QACzB,EAAE;QACF,WAAW;QACX,EAAE;QACF,iMAAiM;QACjM,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,EAAE,MAAc;IAC3C,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACxC,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAChC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC/C,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { BenchmarkSuite, BenchmarkSuiteReport, BenchmarkSuiteTask } from "@devflow-tools/sdk";
|
|
2
|
+
import { type BenchmarkRetrievalResult } from "./lexical-runner.js";
|
|
3
|
+
export type BenchmarkRetrievalRunner = (task: BenchmarkSuiteTask) => Promise<BenchmarkRetrievalResult>;
|
|
4
|
+
export interface SuiteRunOptions {
|
|
5
|
+
rounds?: number;
|
|
6
|
+
baselineRunner?: BenchmarkRetrievalRunner;
|
|
7
|
+
devflowRunner?: BenchmarkRetrievalRunner;
|
|
8
|
+
}
|
|
9
|
+
export declare function runBenchmarkSuite(projectRoot: string, suite: BenchmarkSuite, options?: SuiteRunOptions): Promise<BenchmarkSuiteReport>;
|
|
10
|
+
//# sourceMappingURL=suite-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"suite-runner.d.ts","sourceRoot":"","sources":["../src/suite-runner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAKV,cAAc,EACd,oBAAoB,EACpB,kBAAkB,EAEnB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAEL,KAAK,wBAAwB,EAC9B,MAAM,qBAAqB,CAAC;AAE7B,MAAM,MAAM,wBAAwB,GAAG,CACrC,IAAI,EAAE,kBAAkB,KACrB,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAEvC,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,wBAAwB,CAAC;IAC1C,aAAa,CAAC,EAAE,wBAAwB,CAAC;CAC1C;AAOD,wBAAsB,iBAAiB,CACrC,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,cAAc,EACrB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,oBAAoB,CAAC,CA0C/B"}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { LexicalBenchmarkRunner, } from "./lexical-runner.js";
|
|
4
|
+
export async function runBenchmarkSuite(projectRoot, suite, options = {}) {
|
|
5
|
+
const rounds = normalizeRounds(options.rounds);
|
|
6
|
+
const lexical = options.baselineRunner ? null : new LexicalBenchmarkRunner(projectRoot);
|
|
7
|
+
const baselineRunner = options.baselineRunner ?? ((task) => lexical.run(task));
|
|
8
|
+
const managedDevflow = options.devflowRunner ? null : await createDevFlowRunner(projectRoot);
|
|
9
|
+
const devflowRunner = options.devflowRunner ?? managedDevflow.run;
|
|
10
|
+
const tasks = [];
|
|
11
|
+
try {
|
|
12
|
+
for (const task of suite.tasks) {
|
|
13
|
+
const baseline = await runLeg("baseline", task, baselineRunner, rounds);
|
|
14
|
+
const devflow = await runLeg("devflow", task, devflowRunner, rounds);
|
|
15
|
+
tasks.push({
|
|
16
|
+
taskId: task.id,
|
|
17
|
+
prompt: task.prompt,
|
|
18
|
+
baseline,
|
|
19
|
+
devflow,
|
|
20
|
+
delta: calculateDelta(baseline, devflow),
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
finally {
|
|
25
|
+
managedDevflow?.close();
|
|
26
|
+
}
|
|
27
|
+
const baseline = aggregate(tasks.map((task) => task.baseline));
|
|
28
|
+
const devflow = aggregate(tasks.map((task) => task.devflow));
|
|
29
|
+
const delta = calculateAggregateDelta(baseline, devflow);
|
|
30
|
+
return {
|
|
31
|
+
runId: `benchmark:${Date.now()}:${randomUUID().slice(0, 8)}`,
|
|
32
|
+
suiteId: suite.id,
|
|
33
|
+
suiteName: suite.name,
|
|
34
|
+
suiteVersion: suite.version,
|
|
35
|
+
projectRoot,
|
|
36
|
+
commit: readCommit(projectRoot),
|
|
37
|
+
createdAt: Date.now(),
|
|
38
|
+
rounds,
|
|
39
|
+
status: "completed",
|
|
40
|
+
tasks,
|
|
41
|
+
summary: { baseline, devflow, delta },
|
|
42
|
+
regressions: detectRegressions(baseline, devflow),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
async function createDevFlowRunner(projectRoot) {
|
|
46
|
+
const packageName = "@devflow-tools/context-engine";
|
|
47
|
+
const { createEngines } = await import(packageName);
|
|
48
|
+
const engines = await createEngines(projectRoot, { warmSymbolVectors: false });
|
|
49
|
+
return {
|
|
50
|
+
run: async (task) => {
|
|
51
|
+
const startedAt = performance.now();
|
|
52
|
+
const result = await engines.context.buildTaskContext(task.prompt, {
|
|
53
|
+
mode: "deep",
|
|
54
|
+
maxTokens: 8000,
|
|
55
|
+
sessionId: `benchmark:${task.id}`,
|
|
56
|
+
});
|
|
57
|
+
return {
|
|
58
|
+
files: [...new Set([
|
|
59
|
+
...result.requiredFiles.files,
|
|
60
|
+
...result.optionalFiles.files,
|
|
61
|
+
])],
|
|
62
|
+
symbols: [...new Set(result.keySymbols.map((symbol) => symbol.name))],
|
|
63
|
+
tokenCount: result.tokenCount,
|
|
64
|
+
durationMs: performance.now() - startedAt,
|
|
65
|
+
};
|
|
66
|
+
},
|
|
67
|
+
close: () => {
|
|
68
|
+
engines.context.close();
|
|
69
|
+
engines.memory.close();
|
|
70
|
+
engines.knowledge.close();
|
|
71
|
+
engines.telemetry.close();
|
|
72
|
+
engines.codegraph.close();
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
async function runLeg(mode, task, runner, rounds) {
|
|
77
|
+
const results = [];
|
|
78
|
+
for (let round = 0; round < rounds; round += 1) {
|
|
79
|
+
results.push(await runner(task));
|
|
80
|
+
}
|
|
81
|
+
const filesTouched = [...new Set(results.flatMap((result) => result.files))];
|
|
82
|
+
const symbols = [...new Set(results.flatMap((result) => result.symbols))];
|
|
83
|
+
const match = scoreExpected(task, filesTouched, symbols);
|
|
84
|
+
return {
|
|
85
|
+
mode,
|
|
86
|
+
tokensUsed: average(results.map((result) => result.tokenCount)),
|
|
87
|
+
filesTouched,
|
|
88
|
+
correctnessScore: match.score,
|
|
89
|
+
durationMs: average(results.map((result) => result.durationMs)),
|
|
90
|
+
matchedFiles: match.matchedFiles,
|
|
91
|
+
matchedSymbols: match.matchedSymbols,
|
|
92
|
+
rounds,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
function scoreExpected(task, files, symbols) {
|
|
96
|
+
const matchedFiles = task.expectedFiles.flatMap((expected) => {
|
|
97
|
+
const match = files.find((file) => normalize(file).includes(normalize(expected)));
|
|
98
|
+
return match ? [match] : [];
|
|
99
|
+
});
|
|
100
|
+
const matchedSymbols = (task.expectedSymbols ?? []).flatMap((expected) => {
|
|
101
|
+
const match = symbols.find((symbol) => normalize(symbol) === normalize(expected));
|
|
102
|
+
return match ? [match] : [];
|
|
103
|
+
});
|
|
104
|
+
const expectedCount = task.expectedFiles.length + (task.expectedSymbols?.length ?? 0);
|
|
105
|
+
const matchedCount = matchedFiles.length + matchedSymbols.length;
|
|
106
|
+
return {
|
|
107
|
+
score: expectedCount > 0 ? matchedCount / expectedCount : 0,
|
|
108
|
+
matchedFiles,
|
|
109
|
+
matchedSymbols,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
function aggregate(legs) {
|
|
113
|
+
return {
|
|
114
|
+
avgTokensUsed: average(legs.map((leg) => leg.tokensUsed)),
|
|
115
|
+
avgFilesTouched: average(legs.map((leg) => leg.filesTouched.length)),
|
|
116
|
+
avgCorrectnessScore: average(legs.map((leg) => leg.correctnessScore)),
|
|
117
|
+
avgDurationMs: average(legs.map((leg) => leg.durationMs)),
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
function calculateDelta(baseline, devflow) {
|
|
121
|
+
return {
|
|
122
|
+
tokenReductionRate: reduction(baseline.tokensUsed, devflow.tokensUsed),
|
|
123
|
+
fileReductionRate: reduction(baseline.filesTouched.length, devflow.filesTouched.length),
|
|
124
|
+
correctnessLift: devflow.correctnessScore - baseline.correctnessScore,
|
|
125
|
+
timeSavedRate: reduction(baseline.durationMs, devflow.durationMs),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
function calculateAggregateDelta(baseline, devflow) {
|
|
129
|
+
return {
|
|
130
|
+
tokenReductionRate: reduction(baseline.avgTokensUsed, devflow.avgTokensUsed),
|
|
131
|
+
fileReductionRate: reduction(baseline.avgFilesTouched, devflow.avgFilesTouched),
|
|
132
|
+
correctnessLift: devflow.avgCorrectnessScore - baseline.avgCorrectnessScore,
|
|
133
|
+
timeSavedRate: reduction(baseline.avgDurationMs, devflow.avgDurationMs),
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
function detectRegressions(baseline, devflow) {
|
|
137
|
+
const threshold = 0.1;
|
|
138
|
+
const candidates = [
|
|
139
|
+
regression("tokensUsed", increase(baseline.avgTokensUsed, devflow.avgTokensUsed), threshold),
|
|
140
|
+
regression("filesTouched", increase(baseline.avgFilesTouched, devflow.avgFilesTouched), threshold),
|
|
141
|
+
regression("durationMs", increase(baseline.avgDurationMs, devflow.avgDurationMs), threshold),
|
|
142
|
+
regression("correctnessScore", reduction(baseline.avgCorrectnessScore, devflow.avgCorrectnessScore), threshold),
|
|
143
|
+
];
|
|
144
|
+
return candidates.filter((item) => item !== null);
|
|
145
|
+
}
|
|
146
|
+
function regression(metric, changeRate, threshold) {
|
|
147
|
+
if (!Number.isFinite(changeRate) || changeRate <= threshold)
|
|
148
|
+
return null;
|
|
149
|
+
return {
|
|
150
|
+
metric,
|
|
151
|
+
changeRate,
|
|
152
|
+
threshold,
|
|
153
|
+
message: `${metric} regressed by ${(changeRate * 100).toFixed(1)}% (threshold ${(threshold * 100).toFixed(0)}%)`,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
function normalize(value) {
|
|
157
|
+
return value.replaceAll("\\", "/").toLowerCase();
|
|
158
|
+
}
|
|
159
|
+
function average(values) {
|
|
160
|
+
if (values.length === 0)
|
|
161
|
+
return 0;
|
|
162
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
163
|
+
}
|
|
164
|
+
function reduction(baseline, candidate) {
|
|
165
|
+
return baseline > 0 ? (baseline - candidate) / baseline : 0;
|
|
166
|
+
}
|
|
167
|
+
function increase(baseline, candidate) {
|
|
168
|
+
return baseline > 0 ? (candidate - baseline) / baseline : candidate > 0 ? 1 : 0;
|
|
169
|
+
}
|
|
170
|
+
function normalizeRounds(rounds) {
|
|
171
|
+
if (rounds === undefined)
|
|
172
|
+
return 1;
|
|
173
|
+
if (!Number.isInteger(rounds) || rounds < 1 || rounds > 20) {
|
|
174
|
+
throw new Error("Benchmark rounds must be an integer between 1 and 20");
|
|
175
|
+
}
|
|
176
|
+
return rounds;
|
|
177
|
+
}
|
|
178
|
+
function readCommit(projectRoot) {
|
|
179
|
+
try {
|
|
180
|
+
return execFileSync("git", ["rev-parse", "HEAD"], {
|
|
181
|
+
cwd: projectRoot,
|
|
182
|
+
encoding: "utf8",
|
|
183
|
+
}).trim();
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
return null;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
//# sourceMappingURL=suite-runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"suite-runner.js","sourceRoot":"","sources":["../src/suite-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAWzC,OAAO,EACL,sBAAsB,GAEvB,MAAM,qBAAqB,CAAC;AAiB7B,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,WAAmB,EACnB,KAAqB,EACrB,UAA2B,EAAE;IAE7B,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,CAAC;IACxF,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAChF,MAAM,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,mBAAmB,CAAC,WAAW,CAAC,CAAC;IAC7F,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,cAAe,CAAC,GAAG,CAAC;IACnE,MAAM,KAAK,GAA8B,EAAE,CAAC;IAE5C,IAAI,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;YACxE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YACrE,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ;gBACR,OAAO;gBACP,KAAK,EAAE,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC;aACzC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;YAAS,CAAC;QACT,cAAc,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,uBAAuB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEzD,OAAO;QACL,KAAK,EAAE,aAAa,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAC5D,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,SAAS,EAAE,KAAK,CAAC,IAAI;QACrB,YAAY,EAAE,KAAK,CAAC,OAAO;QAC3B,WAAW;QACX,MAAM,EAAE,UAAU,CAAC,WAAW,CAAC;QAC/B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,MAAM;QACN,MAAM,EAAE,WAAW;QACnB,KAAK;QACL,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE;QACrC,WAAW,EAAE,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,WAAmB;IAEnB,MAAM,WAAW,GAAG,+BAA+B,CAAC;IACpD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAmD,CAAC;IACtG,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,WAAW,EAAE,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/E,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE;gBACjE,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,IAAI;gBACf,SAAS,EAAE,aAAa,IAAI,CAAC,EAAE,EAAE;aAClC,CAAC,CAAC;YACH,OAAO;gBACL,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC;wBACjB,GAAG,MAAM,CAAC,aAAa,CAAC,KAAK;wBAC7B,GAAG,MAAM,CAAC,aAAa,CAAC,KAAK;qBAC9B,CAAC,CAAC;gBACH,OAAO,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAwB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBACvF,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS;aAC1C,CAAC;QACJ,CAAC;QACD,KAAK,EAAE,GAAG,EAAE;YACV,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAC1B,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAC1B,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,IAAmB,EACnB,IAAwB,EACxB,MAAgC,EAChC,MAAc;IAEd,MAAM,OAAO,GAA+B,EAAE,CAAC;IAC/C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IAEzD,OAAO;QACL,IAAI;QACJ,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC/D,YAAY;QACZ,gBAAgB,EAAE,KAAK,CAAC,KAAK;QAC7B,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC/D,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,IAAwB,EACxB,KAAe,EACf,OAAiB;IAEjB,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAClF,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QACvE,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClF,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IACH,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;IACtF,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC;IACjE,OAAO;QACL,KAAK,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC3D,YAAY;QACZ,cAAc;KACf,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,IAA2B;IAC5C,OAAO;QACL,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACzD,eAAe,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACpE,mBAAmB,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACrE,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;KAC1D,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,QAA6B,EAAE,OAA4B;IACjF,OAAO;QACL,kBAAkB,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC;QACtE,iBAAiB,EAAE,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC;QACvF,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,QAAQ,CAAC,gBAAgB;QACrE,aAAa,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,QAAmC,EACnC,OAAkC;IAElC,OAAO;QACL,kBAAkB,EAAE,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC;QAC5E,iBAAiB,EAAE,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC;QAC/E,eAAe,EAAE,OAAO,CAAC,mBAAmB,GAAG,QAAQ,CAAC,mBAAmB;QAC3E,aAAa,EAAE,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC;KACxE,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CACxB,QAAmC,EACnC,OAAkC;IAElC,MAAM,SAAS,GAAG,GAAG,CAAC;IACtB,MAAM,UAAU,GAAsC;QACpD,UAAU,CAAC,YAAY,EAAE,QAAQ,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;QAC5F,UAAU,CAAC,cAAc,EAAE,QAAQ,CAAC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC,EAAE,SAAS,CAAC;QAClG,UAAU,CAAC,YAAY,EAAE,QAAQ,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;QAC5F,UAAU,CACR,kBAAkB,EAClB,SAAS,CAAC,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,CAAC,EACpE,SAAS,CACV;KACF,CAAC;IACF,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAA+B,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AACjF,CAAC;AAED,SAAS,UAAU,CACjB,MAAqC,EACrC,UAAkB,EAClB,SAAiB;IAEjB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC;IACzE,OAAO;QACL,MAAM;QACN,UAAU;QACV,SAAS;QACT,OAAO,EAAE,GAAG,MAAM,iBAAiB,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;KACjH,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB;IAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACvE,CAAC;AAED,SAAS,SAAS,CAAC,QAAgB,EAAE,SAAiB;IACpD,OAAO,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,QAAQ,CAAC,QAAgB,EAAE,SAAiB;IACnD,OAAO,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,eAAe,CAAC,MAA0B;IACjD,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,EAAE,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,WAAmB;IACrC,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE;YAChD,GAAG,EAAE,WAAW;YAChB,QAAQ,EAAE,MAAM;SACjB,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devflow-tools/benchmark",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
"accuracy": "npx tsx src/run-accuracy-tests.ts"
|
|
19
19
|
},
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"@devflow-tools/context-engine": "0.
|
|
22
|
-
"@devflow-tools/sdk": "0.
|
|
21
|
+
"@devflow-tools/context-engine": "0.16.1",
|
|
22
|
+
"@devflow-tools/sdk": "0.16.1"
|
|
23
23
|
},
|
|
24
24
|
"devDependencies": {
|
|
25
25
|
"typescript": "^5.5.0",
|
|
@@ -28,5 +28,5 @@
|
|
|
28
28
|
"files": [
|
|
29
29
|
"dist"
|
|
30
30
|
],
|
|
31
|
-
"gitHead": "
|
|
31
|
+
"gitHead": "dfaeb66efe9c87c388fbe6f1b706e9fcf087ce6d"
|
|
32
32
|
}
|