autodocs-engine 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analysis-builder.d.ts +2 -2
- package/dist/analysis-builder.js +8 -10
- package/dist/analysis-builder.js.map +1 -1
- package/dist/anti-pattern-detector.d.ts +1 -1
- package/dist/architecture-detector.d.ts +2 -2
- package/dist/architecture-detector.js +9 -18
- package/dist/architecture-detector.js.map +1 -1
- package/dist/ast-parser.js +22 -49
- package/dist/ast-parser.js.map +1 -1
- package/dist/benchmark/code-generator.d.ts +1 -1
- package/dist/benchmark/code-generator.js +14 -1
- package/dist/benchmark/code-generator.js.map +1 -1
- package/dist/benchmark/pr-miner.js +20 -15
- package/dist/benchmark/pr-miner.js.map +1 -1
- package/dist/benchmark/pr-runner.js +21 -17
- package/dist/benchmark/pr-runner.js.map +1 -1
- package/dist/benchmark/pr-scorer.d.ts +1 -1
- package/dist/benchmark/pr-scorer.js +6 -8
- package/dist/benchmark/pr-scorer.js.map +1 -1
- package/dist/benchmark/pr-task-gen.js +4 -4
- package/dist/benchmark/pr-task-gen.js.map +1 -1
- package/dist/benchmark/report.js +13 -10
- package/dist/benchmark/report.js.map +1 -1
- package/dist/benchmark/runner.js +18 -22
- package/dist/benchmark/runner.js.map +1 -1
- package/dist/benchmark/scorer.js +66 -52
- package/dist/benchmark/scorer.js.map +1 -1
- package/dist/benchmark/shuffler.js +1 -1
- package/dist/benchmark/shuffler.js.map +1 -1
- package/dist/benchmark/statistics.js +8 -7
- package/dist/benchmark/statistics.js.map +1 -1
- package/dist/benchmark/task-generator.d.ts +1 -1
- package/dist/benchmark/task-generator.js +87 -46
- package/dist/benchmark/task-generator.js.map +1 -1
- package/dist/benchmark/types.d.ts +1 -1
- package/dist/bin/autodocs-engine.js +10 -10
- package/dist/bin/autodocs-engine.js.map +1 -1
- package/dist/bin/benchmark.js +12 -13
- package/dist/bin/benchmark.js.map +1 -1
- package/dist/bin/check.js +4 -5
- package/dist/bin/check.js.map +1 -1
- package/dist/bin/init.js +7 -4
- package/dist/bin/init.js.map +1 -1
- package/dist/bin/serve.js +6 -4
- package/dist/bin/serve.js.map +1 -1
- package/dist/budget-validator.js.map +1 -1
- package/dist/command-extractor.d.ts +2 -2
- package/dist/command-extractor.js +25 -19
- package/dist/command-extractor.js.map +1 -1
- package/dist/config-analyzer.js +28 -22
- package/dist/config-analyzer.js.map +1 -1
- package/dist/config.js +20 -14
- package/dist/config.js.map +1 -1
- package/dist/contribution-patterns.d.ts +1 -1
- package/dist/contribution-patterns.js +17 -12
- package/dist/contribution-patterns.js.map +1 -1
- package/dist/convention-extractor.d.ts +1 -1
- package/dist/convention-extractor.js +5 -5
- package/dist/convention-extractor.js.map +1 -1
- package/dist/cross-package.d.ts +1 -1
- package/dist/cross-package.js.map +1 -1
- package/dist/dependency-analyzer.js +29 -10
- package/dist/dependency-analyzer.js.map +1 -1
- package/dist/detectors/build-tool.js +1 -1
- package/dist/detectors/build-tool.js.map +1 -1
- package/dist/detectors/data-fetching.js +9 -4
- package/dist/detectors/data-fetching.js.map +1 -1
- package/dist/detectors/database.js +8 -8
- package/dist/detectors/database.js.map +1 -1
- package/dist/detectors/file-naming.js +1 -1
- package/dist/detectors/file-naming.js.map +1 -1
- package/dist/detectors/hook-patterns.js +2 -4
- package/dist/detectors/hook-patterns.js.map +1 -1
- package/dist/detectors/import-ordering.js +41 -12
- package/dist/detectors/import-ordering.js.map +1 -1
- package/dist/detectors/test-framework-ecosystem.js.map +1 -1
- package/dist/detectors/test-patterns.js.map +1 -1
- package/dist/detectors/web-framework.js +11 -3
- package/dist/detectors/web-framework.js.map +1 -1
- package/dist/deterministic-formatter.d.ts +1 -1
- package/dist/deterministic-formatter.js +116 -64
- package/dist/deterministic-formatter.js.map +1 -1
- package/dist/diff-analyzer.d.ts +1 -1
- package/dist/diff-analyzer.js +1 -1
- package/dist/diff-analyzer.js.map +1 -1
- package/dist/example-extractor.js +1 -1
- package/dist/example-extractor.js.map +1 -1
- package/dist/existing-docs.js +8 -18
- package/dist/existing-docs.js.map +1 -1
- package/dist/file-discovery.d.ts +1 -1
- package/dist/file-discovery.js +6 -8
- package/dist/file-discovery.js.map +1 -1
- package/dist/git-history.d.ts +2 -2
- package/dist/git-history.js +15 -21
- package/dist/git-history.js.map +1 -1
- package/dist/impact-classifier.d.ts +1 -1
- package/dist/impact-classifier.js +2 -7
- package/dist/impact-classifier.js.map +1 -1
- package/dist/import-chain.d.ts +1 -1
- package/dist/import-chain.js +2 -4
- package/dist/import-chain.js.map +1 -1
- package/dist/index.d.ts +6 -6
- package/dist/index.js +5 -5
- package/dist/index.js.map +1 -1
- package/dist/inferability.js +80 -30
- package/dist/inferability.js.map +1 -1
- package/dist/llm/adapter.d.ts +1 -1
- package/dist/llm/adapter.js +3 -3
- package/dist/llm/adapter.js.map +1 -1
- package/dist/llm/client.js +1 -1
- package/dist/llm/client.js.map +1 -1
- package/dist/llm/hierarchical.d.ts +1 -1
- package/dist/llm/hierarchical.js +8 -8
- package/dist/llm/hierarchical.js.map +1 -1
- package/dist/llm/serializer.d.ts +1 -1
- package/dist/llm/serializer.js +9 -3
- package/dist/llm/serializer.js.map +1 -1
- package/dist/llm/template-selector.js +1 -1
- package/dist/llm/template-selector.js.map +1 -1
- package/dist/llm-adapter.d.ts +1 -1
- package/dist/llm-adapter.js +1 -1
- package/dist/llm-adapter.js.map +1 -1
- package/dist/mcp/cache.js +5 -3
- package/dist/mcp/cache.js.map +1 -1
- package/dist/mcp/errors.js +1 -1
- package/dist/mcp/errors.js.map +1 -1
- package/dist/mcp/queries.d.ts +2 -2
- package/dist/mcp/queries.js +52 -57
- package/dist/mcp/queries.js.map +1 -1
- package/dist/mcp/server.js +32 -23
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/tools.js +60 -23
- package/dist/mcp/tools.js.map +1 -1
- package/dist/mermaid-generator.js +16 -8
- package/dist/mermaid-generator.js.map +1 -1
- package/dist/meta-tool-detector.js +2 -2
- package/dist/meta-tool-detector.js.map +1 -1
- package/dist/output-validator.d.ts +1 -1
- package/dist/output-validator.js +9 -5
- package/dist/output-validator.js.map +1 -1
- package/dist/pattern-fingerprinter.js +22 -20
- package/dist/pattern-fingerprinter.js.map +1 -1
- package/dist/pipeline.js +33 -27
- package/dist/pipeline.js.map +1 -1
- package/dist/plugin-loader.js +2 -2
- package/dist/plugin-loader.js.map +1 -1
- package/dist/role-inferrer.js +72 -34
- package/dist/role-inferrer.js.map +1 -1
- package/dist/symbol-graph.js +13 -17
- package/dist/symbol-graph.js.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/workflow-rules.d.ts +1 -1
- package/dist/workflow-rules.js +6 -6
- package/dist/workflow-rules.js.map +1 -1
- package/package.json +14 -2
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Uses actual developer commits as ground truth for benchmark tasks.
|
|
3
3
|
// Reads all files at commit time (not HEAD) to avoid time-travel leakage.
|
|
4
4
|
import { execSync } from "node:child_process";
|
|
5
|
-
import {
|
|
5
|
+
import { basename, dirname, extname, resolve } from "node:path";
|
|
6
6
|
// ─── Constants ──────────────────────────────────────────────────────────────
|
|
7
7
|
const DEFAULTS = {
|
|
8
8
|
maxTasks: 30,
|
|
@@ -29,9 +29,13 @@ const SKIP_PATTERNS = [
|
|
|
29
29
|
/\/mocks?\//,
|
|
30
30
|
];
|
|
31
31
|
const SKIP_FILENAMES = new Set([
|
|
32
|
-
"index.ts",
|
|
33
|
-
"
|
|
34
|
-
"
|
|
32
|
+
"index.ts",
|
|
33
|
+
"index.tsx",
|
|
34
|
+
"mod.ts", // barrel files
|
|
35
|
+
"types.ts",
|
|
36
|
+
"constants.ts", // typically type-only
|
|
37
|
+
"package.json",
|
|
38
|
+
"tsconfig.json",
|
|
35
39
|
]);
|
|
36
40
|
// ─── Public API ─────────────────────────────────────────────────────────────
|
|
37
41
|
/**
|
|
@@ -114,9 +118,7 @@ function findAddFileCommits(gitRoot, opts) {
|
|
|
114
118
|
// Get total files changed per commit (including non-additions)
|
|
115
119
|
for (const commit of commits) {
|
|
116
120
|
const stat = gitExec(gitRoot, `git diff-tree --no-commit-id --name-only -r ${commit.sha}`);
|
|
117
|
-
commit.totalFilesChanged = stat
|
|
118
|
-
? stat.split("\n").filter(l => l.trim()).length
|
|
119
|
-
: commit.addedFiles.length;
|
|
121
|
+
commit.totalFilesChanged = stat ? stat.split("\n").filter((l) => l.trim()).length : commit.addedFiles.length;
|
|
120
122
|
}
|
|
121
123
|
return commits;
|
|
122
124
|
}
|
|
@@ -133,11 +135,14 @@ export function readFileAtCommit(gitRoot, sha, filePath) {
|
|
|
133
135
|
*/
|
|
134
136
|
export function listDirAtCommit(gitRoot, sha, dirPath) {
|
|
135
137
|
// Ensure dirPath ends with / for git ls-tree
|
|
136
|
-
const dir = dirPath.endsWith("/") ? dirPath : dirPath
|
|
138
|
+
const dir = dirPath.endsWith("/") ? dirPath : `${dirPath}/`;
|
|
137
139
|
const raw = gitExec(gitRoot, `git ls-tree --name-only ${sha} ${dir}`);
|
|
138
140
|
if (!raw)
|
|
139
141
|
return [];
|
|
140
|
-
return raw
|
|
142
|
+
return raw
|
|
143
|
+
.split("\n")
|
|
144
|
+
.filter((l) => l.trim())
|
|
145
|
+
.map((f) => basename(f));
|
|
141
146
|
}
|
|
142
147
|
// ─── Candidate Processing ───────────────────────────────────────────────────
|
|
143
148
|
function processCandidate(gitRoot, commit, filePath, opts, stats) {
|
|
@@ -158,7 +163,7 @@ function processCandidate(gitRoot, commit, filePath, opts, stats) {
|
|
|
158
163
|
if (SKIP_FILENAMES.has(fileName)) {
|
|
159
164
|
return reason("skip-filename");
|
|
160
165
|
}
|
|
161
|
-
if (SKIP_PATTERNS.some(p => p.test(filePath))) {
|
|
166
|
+
if (SKIP_PATTERNS.some((p) => p.test(filePath))) {
|
|
162
167
|
return reason("skip-pattern");
|
|
163
168
|
}
|
|
164
169
|
// Filter: must be TS/TSX
|
|
@@ -180,10 +185,10 @@ function processCandidate(gitRoot, commit, filePath, opts, stats) {
|
|
|
180
185
|
}
|
|
181
186
|
// Get directory context from PARENT commit (before the file was added)
|
|
182
187
|
const dir = dirname(filePath);
|
|
183
|
-
const parentSha = commit.sha
|
|
188
|
+
const parentSha = `${commit.sha}^`;
|
|
184
189
|
// List siblings at parent commit
|
|
185
190
|
const siblings = listDirAtCommit(gitRoot, parentSha, dir);
|
|
186
|
-
const tsSiblings = siblings.filter(f => TS_EXTENSIONS.test(f) && f !== fileName);
|
|
191
|
+
const tsSiblings = siblings.filter((f) => TS_EXTENSIONS.test(f) && f !== fileName);
|
|
187
192
|
// Filter: must have enough siblings for context
|
|
188
193
|
if (tsSiblings.length < opts.minSiblings) {
|
|
189
194
|
return reason("too-few-siblings");
|
|
@@ -192,7 +197,7 @@ function processCandidate(gitRoot, commit, filePath, opts, stats) {
|
|
|
192
197
|
const siblingFiles = [];
|
|
193
198
|
const sibsToRead = tsSiblings.sort().slice(0, 5);
|
|
194
199
|
for (const sib of sibsToRead) {
|
|
195
|
-
const sibPath = dir
|
|
200
|
+
const sibPath = `${dir}/${sib}`;
|
|
196
201
|
const sibContent = readFileAtCommit(gitRoot, parentSha, sibPath);
|
|
197
202
|
if (sibContent) {
|
|
198
203
|
siblingFiles.push({
|
|
@@ -204,7 +209,7 @@ function processCandidate(gitRoot, commit, filePath, opts, stats) {
|
|
|
204
209
|
// Check for barrel file at parent commit
|
|
205
210
|
let barrelFile;
|
|
206
211
|
for (const barrelName of ["index.ts", "index.tsx"]) {
|
|
207
|
-
const barrelPath = dir
|
|
212
|
+
const barrelPath = `${dir}/${barrelName}`;
|
|
208
213
|
const barrelContent = readFileAtCommit(gitRoot, parentSha, barrelPath);
|
|
209
214
|
if (barrelContent) {
|
|
210
215
|
barrelFile = { path: barrelPath, content: barrelContent };
|
|
@@ -286,7 +291,7 @@ function truncate(content, maxLines) {
|
|
|
286
291
|
const lines = content.split("\n");
|
|
287
292
|
if (lines.length <= maxLines)
|
|
288
293
|
return content;
|
|
289
|
-
return lines.slice(0, maxLines).join("\n")
|
|
294
|
+
return `${lines.slice(0, maxLines).join("\n")}\n// ... truncated`;
|
|
290
295
|
}
|
|
291
296
|
function logStats(stats) {
|
|
292
297
|
console.error(`[pr-miner] Commits scanned: ${stats.totalCommits}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pr-miner.js","sourceRoot":"","sources":["../../src/benchmark/pr-miner.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,qEAAqE;AACrE,0EAA0E;AAE1E,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"pr-miner.js","sourceRoot":"","sources":["../../src/benchmark/pr-miner.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,qEAAqE;AACrE,0EAA0E;AAE1E,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA4ChE,+EAA+E;AAE/E,MAAM,QAAQ,GAA2B;IACvC,QAAQ,EAAE,EAAE;IACZ,UAAU,EAAE,GAAG;IACf,SAAS,EAAE,GAAG;IACd,YAAY,EAAE,EAAE;IAChB,YAAY,EAAE,GAAG;IACjB,WAAW,EAAE,CAAC;IACd,gBAAgB,EAAE,EAAE;IACpB,iBAAiB,EAAE,EAAE;IACrB,OAAO,EAAE,KAAK;CACf,CAAC;AAEF,MAAM,aAAa,GAAG,aAAa,CAAC;AACpC,MAAM,aAAa,GAAG;IACpB,mBAAmB;IACnB,mBAAmB;IACnB,gBAAgB;IAChB,qBAAqB;IACrB,sBAAsB;IACtB,kBAAkB;IAClB,eAAe;IACf,UAAU;IACV,eAAe;IACf,YAAY;CACb,CAAC;AACF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,UAAU;IACV,WAAW;IACX,QAAQ,EAAE,eAAe;IACzB,UAAU;IACV,cAAc,EAAE,sBAAsB;IACtC,cAAc;IACd,eAAe;CAChB,CAAC,CAAC;AAEH,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB,EAAE,UAAwB,EAAE;IACtE,MAAM,IAAI,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,yBAAyB,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,CAAC;QACf,kBAAkB,EAAE,CAAC;QACrB,kBAAkB,EAAE,CAAC;QACrB,oBAAoB,EAAE,CAAC;QACvB,QAAQ,EAAE,CAAC;QACX,aAAa,EAAE,EAAE;KAClB,CAAC;IAEF,4CAA4C;IAC5C,MAAM,UAAU,GAAG,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACrD,KAAK,CAAC,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC;IAEvC,gEAAgE;IAChE,MAAM,UAAU,GAAgB,EAAE,CAAC;IACnC,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;QAChC,KAAK,CAAC,kBAAkB,EAAE,CAAC;QAC3B,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YACtE,IAAI,IAAI;gBAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IACD,KAAK,CAAC,kBAAkB,GAAG,UAAU,CAAC,MAAM,CAAC;IAE7C,mEAAmE;IACnE,MAAM,OAAO,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAC7C,KAAK,CAAC,oBAAoB,GAAG,OAAO,CAAC,MAAM,CAAC;IAE5C,uDAAuD;IACvD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjD,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC;IAEjC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AACpC,CAAC;AAYD;;;GAGG;AACH,SAAS,kBAAkB,CAAC,OAAe,EAAE,IAA4B;IACvE,MAAM,GAAG,GAAG,OAAO,CACjB,OAAO,EACP,kDAAkD;QAChD,8BAA8B;QAC9B,YAAY,IAAI,CAAC,SAAS,aAAa;QACvC,MAAM,IAAI,CAAC,UAAU,GAAG;QACxB,mBAAmB,CACtB,CAAC;IACF,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IAEpB,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,IAAI,OAAO,GAAqB,IAAI,CAAC;IAErC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/B,IAAI,OAAO;gBAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACvC,OAAO,GAAG;gBACR,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;gBACb,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,wBAAwB;gBAC/D,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC7B,iBAAiB,EAAE,CAAC;gBACpB,UAAU,EAAE,EAAE;aACf,CAAC;QACJ,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,EAAE,IAAI,OAAO,EAAE,CAAC;YAClC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IACD,IAAI,OAAO;QAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEnC,+DAA+D;IAC/D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,+CAA+C,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QAC3F,MAAM,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;IAC/G,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,GAAW,EAAE,QAAgB;IAC7E,OAAO,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,QAAQ,EAAE,CAAC,CAAC;AACzD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe,EAAE,GAAW,EAAE,OAAe;IAC3E,6CAA6C;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC;IAC5D,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC;IACtE,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IACpB,OAAO,GAAG;SACP,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7B,CAAC;AAED,+EAA+E;AAE/E,SAAS,gBAAgB,CACvB,OAAe,EACf,MAAiB,EACjB,QAAgB,EAChB,IAA4B,EAC5B,KAAiB;IAEjB,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE;QAC3B,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3D,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,+DAA+D;IAC/D,IAAI,MAAM,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtD,OAAO,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAC1C,CAAC;IAED,mCAAmC;IACnC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAClD,OAAO,MAAM,CAAC,sBAAsB,CAAC,CAAC;IACxC,CAAC;IAED,8CAA8C;IAC9C,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjC,OAAO,MAAM,CAAC,eAAe,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QAChD,OAAO,MAAM,CAAC,cAAc,CAAC,CAAC;IAChC,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClC,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAED,mCAAmC;IACnC,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAChE,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACnC,CAAC;IAED,oBAAoB;IACpB,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAClC,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC;IAC7B,CAAC;IACD,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAClC,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC;IAC7B,CAAC;IAED,uEAAuE;IACvE,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC;IAEnC,iCAAiC;IACjC,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,CAAC;IAEnF,gDAAgD;IAChD,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,OAAO,MAAM,CAAC,kBAAkB,CAAC,CAAC;IACpC,CAAC;IAED,sEAAsE;IACtE,MAAM,YAAY,GAAwC,EAAE,CAAC;IAC7D,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAChC,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QACjE,IAAI,UAAU,EAAE,CAAC;YACf,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC;aACnC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yCAAyC;IACzC,IAAI,UAAyD,CAAC;IAC9D,KAAK,MAAM,UAAU,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC;QACnD,MAAM,UAAU,GAAG,GAAG,GAAG,IAAI,UAAU,EAAE,CAAC;QAC1C,MAAM,aAAa,GAAG,gBAAgB,CAAC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QACvE,IAAI,aAAa,EAAE,CAAC;YAClB,UAAU,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;YAC1D,MAAM;QACR,CAAC;IACH,CAAC;IAED,8DAA8D;IAC9D,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;IAE5D,4CAA4C;IAC5C,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;IAEpF,OAAO;QACL,EAAE;QACF,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,aAAa,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE;QACpC,UAAU,EAAE,MAAM,CAAC,IAAI;QACvB,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,OAAO;YACP,SAAS,EAAE,GAAG;YACd,QAAQ,EAAE,QAAQ;YAClB,SAAS;SACV;QACD,OAAO,EAAE;YACP,YAAY;YACZ,gBAAgB,EAAE,UAAU;YAC5B,UAAU;SACX;KACF,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E;;;GAGG;AACH,SAAS,gBAAgB,CAAC,UAAuB;IAC/C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC;QACvC,MAAM,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QAEhC,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEzC,IAAI,QAAQ,IAAI,CAAC,IAAI,QAAQ,IAAI,CAAC;YAAE,SAAS;QAE7C,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;QACjC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,GAAW;IACjC,sFAAsF;IACtF,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,UAAU,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7D,CAAC;AAED,+EAA+E;AAE/E,SAAS,OAAO,CAAC,GAAW,EAAE,OAAe;IAC3C,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,OAAO,EAAE;YACvB,GAAG;YACH,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;YAC/B,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;SAC5B,CAAC,CAAC,OAAO,EAAE,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC7B,OAAO,OAAO,CAAC,GAAG,EAAE,+BAA+B,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,QAAQ,CAAC,OAAe,EAAE,QAAgB;IACjD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,OAAO,CAAC;IAC7C,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACpE,CAAC;AAED,SAAS,QAAQ,CAAC,KAAiB;IACjC,OAAO,CAAC,KAAK,CAAC,+BAA+B,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC;IACnE,OAAO,CAAC,KAAK,CAAC,mCAAmC,KAAK,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC7E,OAAO,CAAC,KAAK,CAAC,oCAAoC,KAAK,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC9E,OAAO,CAAC,KAAK,CAAC,sCAAsC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAClF,OAAO,CAAC,KAAK,CAAC,wBAAwB,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxD,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9F,OAAO,CAAC,KAAK,CAAC,KAAK,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -1,25 +1,21 @@
|
|
|
1
1
|
// src/benchmark/pr-runner.ts — Orchestrate PR-based benchmark runs
|
|
2
2
|
// Wires together: miner → task-gen → code-gen → scorer → stats → report
|
|
3
|
-
import { resolve, basename as pathBasename } from "node:path";
|
|
4
3
|
import { readFileSync } from "node:fs";
|
|
4
|
+
import { basename as pathBasename, resolve } from "node:path";
|
|
5
5
|
import { callLLMWithRetry } from "../llm/client.js";
|
|
6
6
|
import { parseCodeBlocks } from "./code-generator.js";
|
|
7
7
|
import { mineCommits } from "./pr-miner.js";
|
|
8
|
-
import { generateTaskPrompt } from "./pr-task-gen.js";
|
|
9
8
|
import { scorePROutput } from "./pr-scorer.js";
|
|
10
|
-
import {
|
|
9
|
+
import { generateTaskPrompt } from "./pr-task-gen.js";
|
|
10
|
+
import { bootstrapCI, cohensD, wilcoxonSignedRank } from "./statistics.js";
|
|
11
11
|
// ─── Constants ──────────────────────────────────────────────────────────────
|
|
12
|
-
const CONDITIONS = [
|
|
13
|
-
"treatment",
|
|
14
|
-
"realistic-control",
|
|
15
|
-
"impoverished-control",
|
|
16
|
-
];
|
|
12
|
+
const CONDITIONS = ["treatment", "realistic-control", "impoverished-control"];
|
|
17
13
|
// ─── Public API ─────────────────────────────────────────────────────────────
|
|
18
14
|
/**
|
|
19
15
|
* Run the PR-based benchmark on a repository.
|
|
20
16
|
*/
|
|
21
17
|
export async function runPRBenchmark(options, llmConfig) {
|
|
22
|
-
const { repoPath, mode, agentsMd, verbose, dryRun
|
|
18
|
+
const { repoPath, mode, agentsMd, verbose, dryRun } = options;
|
|
23
19
|
const maxTasks = options.maxTasks ?? (mode === "quick" ? 30 : 50);
|
|
24
20
|
// Phase 1: Mine commits
|
|
25
21
|
if (verbose)
|
|
@@ -181,7 +177,7 @@ function computeSummary(tasks) {
|
|
|
181
177
|
conditionData[c].tokens.push(r.tokensUsed);
|
|
182
178
|
}
|
|
183
179
|
}
|
|
184
|
-
const mean = (arr) => arr.length > 0 ? arr.reduce((a, b) => a + b, 0) / arr.length : 0;
|
|
180
|
+
const mean = (arr) => (arr.length > 0 ? arr.reduce((a, b) => a + b, 0) / arr.length : 0);
|
|
185
181
|
const conditions = {};
|
|
186
182
|
for (const c of CONDITIONS) {
|
|
187
183
|
conditions[c] = {
|
|
@@ -192,7 +188,7 @@ function computeSummary(tasks) {
|
|
|
192
188
|
};
|
|
193
189
|
}
|
|
194
190
|
// Headline: A - B on file placement
|
|
195
|
-
const aScores = conditionData
|
|
191
|
+
const aScores = conditionData.treatment.placements;
|
|
196
192
|
const bScores = conditionData["realistic-control"].placements;
|
|
197
193
|
const cScores = conditionData["impoverished-control"].placements;
|
|
198
194
|
const headlineDelta = Math.round((mean(aScores) - mean(bScores)) * 10) / 10;
|
|
@@ -211,16 +207,22 @@ function computeSummary(tasks) {
|
|
|
211
207
|
const wilcoxon = wilcoxonSignedRank(aScores, bScores);
|
|
212
208
|
stats.pWilcoxon = Math.round(wilcoxon.p * 10000) / 10000;
|
|
213
209
|
}
|
|
214
|
-
catch {
|
|
210
|
+
catch {
|
|
211
|
+
/* not enough data */
|
|
212
|
+
}
|
|
215
213
|
try {
|
|
216
214
|
stats.effectSize = Math.round(cohensD(aScores, bScores) * 100) / 100;
|
|
217
215
|
}
|
|
218
|
-
catch {
|
|
216
|
+
catch {
|
|
217
|
+
/* not enough data */
|
|
218
|
+
}
|
|
219
219
|
try {
|
|
220
220
|
const ci = bootstrapCI(aScores, bScores, 0.05, 10000);
|
|
221
221
|
stats.ci95 = [Math.round(ci[0] * 10) / 10, Math.round(ci[1] * 10) / 10];
|
|
222
222
|
}
|
|
223
|
-
catch {
|
|
223
|
+
catch {
|
|
224
|
+
/* not enough data */
|
|
225
|
+
}
|
|
224
226
|
}
|
|
225
227
|
return {
|
|
226
228
|
headlineDelta,
|
|
@@ -287,8 +289,10 @@ export function generatePRReport(results) {
|
|
|
287
289
|
lines.push("| Condition | Mean Placement | Mean Naming | Mean Barrel | Mean Tokens |");
|
|
288
290
|
lines.push("|-----------|:---:|:---:|:---:|:---:|");
|
|
289
291
|
for (const c of ["treatment", "realistic-control", "impoverished-control"]) {
|
|
290
|
-
const label = c === "treatment"
|
|
291
|
-
|
|
292
|
+
const label = c === "treatment"
|
|
293
|
+
? "A: AGENTS.md + source"
|
|
294
|
+
: c === "realistic-control"
|
|
295
|
+
? "B: Source only"
|
|
292
296
|
: "C: Dir listing only";
|
|
293
297
|
const d = summary.conditions[c];
|
|
294
298
|
lines.push(`| ${label} | ${d.meanPlacement}% | ${d.meanNaming}% | ${d.meanBarrel}% | ${d.meanTokens} |`);
|
|
@@ -301,7 +305,7 @@ export function generatePRReport(results) {
|
|
|
301
305
|
lines.push("|---|------|---------|:---:|:---:|:---:|:---:|");
|
|
302
306
|
for (let i = 0; i < tasks.length; i++) {
|
|
303
307
|
const t = tasks[i];
|
|
304
|
-
const a = t.results
|
|
308
|
+
const a = t.results.treatment.dimensions.filePlacement.score;
|
|
305
309
|
const b = t.results["realistic-control"].dimensions.filePlacement.score;
|
|
306
310
|
const c = t.results["impoverished-control"].dimensions.filePlacement.score;
|
|
307
311
|
const delta = a - b;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pr-runner.js","sourceRoot":"","sources":["../../src/benchmark/pr-runner.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,wEAAwE;AAExE,OAAO,EAAE,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAoB,MAAM,eAAe,CAAC;AAE9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAyD3E,+EAA+E;AAE/E,MAAM,UAAU,GAAkB;IAChC,WAAW;IACX,mBAAmB;IACnB,sBAAsB;CACvB,CAAC;AAEF,+EAA+E;AAE/E;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAA2B,EAC3B,SAAgC;IAEhC,MAAM,EACJ,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAC1C,GAAG,OAAO,CAAC;IACZ,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAElE,wBAAwB;IACxB,IAAI,OAAO;QAAE,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAiB;QAC9B,QAAQ;QACR,UAAU,EAAE,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;QACxC,SAAS,EAAE,GAAG;QACd,OAAO;KACR,CAAC;IACF,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAElF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,kCAAkC,QAAQ,oEAAoE,CAAC,CAAC;IAClI,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,MAAM,QAAQ,CAAC,CAAC;IACnE,CAAC;IAED,wCAAwC;IACxC,MAAM,WAAW,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAEhD,mCAAmC;IACnC,MAAM,WAAW,GAAmB,EAAE,CAAC;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAErD,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,OAAO,GAAkC,EAAE,CAAC;QAElD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,CAAC,SAAS,CAAC,GAAG,gBAAgB,EAAE,CAAC;gBACxC,SAAS;YACX,CAAC;YAED,MAAM,YAAY,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;YACtD,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;YAExE,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,YAAY,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;gBAC7E,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;gBACxD,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;gBACxC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAC3B,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAChE,CAAC;gBAEF,OAAO,CAAC,SAAS,CAAC,GAAG,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YACzE,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;gBACxD,OAAO,CAAC,SAAS,CAAC,GAAG,aAAa,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;YAClF,CAAC;YAED,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;gBAC7B,MAAM,KAAK,GAAG,SAAS,KAAK,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,KAAK,mBAAmB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;gBAC9F,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,eAAe,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,GAAG,CAAC,CAAC;YAC9E,CAAC;QACH,CAAC;QAED,WAAW,CAAC,IAAI,CAAC;YACf,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;YACtC,OAAO,EAAE,OAA6C;SACvD,CAAC,CAAC;IACL,CAAC;IAED,8BAA8B;IAC9B,MAAM,OAAO,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAE5C,OAAO;QACL,IAAI,EAAE;YACJ,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC;YAC3B,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,0BAA0B;YACpD,IAAI;YACJ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,UAAU,EAAE,UAAU;YACtB,QAAQ,EAAE,WAAW,CAAC,MAAM;YAC5B,UAAU;SACX;QACD,OAAO;QACP,KAAK,EAAE,WAAW;KACnB,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E,SAAS,mBAAmB,CAAC,WAAmB;IAC9C,OAAO;QACL,yDAAyD,WAAW,WAAW;QAC/E,gDAAgD;QAChD,+DAA+D;QAC/D,EAAE;QACF,aAAa;QACb,sBAAsB;QACtB,KAAK;QACL,EAAE;QACF,0GAA0G;QAC1G,4FAA4F;QAC5F,kDAAkD;KACnD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CACxB,IAAe,EACf,SAAsB,EACtB,QAAgB,EAChB,UAAkB;IAElB,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,+BAA+B;IAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,6BAA6B;IAC7B,QAAQ,SAAS,EAAE,CAAC;QAClB,KAAK,WAAW;YACd,6CAA6C;YAC7C,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrB,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;gBAChD,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;gBAC5C,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACtB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjB,CAAC;YACD,MAAM;QAER,KAAK,mBAAmB;YACtB,gDAAgD;YAChD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;gBAChD,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;gBAC5C,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACtB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjB,CAAC;YACD,MAAM;QAER,KAAK,sBAAsB;YACzB,sBAAsB;YACtB,MAAM;IACV,CAAC;IAED,qCAAqC;IACrC,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,wBAAwB,GAAG,GAAG,CAAC,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YAC9D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,cAAc;IACd,KAAK,CAAC,IAAI,CAAC,SAAS,UAAU,EAAE,CAAC,CAAC;IAElC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,+EAA+E;AAE/E,SAAS,cAAc,CAAC,KAAqB;IAC3C,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IAEvB,2BAA2B;IAC3B,MAAM,aAAa,GAKd,EAAE,CAAC;IAER,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,aAAa,CAAC,CAAC,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAC9E,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC1B,aAAa,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACnE,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACnE,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/D,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAa,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjG,MAAM,UAAU,GAAG,EAA6B,CAAC;IACjD,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,UAAU,CAAC,CAAC,CAAC,GAAG;YACd,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE;YACtE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE;YAChE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE;YAChE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;SACtD,CAAC;IACJ,CAAC;IAED,oCAAoC;IACpC,MAAM,OAAO,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC;IACtD,MAAM,OAAO,GAAG,aAAa,CAAC,mBAAmB,CAAC,CAAC,UAAU,CAAC;IAC9D,MAAM,OAAO,GAAG,aAAa,CAAC,sBAAsB,CAAC,CAAC,UAAU,CAAC;IAEjE,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;IAC5E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;IAE9E,uDAAuD;IACvD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YAAE,KAAK,EAAE,CAAC;IACvC,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjE,wCAAwC;IACxC,MAAM,KAAK,GAAuB,EAAE,CAAC,EAAE,CAAC;IACxC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACtD,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC;QAEjC,IAAI,CAAC;YACH,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;QACvE,CAAC;QAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC;QAEjC,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YACtD,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1E,CAAC;QAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC;IACnC,CAAC;IAED,OAAO;QACL,aAAa;QACb,eAAe;QACf,QAAQ;QACR,UAAU;QACV,KAAK;KACN,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E,SAAS,iBAAiB,CAAC,QAAgB;IACzC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACjF,OAAO,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACvB,OAAO;QACL,KAAK,EAAE,EAAE;QACT,UAAU,EAAE;YACV,aAAa,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE;YAC9D,gBAAgB,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE;YACjE,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE;SAC9D;QACD,YAAY,EAAE,EAAE;QAChB,UAAU,EAAE,CAAC;QACb,SAAS,EAAE,CAAC;KACb,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA2B;IAC1D,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IACzC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,gCAAgC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC1E,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,SAAS,aAAa,IAAI,CAAC,KAAK,YAAY,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACvF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;IAC5F,KAAK,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;IAC/E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,WAAW;IACX,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,gCAAgC,WAAW,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IAClF,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,gBAAgB,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7G,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,oBAAoB,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACxG,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,qBAAqB,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,QAAQ,wBAAwB,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,wBAAwB;IACxB,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACvC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,0EAA0E,CAAC,CAAC;IACvF,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,mBAAmB,EAAE,sBAAsB,CAAU,EAAE,CAAC;QACpF,MAAM,KAAK,GAAG,CAAC,KAAK,WAAW,CAAC,CAAC,CAAC,uBAAuB;YACvD,CAAC,CAAC,CAAC,KAAK,mBAAmB,CAAC,CAAC,CAAC,gBAAgB;gBAC9C,CAAC,CAAC,qBAAqB,CAAC;QAC1B,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,aAAa,OAAO,CAAC,CAAC,UAAU,OAAO,CAAC,CAAC,UAAU,OAAO,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;IAC3G,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,mBAAmB;IACnB,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAE7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC;QAChE,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC;QACxE,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC;QAC3E,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;QACpB,MAAM,GAAG,GAAG,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC9G,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,cAAc;IACd,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,QAAQ,2CAA2C,CAAC,CAAC;IACjF,KAAK,CAAC,IAAI,CAAC,sBAAsB,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC;IACjE,KAAK,CAAC,IAAI,CAAC,2BAA2B,IAAI,CAAC,UAAU,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;IACxF,KAAK,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;IACjF,KAAK,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;IAChF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxG,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;QACtC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxB,IAAI,GAAG,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IACnC,IAAI,GAAG,GAAG,GAAG;QAAE,OAAO,OAAO,CAAC;IAC9B,IAAI,GAAG,GAAG,GAAG;QAAE,OAAO,QAAQ,CAAC;IAC/B,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
1
|
+
{"version":3,"file":"pr-runner.js","sourceRoot":"","sources":["../../src/benchmark/pr-runner.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,wEAAwE;AAExE,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AA4D3E,+EAA+E;AAE/E,MAAM,UAAU,GAAkB,CAAC,WAAW,EAAE,mBAAmB,EAAE,sBAAsB,CAAC,CAAC;AAE7F,+EAA+E;AAE/E;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAA2B,EAC3B,SAAgC;IAEhC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAElE,wBAAwB;IACxB,IAAI,OAAO;QAAE,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAiB;QAC9B,QAAQ;QACR,UAAU,EAAE,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;QACxC,SAAS,EAAE,GAAG;QACd,OAAO;KACR,CAAC;IACF,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAElF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,oEAAoE,CAC/G,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,MAAM,QAAQ,CAAC,CAAC;IACnE,CAAC;IAED,wCAAwC;IACxC,MAAM,WAAW,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAEhD,mCAAmC;IACnC,MAAM,WAAW,GAAmB,EAAE,CAAC;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAErD,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,OAAO,GAAkC,EAAE,CAAC;QAElD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,CAAC,SAAS,CAAC,GAAG,gBAAgB,EAAE,CAAC;gBACxC,SAAS;YACX,CAAC;YAED,MAAM,YAAY,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;YACtD,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;YAExE,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,YAAY,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;gBAC7E,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;gBACxD,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;gBACxC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAE/F,OAAO,CAAC,SAAS,CAAC,GAAG,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YACzE,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;gBACxD,OAAO,CAAC,SAAS,CAAC,GAAG,aAAa,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;YAClF,CAAC;YAED,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;gBAC7B,MAAM,KAAK,GAAG,SAAS,KAAK,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,KAAK,mBAAmB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;gBAC9F,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,eAAe,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,GAAG,CAAC,CAAC;YAC9E,CAAC;QACH,CAAC;QAED,WAAW,CAAC,IAAI,CAAC;YACf,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;YACtC,OAAO,EAAE,OAA6C;SACvD,CAAC,CAAC;IACL,CAAC;IAED,8BAA8B;IAC9B,MAAM,OAAO,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAE5C,OAAO;QACL,IAAI,EAAE;YACJ,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC;YAC3B,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,0BAA0B;YACpD,IAAI;YACJ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,UAAU,EAAE,UAAU;YACtB,QAAQ,EAAE,WAAW,CAAC,MAAM;YAC5B,UAAU;SACX;QACD,OAAO;QACP,KAAK,EAAE,WAAW;KACnB,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E,SAAS,mBAAmB,CAAC,WAAmB;IAC9C,OAAO;QACL,yDAAyD,WAAW,WAAW;QAC/E,gDAAgD;QAChD,+DAA+D;QAC/D,EAAE;QACF,aAAa;QACb,sBAAsB;QACtB,KAAK;QACL,EAAE;QACF,0GAA0G;QAC1G,4FAA4F;QAC5F,kDAAkD;KACnD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAe,EAAE,SAAsB,EAAE,QAAgB,EAAE,UAAkB;IACtG,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,+BAA+B;IAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,6BAA6B;IAC7B,QAAQ,SAAS,EAAE,CAAC;QAClB,KAAK,WAAW;YACd,6CAA6C;YAC7C,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrB,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;gBAChD,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;gBAC5C,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACtB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjB,CAAC;YACD,MAAM;QAER,KAAK,mBAAmB;YACtB,gDAAgD;YAChD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;gBAChD,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;gBAC5C,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACtB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjB,CAAC;YACD,MAAM;QAER,KAAK,sBAAsB;YACzB,sBAAsB;YACtB,MAAM;IACV,CAAC;IAED,qCAAqC;IACrC,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,wBAAwB,GAAG,GAAG,CAAC,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YAC9D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,cAAc;IACd,KAAK,CAAC,IAAI,CAAC,SAAS,UAAU,EAAE,CAAC,CAAC;IAElC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,+EAA+E;AAE/E,SAAS,cAAc,CAAC,KAAqB;IAC3C,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IAEvB,2BAA2B;IAC3B,MAAM,aAAa,GAQf,EAAE,CAAC;IAEP,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,aAAa,CAAC,CAAC,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAC9E,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC1B,aAAa,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACnE,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACnE,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/D,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAa,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnG,MAAM,UAAU,GAAG,EAA6B,CAAC;IACjD,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,UAAU,CAAC,CAAC,CAAC,GAAG;YACd,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE;YACtE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE;YAChE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE;YAChE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;SACtD,CAAC;IACJ,CAAC;IAED,oCAAoC;IACpC,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,UAAU,CAAC;IACnD,MAAM,OAAO,GAAG,aAAa,CAAC,mBAAmB,CAAC,CAAC,UAAU,CAAC;IAC9D,MAAM,OAAO,GAAG,aAAa,CAAC,sBAAsB,CAAC,CAAC,UAAU,CAAC;IAEjE,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;IAC5E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;IAE9E,uDAAuD;IACvD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YAAE,KAAK,EAAE,CAAC;IACvC,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjE,wCAAwC;IACxC,MAAM,KAAK,GAAuB,EAAE,CAAC,EAAE,CAAC;IACxC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACtD,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;QAED,IAAI,CAAC;YACH,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;QACvE,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YACtD,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1E,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;IACH,CAAC;IAED,OAAO;QACL,aAAa;QACb,eAAe;QACf,QAAQ;QACR,UAAU;QACV,KAAK;KACN,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E,SAAS,iBAAiB,CAAC,QAAgB;IACzC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACjF,OAAO,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACvB,OAAO;QACL,KAAK,EAAE,EAAE;QACT,UAAU,EAAE;YACV,aAAa,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE;YAC9D,gBAAgB,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE;YACjE,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE;SAC9D;QACD,YAAY,EAAE,EAAE;QAChB,UAAU,EAAE,CAAC;QACb,SAAS,EAAE,CAAC;KACb,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA2B;IAC1D,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IACzC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,gCAAgC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC1E,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,SAAS,aAAa,IAAI,CAAC,KAAK,YAAY,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACvF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;IAC5F,KAAK,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;IAC/E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,WAAW;IACX,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,gCAAgC,WAAW,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IAClF,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,gBAAgB,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7G,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,oBAAoB,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACxG,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,qBAAqB,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,QAAQ,wBAAwB,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,wBAAwB;IACxB,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACvC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,0EAA0E,CAAC,CAAC;IACvF,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,mBAAmB,EAAE,sBAAsB,CAAU,EAAE,CAAC;QACpF,MAAM,KAAK,GACT,CAAC,KAAK,WAAW;YACf,CAAC,CAAC,uBAAuB;YACzB,CAAC,CAAC,CAAC,KAAK,mBAAmB;gBACzB,CAAC,CAAC,gBAAgB;gBAClB,CAAC,CAAC,qBAAqB,CAAC;QAC9B,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,aAAa,OAAO,CAAC,CAAC,UAAU,OAAO,CAAC,CAAC,UAAU,OAAO,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;IAC3G,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,mBAAmB;IACnB,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAE7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC;QAC7D,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC;QACxE,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC;QAC3E,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;QACpB,MAAM,GAAG,GAAG,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC9G,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,cAAc;IACd,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,QAAQ,2CAA2C,CAAC,CAAC;IACjF,KAAK,CAAC,IAAI,CAAC,sBAAsB,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC;IACjE,KAAK,CAAC,IAAI,CAAC,2BAA2B,IAAI,CAAC,UAAU,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;IACxF,KAAK,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;IACjF,KAAK,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;IAChF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxG,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;QACtC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxB,IAAI,GAAG,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IACnC,IAAI,GAAG,GAAG,GAAG;QAAE,OAAO,OAAO,CAAC;IAC9B,IAAI,GAAG,GAAG,GAAG;QAAE,OAAO,QAAQ,CAAC;IAC/B,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// src/benchmark/pr-scorer.ts — Score AI output against real commit ground truth
|
|
2
2
|
// Phase 0: File placement accuracy as the primary metric.
|
|
3
3
|
// Designed to be extended with naming, imports, exports in later phases.
|
|
4
|
-
import {
|
|
4
|
+
import { basename, dirname, extname } from "node:path";
|
|
5
5
|
// ─── Public API ─────────────────────────────────────────────────────────────
|
|
6
6
|
/**
|
|
7
7
|
* Score AI-generated files against ground truth from a real commit.
|
|
@@ -23,7 +23,7 @@ export function scorePROutput(files, task, tokensUsed, latencyMs, error) {
|
|
|
23
23
|
};
|
|
24
24
|
}
|
|
25
25
|
// Filter to implementation files (not test, not barrel, not response wrapper)
|
|
26
|
-
const implFiles = files.filter(f => !f.path.includes(".test.") &&
|
|
26
|
+
const implFiles = files.filter((f) => !f.path.includes(".test.") &&
|
|
27
27
|
!f.path.includes(".spec.") &&
|
|
28
28
|
!f.path.includes("__response__") &&
|
|
29
29
|
!isBarrelFile(f.path));
|
|
@@ -36,7 +36,7 @@ export function scorePROutput(files, task, tokensUsed, latencyMs, error) {
|
|
|
36
36
|
return {
|
|
37
37
|
score,
|
|
38
38
|
dimensions: { filePlacement, namingConvention, barrelUpdate },
|
|
39
|
-
filesCreated: files.map(f => f.path),
|
|
39
|
+
filesCreated: files.map((f) => f.path),
|
|
40
40
|
tokensUsed,
|
|
41
41
|
latencyMs,
|
|
42
42
|
};
|
|
@@ -93,7 +93,7 @@ export function pathSimilarity(a, b) {
|
|
|
93
93
|
while (lcp < aParts.length && lcp < bParts.length && aParts[lcp] === bParts[lcp]) {
|
|
94
94
|
lcp++;
|
|
95
95
|
}
|
|
96
|
-
const distance =
|
|
96
|
+
const distance = aParts.length - lcp + (bParts.length - lcp);
|
|
97
97
|
if (distance === 0)
|
|
98
98
|
return 1.0;
|
|
99
99
|
// Exponential decay with distance
|
|
@@ -179,16 +179,14 @@ export function scoreBarrelUpdate(allFiles, task) {
|
|
|
179
179
|
// Find the AI's barrel file
|
|
180
180
|
const barrelPath = task.context.barrelFile.path;
|
|
181
181
|
const barrelName = basename(barrelPath);
|
|
182
|
-
const aiBarrel = allFiles.find(f => f.path === barrelPath ||
|
|
183
|
-
f.path.endsWith("/" + barrelName) ||
|
|
184
|
-
basename(f.path) === barrelName);
|
|
182
|
+
const aiBarrel = allFiles.find((f) => f.path === barrelPath || f.path.endsWith(`/${barrelName}`) || basename(f.path) === barrelName);
|
|
185
183
|
if (!aiBarrel) {
|
|
186
184
|
return { score: 0, detail: "Barrel file exists but AI did not update it", passed: false };
|
|
187
185
|
}
|
|
188
186
|
// Check if the AI added a new export
|
|
189
187
|
const originalExports = extractReExports(task.context.barrelFile.content);
|
|
190
188
|
const aiExports = extractReExports(aiBarrel.content);
|
|
191
|
-
const newExports = aiExports.filter(e => !originalExports.includes(e));
|
|
189
|
+
const newExports = aiExports.filter((e) => !originalExports.includes(e));
|
|
192
190
|
if (newExports.length > 0) {
|
|
193
191
|
return {
|
|
194
192
|
score: 100,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pr-scorer.js","sourceRoot":"","sources":["../../src/benchmark/pr-scorer.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,0DAA0D;AAC1D,yEAAyE;
|
|
1
|
+
{"version":3,"file":"pr-scorer.js","sourceRoot":"","sources":["../../src/benchmark/pr-scorer.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,0DAA0D;AAC1D,yEAAyE;AAEzE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA6BvD,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC3B,KAAsB,EACtB,IAAe,EACf,UAAkB,EAClB,SAAiB,EACjB,KAAc;IAEd,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO;YACL,KAAK,EAAE,CAAC;YACR,UAAU,EAAE;gBACV,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,IAAI,oBAAoB,EAAE,MAAM,EAAE,KAAK,EAAE;gBACjF,gBAAgB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,MAAM,EAAE,KAAK,EAAE;gBAC3E,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,MAAM,EAAE,KAAK,EAAE;aACxE;YACD,YAAY,EAAE,EAAE;YAChB,UAAU;YACV,SAAS;YACT,KAAK;SACN,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAC5B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC1B,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC1B,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;QAChC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CACxB,CAAC;IAEF,MAAM,aAAa,GAAG,kBAAkB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC1D,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAEpD,iDAAiD;IACjD,sCAAsC;IACtC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC;IAElC,OAAO;QACL,KAAK;QACL,UAAU,EAAE,EAAE,aAAa,EAAE,gBAAgB,EAAE,YAAY,EAAE;QAC7D,YAAY,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACtC,UAAU;QACV,SAAS;KACV,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAA0B,EAAE,IAAe;IAC5E,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAClF,CAAC;IAED,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;IAE7D,mDAAmD;IACnD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAElB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAC/C,IAAI,CAAC,GAAG,SAAS,EAAE,CAAC;YAClB,SAAS,GAAG,CAAC,CAAC;YACd,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,SAAS,IAAI,GAAG,CAAC,CAAC,4BAA4B;IAE7D,IAAI,MAAc,CAAC;IACnB,IAAI,SAAS,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,GAAG,YAAY,OAAO,CAAC,QAAQ,CAAC,YAAY,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;IACjF,CAAC;SAAM,IAAI,SAAS,IAAI,GAAG,EAAE,CAAC;QAC5B,MAAM,GAAG,UAAU,OAAO,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,WAAW,CAAC,SAAS,KAAK,KAAK,IAAI,CAAC;IACxF,CAAC;SAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,GAAG,UAAU,OAAO,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,WAAW,CAAC,SAAS,KAAK,KAAK,IAAI,CAAC;IAC7F,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,UAAU,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,cAAc,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;IACxG,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AACnC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,CAAS,EAAE,CAAS;IACjD,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAExB,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE5C,uBAAuB;IACvB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,OAAO,GAAG,GAAG,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACjF,GAAG,EAAE,CAAC;IACR,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IAC7D,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAE/B,kCAAkC;IAClC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB;IAC/C,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC;IAEzC,uCAAuC;IACvC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClC,KAAK,IAAI,GAAG,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,UAAU,CAAC,KAAe;IACjC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,UAAU,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,SAA0B,EAAE,IAAe;IAC/E,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvF,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAE9C,qDAAqD;IACrD,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAElB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,YAAY,KAAK,YAAY,EAAE,CAAC;YAClC,SAAS,GAAG,IAAI,CAAC;YACjB,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;YACrB,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,qBAAqB,YAAY,KAAK,QAAQ,GAAG;YACzD,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3D,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAE9C,OAAO;QACL,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,gCAAgC,YAAY,cAAc,YAAY,EAAE;QAChF,MAAM,EAAE,KAAK;KACd,CAAC;AACJ,CAAC;AAID,SAAS,gBAAgB,CAAC,IAAY;IACpC,IAAI,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC;IACpE,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC,CAAC,8BAA8B;IACtF,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAC;IAC/E,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC;IAC1D,IAAI,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC;IACpE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAyB,EAAE,IAAe;IAC1E,gDAAgD;IAChD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,sBAAsB,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACtE,CAAC;IAED,4BAA4B;IAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;IAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAC5B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,UAAU,EAAE,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,UAAU,CACrG,CAAC;IAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,6CAA6C,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC5F,CAAC;IAED,qCAAqC;IACrC,MAAM,eAAe,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC1E,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,yBAAyB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACxD,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IAED,OAAO;QACL,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,8CAA8C;QACtD,MAAM,EAAE,KAAK;KACd,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,sDAAsD,CAAC;IACrE,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,+EAA+E;AAE/E,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,OAAO,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,QAAQ,CAAC;AAC1E,CAAC"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// src/benchmark/pr-task-gen.ts — Generate task prompts from mined commits
|
|
2
2
|
// Derives fair, deliberately vague prompts that test whether AGENTS.md
|
|
3
3
|
// helps with file placement and convention adherence.
|
|
4
|
-
import ts from "typescript";
|
|
5
4
|
import { basename, extname } from "node:path";
|
|
5
|
+
import ts from "typescript";
|
|
6
6
|
// ─── Public API ─────────────────────────────────────────────────────────────
|
|
7
7
|
/**
|
|
8
8
|
* Generate a task prompt from a mined commit.
|
|
@@ -31,7 +31,7 @@ export function generateTaskPrompt(task, packageName) {
|
|
|
31
31
|
function buildPrompt(description, packageName) {
|
|
32
32
|
// Capitalize first letter
|
|
33
33
|
const desc = description.charAt(0).toUpperCase() + description.slice(1);
|
|
34
|
-
return (`${desc.endsWith(".") ? desc : desc
|
|
34
|
+
return (`${desc.endsWith(".") ? desc : `${desc}.`}\n\n` +
|
|
35
35
|
`This is for the ${packageName} project. ` +
|
|
36
36
|
`Follow the project's conventions for file naming, imports, exports, and code style. ` +
|
|
37
37
|
`Include the implementation file and any necessary updates to barrel/index files.`);
|
|
@@ -53,7 +53,7 @@ export function cleanCommitMessage(message) {
|
|
|
53
53
|
cleaned = cleaned.replace(/\s*\(#\d+\)\s*$/, "");
|
|
54
54
|
cleaned = cleaned.replace(/#\d+/g, "");
|
|
55
55
|
// Strip directory/path hints (e.g., "to src/utils", "in packages/auth")
|
|
56
|
-
cleaned = cleaned.replace(/\s+(to|in|at|from|under)\s+[a-z]+\/[a-zA-Z0-9_
|
|
56
|
+
cleaned = cleaned.replace(/\s+(to|in|at|from|under)\s+[a-z]+\/[a-zA-Z0-9_\-/]+/g, "");
|
|
57
57
|
// Strip filename references (e.g., "add auth-service.ts")
|
|
58
58
|
cleaned = cleaned.replace(/\s+[a-z][a-z0-9\-_]*\.(ts|tsx|js|jsx)\b/gi, "");
|
|
59
59
|
// Strip leading/trailing whitespace and common noise words
|
|
@@ -94,7 +94,7 @@ function extractExports(sourceFile) {
|
|
|
94
94
|
const exports = [];
|
|
95
95
|
ts.forEachChild(sourceFile, (node) => {
|
|
96
96
|
const mods = ts.canHaveModifiers(node) ? ts.getModifiers(node) : undefined;
|
|
97
|
-
const isExported = mods?.some(m => m.kind === ts.SyntaxKind.ExportKeyword);
|
|
97
|
+
const isExported = mods?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword);
|
|
98
98
|
if (!isExported)
|
|
99
99
|
return;
|
|
100
100
|
if (ts.isFunctionDeclaration(node) && node.name) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pr-task-gen.js","sourceRoot":"","sources":["../../src/benchmark/pr-task-gen.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,uEAAuE;AACvE,sDAAsD;AAEtD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"pr-task-gen.js","sourceRoot":"","sources":["../../src/benchmark/pr-task-gen.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,uEAAuE;AACvE,sDAAsD;AAEtD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,MAAM,YAAY,CAAC;AAG5B,+EAA+E;AAE/E;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAe,EAAE,WAAmB;IACrE,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC;IAE9B,2DAA2D;IAC3D,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAI,OAAO,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QACzB,OAAO,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC3C,CAAC;IAED,gDAAgD;IAChD,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5D,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC3C,CAAC;IAED,4CAA4C;IAC5C,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC7C,OAAO,WAAW,CAAC,wBAAwB,SAAS,gBAAgB,EAAE,WAAW,CAAC,CAAC;AACrF,CAAC;AAED,+EAA+E;AAE/E,SAAS,WAAW,CAAC,WAAmB,EAAE,WAAmB;IAC3D,0BAA0B;IAC1B,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAExE,OAAO,CACL,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,MAAM;QAC/C,mBAAmB,WAAW,YAAY;QAC1C,sFAAsF;QACtF,kFAAkF,CACnF,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,IAAI,OAAO,GAAG,OAAO,CAAC;IAEtB,qCAAqC;IACrC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,gFAAgF,EAAE,EAAE,CAAC,CAAC;IAEhH,4BAA4B;IAC5B,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;IACjD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAEvC,wEAAwE;IACxE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,sDAAsD,EAAE,EAAE,CAAC,CAAC;IAEtF,0DAA0D;IAC1D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,2CAA2C,EAAE,EAAE,CAAC,CAAC;IAE3E,2DAA2D;IAC3D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,0CAA0C,EAAE,MAAM,CAAC,CAAC;IAE9E,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,+EAA+E;AAE/E;;GAEG;AACH,SAAS,cAAc,CAAC,OAAe,EAAE,QAAgB;IACvD,MAAM,UAAU,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAExF,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEtC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAE1B,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAEtC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,UAAU;YACb,OAAO,kCAAkC,SAAS,EAAE,CAAC;QACvD,KAAK,OAAO;YACV,OAAO,aAAa,SAAS,QAAQ,CAAC;QACxC,KAAK,WAAW,CAAC;QACjB,KAAK,MAAM;YACT,OAAO,4BAA4B,SAAS,EAAE,CAAC;QACjD,KAAK,OAAO,CAAC;QACb,KAAK,UAAU;YACb,OAAO,SAAS,SAAS,SAAS,CAAC;QACrC;YACE,OAAO,wBAAwB,SAAS,EAAE,CAAC;IAC/C,CAAC;AACH,CAAC;AAOD,SAAS,cAAc,CAAC,UAAyB;IAC/C,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE;QACnC,MAAM,IAAI,GAAG,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3E,MAAM,UAAU,GAAG,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAC7E,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,IAAI,EAAE,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAC3D,CAAC;aAAM,IAAI,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,EAAE,CAAC,sBAAsB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;QAC5D,CAAC;aAAM,IAAI,EAAE,CAAC,sBAAsB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;gBACrD,IAAI,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC/B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,sDAAsD;IACtD,OAAO,IAAI;SACR,OAAO,CAAC,iBAAiB,EAAE,OAAO,CAAC;SACnC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,WAAW,EAAE,CAAC;AACnB,CAAC"}
|
package/dist/benchmark/report.js
CHANGED
|
@@ -21,7 +21,7 @@ export function generateMarkdownReport(results) {
|
|
|
21
21
|
lines.push("| Condition | Mean Score | Pass Rate | Avg Tokens |");
|
|
22
22
|
lines.push("|-----------|-----------|-----------|------------|");
|
|
23
23
|
const conditionLabels = {
|
|
24
|
-
|
|
24
|
+
treatment: "A: AGENTS.md + source",
|
|
25
25
|
"realistic-control": "B: Source only",
|
|
26
26
|
"impoverished-control": "C: Dir listing only",
|
|
27
27
|
"negative-control": "N: Shuffled AGENTS.md",
|
|
@@ -31,7 +31,7 @@ export function generateMarkdownReport(results) {
|
|
|
31
31
|
if (!data)
|
|
32
32
|
continue;
|
|
33
33
|
const label = conditionLabels[cond];
|
|
34
|
-
const score = data.meanScore.toFixed(1)
|
|
34
|
+
const score = `${data.meanScore.toFixed(1)}%`;
|
|
35
35
|
const pass = `${Math.round(data.passRate * summary.tasksRun)}/${summary.tasksRun}`;
|
|
36
36
|
const tokens = Math.round(data.meanTokens).toLocaleString();
|
|
37
37
|
lines.push(`| ${label} | ${score} | ${pass} | ${tokens} |`);
|
|
@@ -40,7 +40,7 @@ export function generateMarkdownReport(results) {
|
|
|
40
40
|
lines.push(`**Headline (A - B):** ${summary.headlineDelta >= 0 ? "+" : ""}${summary.headlineDelta.toFixed(1)}%`);
|
|
41
41
|
lines.push(`**Upper bound (A - C):** ${summary.upperBoundDelta >= 0 ? "+" : ""}${summary.upperBoundDelta.toFixed(1)}%`);
|
|
42
42
|
// Per-task-type breakdown
|
|
43
|
-
const taskTypes = [...new Set(tasks.map(t => t.taskType))];
|
|
43
|
+
const taskTypes = [...new Set(tasks.map((t) => t.taskType))];
|
|
44
44
|
if (taskTypes.length > 1) {
|
|
45
45
|
lines.push("");
|
|
46
46
|
lines.push("## Results by Task Type");
|
|
@@ -48,9 +48,9 @@ export function generateMarkdownReport(results) {
|
|
|
48
48
|
lines.push("| Type | Tasks | A Mean | B Mean | A-B Delta |");
|
|
49
49
|
lines.push("|------|-------|--------|--------|-----------|");
|
|
50
50
|
for (const type of taskTypes) {
|
|
51
|
-
const typeTasks = tasks.filter(t => t.taskType === type);
|
|
52
|
-
const aScores = typeTasks.map(t => t.results
|
|
53
|
-
const bScores = typeTasks.map(t => t.results["realistic-control"]?.score ?? 0);
|
|
51
|
+
const typeTasks = tasks.filter((t) => t.taskType === type);
|
|
52
|
+
const aScores = typeTasks.map((t) => t.results.treatment?.score ?? 0);
|
|
53
|
+
const bScores = typeTasks.map((t) => t.results["realistic-control"]?.score ?? 0);
|
|
54
54
|
const aMean = aScores.reduce((s, v) => s + v, 0) / aScores.length;
|
|
55
55
|
const bMean = bScores.reduce((s, v) => s + v, 0) / bScores.length;
|
|
56
56
|
const delta = aMean - bMean;
|
|
@@ -111,10 +111,13 @@ export function generateMarkdownReport(results) {
|
|
|
111
111
|
const data = summary.conditions[cond];
|
|
112
112
|
if (!data)
|
|
113
113
|
continue;
|
|
114
|
-
const note = cond === "treatment"
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
114
|
+
const note = cond === "treatment"
|
|
115
|
+
? "Includes AGENTS.md context"
|
|
116
|
+
: cond === "realistic-control"
|
|
117
|
+
? "Sibling files only"
|
|
118
|
+
: cond === "impoverished-control"
|
|
119
|
+
? "Minimal context"
|
|
120
|
+
: "Shuffled context";
|
|
118
121
|
lines.push(`| ${conditionLabels[cond]} | ${Math.round(data.meanTokens).toLocaleString()} | ${note} |`);
|
|
119
122
|
}
|
|
120
123
|
lines.push("");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/benchmark/report.ts"],"names":[],"mappings":"AAAA,yEAAyE;
|
|
1
|
+
{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/benchmark/report.ts"],"names":[],"mappings":"AAAA,yEAAyE;AAEzE,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGlD,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAAyB;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IAEzC,KAAK,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACpE,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,SAAS,aAAa,IAAI,CAAC,KAAK,YAAY,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACvF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;IACjG,KAAK,CAAC,IAAI,CAAC,qFAAqF,CAAC,CAAC;IAClG,KAAK,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;IAClE,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;IAEjE,MAAM,eAAe,GAAuC;QAC1D,SAAS,EAAE,uBAAuB;QAClC,mBAAmB,EAAE,gBAAgB;QACrC,sBAAsB,EAAE,qBAAqB;QAC7C,kBAAkB,EAAE,uBAAuB;KAC5C,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9C,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACnF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,MAAM,KAAK,MAAM,IAAI,MAAM,MAAM,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,yBAAyB,OAAO,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACjH,KAAK,CAAC,IAAI,CACR,4BAA4B,OAAO,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC5G,CAAC;IAEF,0BAA0B;IAC1B,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC7D,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QAC7D,KAAK,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QAE7D,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;YAC3D,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;YACtE,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;YACjF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;YAClE,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;YAClE,MAAM,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;YAC5B,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,MAAM,SAAS,CAAC,MAAM,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CACjI,CAAC;QACJ,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACzD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,4BAA4B,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACpE,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,oBAAoB,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,eAAe,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC3G,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC,4BAA4B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;IACpF,CAAC;IAED,qBAAqB;IACrB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,UAAU,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC/B,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YAClE,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,OAAO,GAAG,CAAC,KAAK,MAAM,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,SAAS,WAAW,MAAM,EAAE,CAAC,CAAC;YAE7F,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBACd,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;oBAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;oBAC1C,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC7F,CAAC;YACH,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAChC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IACxD,KAAK,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;IACvD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,IAAI,GACR,IAAI,KAAK,WAAW;YAClB,CAAC,CAAC,4BAA4B;YAC9B,CAAC,CAAC,IAAI,KAAK,mBAAmB;gBAC5B,CAAC,CAAC,oBAAoB;gBACtB,CAAC,CAAC,IAAI,KAAK,sBAAsB;oBAC/B,CAAC,CAAC,iBAAiB;oBACnB,CAAC,CAAC,kBAAkB,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,KAAK,eAAe,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,cAAc,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC;IACzG,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,uFAAuF,CAAC,CAAC;IAEpG,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAyB;IAC1D,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC1C,CAAC"}
|