@codeatlas-ai/benchmark 0.4.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 CodeAtlas Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+
+export {  }

package/dist/index.js

@@ -0,0 +1,296 @@
+// src/index.ts
+import fs2 from "fs";
+import path2 from "path";
+import chalk from "chalk";
+
+// src/datasets.ts
+var EXPRESS_BENCHMARK_DATASET = {
+  name: "expressjs/express",
+  repositoryUrl: "https://github.com/expressjs/express.git",
+  branch: "master",
+  tasks: [
+    {
+      id: "express-routing-matching",
+      query: "Find how routes and middleware are registered and dispatched on the express application",
+      description: "Locate core application routing and middleware initialization",
+      expectedFiles: ["lib/application.js", "lib/express.js"]
+    },
+    {
+      id: "express-app-listen",
+      query: "Where is app.listen defined and how does it start the HTTP server",
+      description: "Locate application initialization and HTTP server bootstrap",
+      expectedFiles: ["lib/application.js", "lib/express.js"]
+    },
+    {
+      id: "express-response-json",
+      query: "Locate the logic that formats and serializes response data to JSON with res.json",
+      description: "Trace res.json and res.send response prototype helpers",
+      expectedFiles: ["lib/response.js"]
+    },
+    {
+      id: "express-request-cookies",
+      query: "Where are request cookies and headers parsed in the request prototype",
+      description: "Locate req.cookies and header parsing helpers in request prototype",
+      expectedFiles: ["lib/request.js"]
+    },
+    {
+      id: "express-view-render",
+      query: "Find template rendering and view engine resolution with app.render and res.render",
+      description: "Trace view engine lookup and rendering pipeline",
+      expectedFiles: ["lib/view.js", "lib/application.js", "lib/response.js"]
+    }
+  ]
+};
+var ALL_DATASETS = [EXPRESS_BENCHMARK_DATASET];
+
+// src/runner.ts
+import fs from "fs";
+import path from "path";
+import { execSync } from "child_process";
+import {
+  AtlasDatabase,
+  runMigrations,
+  FileRepository,
+  SearchRepository,
+  DependencyRepository
+} from "@codeatlas-ai/storage";
+import { Indexer } from "@codeatlas-ai/indexer";
+import { DependencyGraph } from "@codeatlas-ai/graph";
+import { RetrievalEngine } from "@codeatlas-ai/retrieval";
+import { Ranker } from "@codeatlas-ai/ranking";
+import { ContextEngine } from "@codeatlas-ai/context";
+import { RuleEngine } from "@codeatlas-ai/rules";
+import { TokenCounter } from "@codeatlas-ai/token-counter";
+var BenchmarkRunner = class {
+  baseDir;
+  tokenCounter;
+  constructor(baseDir = path.resolve(process.cwd(), ".benchmarks")) {
+    this.baseDir = baseDir;
+    this.tokenCounter = new TokenCounter();
+  }
+  async run(dataset) {
+    const repoDir = this.ensureRepository(dataset);
+    const dbPath = path.join(repoDir, ".atlas", "atlas.db");
+    if (fs.existsSync(path.dirname(dbPath))) {
+      fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+    }
+    fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+    const db = new AtlasDatabase(dbPath);
+    runMigrations(db);
+    const projectRow = db.get("SELECT id FROM projects WHERE root = ?", repoDir);
+    let projectId = projectRow?.id;
+    if (!projectId) {
+      const res = db.run("INSERT INTO projects (name, root) VALUES (?, ?)", dataset.name, repoDir);
+      projectId = Number(res.lastInsertRowid);
+    }
+    console.log(`
+\u23F3 Indexing dataset repository: ${dataset.name}...`);
+    const indexer = new Indexer({
+      root: repoDir,
+      db,
+      projectId,
+      includeTests: false
+    });
+    await indexer.index();
+    const fileRepo = new FileRepository(db);
+    const allFiles = fileRepo.getAll(projectId);
+    const filesByPath = new Map(allFiles.map((f) => [f.relativePath, f]));
+    let totalRepoRawTokens = 0;
+    for (const file of allFiles) {
+      const fullPath = path.join(repoDir, file.relativePath);
+      if (fs.existsSync(fullPath)) {
+        try {
+          const content = fs.readFileSync(fullPath, "utf-8");
+          totalRepoRawTokens += this.tokenCounter.count(content);
+        } catch {
+        }
+      }
+    }
+    const depRepo = new DependencyRepository(db);
+    const deps = depRepo.getAll(projectId);
+    const graph = new DependencyGraph();
+    graph.addEdges(deps);
+    const searchRepo = new SearchRepository(db);
+    const retrieval = new RetrievalEngine(searchRepo, graph, filesByPath);
+    const ruleEngine = new RuleEngine(repoDir);
+    const rules = ruleEngine.discover();
+    const projectMeta = {
+      name: dataset.name,
+      root: repoDir.replace(/\\/g, "/"),
+      languages: ["javascript"],
+      frameworks: [],
+      packageManager: "npm",
+      fileCount: allFiles.length,
+      symbolCount: 0,
+      dependencyCount: deps.length,
+      isMonorepo: false,
+      workspaces: []
+    };
+    const taskResults = [];
+    console.log(`\u{1F9EA} Running ${dataset.tasks.length} benchmark task queries...`);
+    for (const task of dataset.tasks) {
+      const start = performance.now();
+      const retrievalResult = retrieval.retrieve(task.query, 15);
+      const ranker = new Ranker({
+        weights: {
+          lexical_weight: 0.35,
+          symbol_weight: 0.2,
+          path_weight: 0.15,
+          dependency_weight: 0.1,
+          module_weight: 0.1,
+          rule_weight: 0.05,
+          recency_weight: 0.05
+        },
+        queryTerms: retrievalResult.queryTerms
+      });
+      const ranked = ranker.rank(retrievalResult.candidates);
+      const contextEngine = new ContextEngine({
+        tokenBudget: 8e3,
+        defaultMode: "signature",
+        repositoryRoot: repoDir
+      });
+      const pack = contextEngine.build({
+        task: task.query,
+        project: projectMeta,
+        rankedResults: ranked,
+        rules
+      });
+      const durationMs = Math.round(performance.now() - start);
+      const retrievedPaths = pack.files.map((f) => f.relativePath.replace(/\\/g, "/"));
+      const normalizedExpected = task.expectedFiles.map((f) => f.replace(/\\/g, "/"));
+      const hits = normalizedExpected.filter(
+        (exp) => retrievedPaths.some((ret) => ret === exp || ret.endsWith(exp))
+      );
+      const precision = retrievedPaths.length > 0 ? hits.length / retrievedPaths.length : 0;
+      const recall = normalizedExpected.length > 0 ? hits.length / normalizedExpected.length : 0;
+      const atlasTokens = pack.tokenUsage;
+      const tokenSavingsPct = totalRepoRawTokens > 0 ? Math.max(0, Math.round((totalRepoRawTokens - atlasTokens) / totalRepoRawTokens * 100)) : 0;
+      taskResults.push({
+        taskId: task.id,
+        query: task.query,
+        retrievedFiles: retrievedPaths,
+        expectedFiles: normalizedExpected,
+        precision: Math.round(precision * 100) / 100,
+        recall: Math.round(recall * 100) / 100,
+        rawTokens: totalRepoRawTokens,
+        atlasTokens,
+        tokenSavingsPct,
+        durationMs
+      });
+    }
+    db.close();
+    const overallPrecision = Math.round(
+      taskResults.reduce((acc, t) => acc + t.precision, 0) / taskResults.length * 100
+    ) / 100;
+    const overallRecall = Math.round(taskResults.reduce((acc, t) => acc + t.recall, 0) / taskResults.length * 100) / 100;
+    const overallTokenSavingsPct = Math.round(
+      taskResults.reduce((acc, t) => acc + t.tokenSavingsPct, 0) / taskResults.length
+    );
+    const averageLatencyMs = Math.round(
+      taskResults.reduce((acc, t) => acc + t.durationMs, 0) / taskResults.length
+    );
+    return {
+      datasetName: dataset.name,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      taskResults,
+      overallPrecision,
+      overallRecall,
+      overallTokenSavingsPct,
+      averageLatencyMs
+    };
+  }
+  ensureRepository(dataset) {
+    const reposDir = path.join(this.baseDir, "repos");
+    fs.mkdirSync(reposDir, { recursive: true });
+    const safeRepoName = dataset.name.replace(/[^a-zA-Z0-9_-]/g, "_");
+    const targetDir = path.join(reposDir, safeRepoName);
+    if (!fs.existsSync(path.join(targetDir, ".git"))) {
+      console.log(`\u{1F4E6} Cloning benchmark target [${dataset.name}] into ${targetDir}...`);
+      try {
+        execSync(
+          `git clone --depth 1 --branch ${dataset.branch} ${dataset.repositoryUrl} "${targetDir}"`,
+          {
+            stdio: "inherit"
+          }
+        );
+      } catch {
+        execSync(`git clone --depth 1 ${dataset.repositoryUrl} "${targetDir}"`, {
+          stdio: "inherit"
+        });
+      }
+    }
+    return targetDir;
+  }
+};
+
+// src/index.ts
+async function main() {
+  console.log(chalk.bold.cyan("\n\u{1F3C6} CodeAtlas Intelligence Benchmark Suite"));
+  console.log(chalk.dim("Empirical evaluation of AST Retrieval, Token Savings, and Precision\n"));
+  const runner = new BenchmarkRunner();
+  const reports = [];
+  for (const dataset of ALL_DATASETS) {
+    const report = await runner.run(dataset);
+    reports.push(report);
+    console.log(chalk.bold.green(`
+\u{1F4CA} Benchmark Results for: ${report.datasetName}`));
+    console.log("\u2500".repeat(78));
+    console.log(
+      chalk.bold(
+        `${"Task ID".padEnd(28)} ${"Recall".padStart(8)} ${"Precision".padStart(10)} ${"Atlas Tokens".padStart(14)} ${"Savings".padStart(8)} ${"Latency".padStart(8)}`
+      )
+    );
+    console.log("\u2500".repeat(78));
+    for (const task of report.taskResults) {
+      const recallStr = `${Math.round(task.recall * 100)}%`;
+      const precStr = `${Math.round(task.precision * 100)}%`;
+      const tokensStr = `${task.atlasTokens.toLocaleString()} tkn`;
+      const savingsStr = `${task.tokenSavingsPct}%`;
+      const latencyStr = `${task.durationMs}ms`;
+      console.log(
+        `${chalk.cyan(task.taskId.padEnd(28))} ${chalk.green(recallStr.padStart(8))} ${chalk.yellow(precStr.padStart(10))} ${chalk.magenta(tokensStr.padStart(14))} ${chalk.bold.green(savingsStr.padStart(8))} ${chalk.dim(latencyStr.padStart(8))}`
+      );
+    }
+    console.log("\u2500".repeat(78));
+    console.log(
+      chalk.bold(
+        `${"Overall Averages".padEnd(28)} ${chalk.green((Math.round(report.overallRecall * 100) + "%").padStart(8))} ${chalk.yellow((Math.round(report.overallPrecision * 100) + "%").padStart(10))} ${"\u2500".padStart(14)} ${chalk.bold.green((report.overallTokenSavingsPct + "%").padStart(8))} ${chalk.dim((report.averageLatencyMs + "ms").padStart(8))}`
+      )
+    );
+    console.log("\u2500".repeat(78));
+  }
+  const resultsDir = path2.resolve(process.cwd(), ".benchmarks", "results");
+  fs2.mkdirSync(resultsDir, { recursive: true });
+  fs2.writeFileSync(path2.join(resultsDir, "report.json"), JSON.stringify(reports, null, 2), "utf-8");
+  let md = "# \u{1F4CA} CodeAtlas Empirical Benchmark Results\n\n";
+  md += "> Verified automated evaluation measuring context precision, recall, and token reduction.\n\n";
+  for (const report of reports) {
+    md += `## \u{1F4E6} Dataset: \`${report.datasetName}\`
+
+`;
+    md += `| Task | Recall | Precision | Context Tokens | Token Savings | Latency |
+`;
+    md += `| :--- | :--- | :--- | :--- | :--- | :--- |
+`;
+    for (const task of report.taskResults) {
+      md += `| \`${task.taskId}\` | **${Math.round(task.recall * 100)}%** | ${Math.round(task.precision * 100)}% | ${task.atlasTokens.toLocaleString()} | **${task.tokenSavingsPct}%** | ${task.durationMs}ms |
+`;
+    }
+    md += `| **Overall Average** | **${Math.round(report.overallRecall * 100)}%** | **${Math.round(report.overallPrecision * 100)}%** | \u2014 | **${report.overallTokenSavingsPct}%** | **${report.averageLatencyMs}ms** |
+
+`;
+  }
+  fs2.writeFileSync(path2.join(resultsDir, "LEADERBOARD.md"), md, "utf-8");
+  console.log(
+    chalk.bold.green(
+      `
+\u2705 Benchmark complete! Leaderboard written to .benchmarks/results/LEADERBOARD.md
+`
+    )
+  );
+}
+main().catch((err) => {
+  console.error(chalk.red("Benchmark failed:"), err);
+  process.exit(1);
+});
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts","../src/datasets.ts","../src/runner.ts"],"sourcesContent":["import fs from 'node:fs';\nimport path from 'node:path';\nimport chalk from 'chalk';\nimport { ALL_DATASETS } from './datasets.js';\nimport { BenchmarkRunner } from './runner.js';\nimport type { BenchmarkReport } from './types.js';\n\nasync function main() {\n  console.log(chalk.bold.cyan('\\n🏆 CodeAtlas Intelligence Benchmark Suite'));\n  console.log(chalk.dim('Empirical evaluation of AST Retrieval, Token Savings, and Precision\\n'));\n\n  const runner = new BenchmarkRunner();\n  const reports: BenchmarkReport[] = [];\n\n  for (const dataset of ALL_DATASETS) {\n    const report = await runner.run(dataset);\n    reports.push(report);\n\n    console.log(chalk.bold.green(`\\n📊 Benchmark Results for: ${report.datasetName}`));\n    console.log('─'.repeat(78));\n    console.log(\n      chalk.bold(\n        `${'Task ID'.padEnd(28)} ${'Recall'.padStart(8)} ${'Precision'.padStart(10)} ${'Atlas Tokens'.padStart(14)} ${'Savings'.padStart(8)} ${'Latency'.padStart(8)}`,\n      ),\n    );\n    console.log('─'.repeat(78));\n\n    for (const task of report.taskResults) {\n      const recallStr = `${Math.round(task.recall * 100)}%`;\n      const precStr = `${Math.round(task.precision * 100)}%`;\n      const tokensStr = `${task.atlasTokens.toLocaleString()} tkn`;\n      const savingsStr = `${task.tokenSavingsPct}%`;\n      const latencyStr = `${task.durationMs}ms`;\n\n      console.log(\n        `${chalk.cyan(task.taskId.padEnd(28))} ${chalk.green(recallStr.padStart(8))} ${chalk.yellow(precStr.padStart(10))} ${chalk.magenta(tokensStr.padStart(14))} ${chalk.bold.green(savingsStr.padStart(8))} ${chalk.dim(latencyStr.padStart(8))}`,\n      );\n    }\n\n    console.log('─'.repeat(78));\n    console.log(\n      chalk.bold(\n        `${'Overall Averages'.padEnd(28)} ${chalk.green((Math.round(report.overallRecall * 100) + '%').padStart(8))} ${chalk.yellow((Math.round(report.overallPrecision * 100) + '%').padStart(10))} ${'─'.padStart(14)} ${chalk.bold.green((report.overallTokenSavingsPct + '%').padStart(8))} ${chalk.dim((report.averageLatencyMs + 'ms').padStart(8))}`,\n      ),\n    );\n    console.log('─'.repeat(78));\n  }\n\n  // Save report to disk\n  const resultsDir = path.resolve(process.cwd(), '.benchmarks', 'results');\n  fs.mkdirSync(resultsDir, { recursive: true });\n\n  fs.writeFileSync(path.join(resultsDir, 'report.json'), JSON.stringify(reports, null, 2), 'utf-8');\n\n  // Generate Markdown Leaderboard\n  let md = '# 📊 CodeAtlas Empirical Benchmark Results\\n\\n';\n  md +=\n    '> Verified automated evaluation measuring context precision, recall, and token reduction.\\n\\n';\n\n  for (const report of reports) {\n    md += `## 📦 Dataset: \\`${report.datasetName}\\`\\n\\n`;\n    md += `| Task | Recall | Precision | Context Tokens | Token Savings | Latency |\\n`;\n    md += `| :--- | :--- | :--- | :--- | :--- | :--- |\\n`;\n\n    for (const task of report.taskResults) {\n      md += `| \\`${task.taskId}\\` | **${Math.round(task.recall * 100)}%** | ${Math.round(task.precision * 100)}% | ${task.atlasTokens.toLocaleString()} | **${task.tokenSavingsPct}%** | ${task.durationMs}ms |\\n`;\n    }\n\n    md += `| **Overall Average** | **${Math.round(report.overallRecall * 100)}%** | **${Math.round(report.overallPrecision * 100)}%** | — | **${report.overallTokenSavingsPct}%** | **${report.averageLatencyMs}ms** |\\n\\n`;\n  }\n\n  fs.writeFileSync(path.join(resultsDir, 'LEADERBOARD.md'), md, 'utf-8');\n\n  console.log(\n    chalk.bold.green(\n      `\\n✅ Benchmark complete! Leaderboard written to .benchmarks/results/LEADERBOARD.md\\n`,\n    ),\n  );\n}\n\nmain().catch((err) => {\n  console.error(chalk.red('Benchmark failed:'), err);\n  process.exit(1);\n});\n","import type { BenchmarkDataset } from './types.js';\n\nexport const EXPRESS_BENCHMARK_DATASET: BenchmarkDataset = {\n  name: 'expressjs/express',\n  repositoryUrl: 'https://github.com/expressjs/express.git',\n  branch: 'master',\n  tasks: [\n    {\n      id: 'express-routing-matching',\n      query:\n        'Find how routes and middleware are registered and dispatched on the express application',\n      description: 'Locate core application routing and middleware initialization',\n      expectedFiles: ['lib/application.js', 'lib/express.js'],\n    },\n    {\n      id: 'express-app-listen',\n      query: 'Where is app.listen defined and how does it start the HTTP server',\n      description: 'Locate application initialization and HTTP server bootstrap',\n      expectedFiles: ['lib/application.js', 'lib/express.js'],\n    },\n    {\n      id: 'express-response-json',\n      query: 'Locate the logic that formats and serializes response data to JSON with res.json',\n      description: 'Trace res.json and res.send response prototype helpers',\n      expectedFiles: ['lib/response.js'],\n    },\n    {\n      id: 'express-request-cookies',\n      query: 'Where are request cookies and headers parsed in the request prototype',\n      description: 'Locate req.cookies and header parsing helpers in request prototype',\n      expectedFiles: ['lib/request.js'],\n    },\n    {\n      id: 'express-view-render',\n      query: 'Find template rendering and view engine resolution with app.render and res.render',\n      description: 'Trace view engine lookup and rendering pipeline',\n      expectedFiles: ['lib/view.js', 'lib/application.js', 'lib/response.js'],\n    },\n  ],\n};\n\nexport const ALL_DATASETS: BenchmarkDataset[] = [EXPRESS_BENCHMARK_DATASET];\n","import fs from 'node:fs';\nimport path from 'node:path';\nimport { execSync } from 'node:child_process';\nimport type { BenchmarkDataset, BenchmarkReport, TaskResult } from './types.js';\nimport {\n  AtlasDatabase,\n  runMigrations,\n  FileRepository,\n  SearchRepository,\n  DependencyRepository,\n} from '@codeatlas-ai/storage';\nimport { Indexer } from '@codeatlas-ai/indexer';\nimport { DependencyGraph } from '@codeatlas-ai/graph';\nimport { RetrievalEngine } from '@codeatlas-ai/retrieval';\nimport { Ranker } from '@codeatlas-ai/ranking';\nimport { ContextEngine } from '@codeatlas-ai/context';\nimport { RuleEngine } from '@codeatlas-ai/rules';\nimport { TokenCounter } from '@codeatlas-ai/token-counter';\nimport type { FileInfo, ProjectMeta } from '@codeatlas-ai/core';\n\nexport class BenchmarkRunner {\n  private baseDir: string;\n  private tokenCounter: TokenCounter;\n\n  constructor(baseDir: string = path.resolve(process.cwd(), '.benchmarks')) {\n    this.baseDir = baseDir;\n    this.tokenCounter = new TokenCounter();\n  }\n\n  public async run(dataset: BenchmarkDataset): Promise<BenchmarkReport> {\n    const repoDir = this.ensureRepository(dataset);\n    const dbPath = path.join(repoDir, '.atlas', 'atlas.db');\n\n    if (fs.existsSync(path.dirname(dbPath))) {\n      fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });\n    }\n    fs.mkdirSync(path.dirname(dbPath), { recursive: true });\n\n    const db = new AtlasDatabase(dbPath);\n    runMigrations(db);\n\n    const projectRow = db.get<{ id: number }>('SELECT id FROM projects WHERE root = ?', repoDir);\n    let projectId = projectRow?.id;\n    if (!projectId) {\n      const res = db.run('INSERT INTO projects (name, root) VALUES (?, ?)', dataset.name, repoDir);\n      projectId = Number(res.lastInsertRowid);\n    }\n\n    console.log(`\\n⏳ Indexing dataset repository: ${dataset.name}...`);\n    const indexer = new Indexer({\n      root: repoDir,\n      db,\n      projectId: projectId as number,\n      includeTests: false,\n    });\n    await indexer.index();\n\n    const fileRepo = new FileRepository(db);\n    const allFiles = fileRepo.getAll(projectId as number);\n    const filesByPath = new Map<string, FileInfo>(allFiles.map((f) => [f.relativePath, f]));\n\n    // Calculate total raw tokens across the entire codebase\n    let totalRepoRawTokens = 0;\n    for (const file of allFiles) {\n      const fullPath = path.join(repoDir, file.relativePath);\n      if (fs.existsSync(fullPath)) {\n        try {\n          const content = fs.readFileSync(fullPath, 'utf-8');\n          totalRepoRawTokens += this.tokenCounter.count(content);\n        } catch {\n          // ignore binary / unreadable\n        }\n      }\n    }\n\n    const depRepo = new DependencyRepository(db);\n    const deps = depRepo.getAll(projectId as number);\n    const graph = new DependencyGraph();\n    graph.addEdges(deps);\n\n    const searchRepo = new SearchRepository(db);\n    const retrieval = new RetrievalEngine(searchRepo, graph, filesByPath);\n    const ruleEngine = new RuleEngine(repoDir);\n    const rules = ruleEngine.discover();\n\n    const projectMeta: ProjectMeta = {\n      name: dataset.name,\n      root: repoDir.replace(/\\\\/g, '/'),\n      languages: ['javascript'],\n      frameworks: [],\n      packageManager: 'npm',\n      fileCount: allFiles.length,\n      symbolCount: 0,\n      dependencyCount: deps.length,\n      isMonorepo: false,\n      workspaces: [],\n    };\n\n    const taskResults: TaskResult[] = [];\n\n    console.log(`🧪 Running ${dataset.tasks.length} benchmark task queries...`);\n\n    for (const task of dataset.tasks) {\n      const start = performance.now();\n\n      // 1. Retrieve candidates\n      const retrievalResult = retrieval.retrieve(task.query, 15);\n\n      // 2. Rank candidates\n      const ranker = new Ranker({\n        weights: {\n          lexical_weight: 0.35,\n          symbol_weight: 0.20,\n          path_weight: 0.15,\n          dependency_weight: 0.10,\n          module_weight: 0.10,\n          rule_weight: 0.05,\n          recency_weight: 0.05,\n        },\n        queryTerms: retrievalResult.queryTerms,\n      });\n      const ranked = ranker.rank(retrievalResult.candidates);\n\n      // 3. Build Context Pack\n      const contextEngine = new ContextEngine({\n        tokenBudget: 8000,\n        defaultMode: 'signature',\n        repositoryRoot: repoDir,\n      });\n      const pack = contextEngine.build({\n        task: task.query,\n        project: projectMeta,\n        rankedResults: ranked,\n        rules,\n      });\n\n      const durationMs = Math.round(performance.now() - start);\n\n      const retrievedPaths = pack.files.map((f) => f.relativePath.replace(/\\\\/g, '/'));\n      const normalizedExpected = task.expectedFiles.map((f) => f.replace(/\\\\/g, '/'));\n\n      // Calculate precision and recall\n      const hits = normalizedExpected.filter((exp) =>\n        retrievedPaths.some((ret) => ret === exp || ret.endsWith(exp)),\n      );\n\n      const precision = retrievedPaths.length > 0 ? hits.length / retrievedPaths.length : 0;\n      const recall = normalizedExpected.length > 0 ? hits.length / normalizedExpected.length : 0;\n\n      const atlasTokens = pack.tokenUsage;\n      const tokenSavingsPct =\n        totalRepoRawTokens > 0\n          ? Math.max(0, Math.round(((totalRepoRawTokens - atlasTokens) / totalRepoRawTokens) * 100))\n          : 0;\n\n      taskResults.push({\n        taskId: task.id,\n        query: task.query,\n        retrievedFiles: retrievedPaths,\n        expectedFiles: normalizedExpected,\n        precision: Math.round(precision * 100) / 100,\n        recall: Math.round(recall * 100) / 100,\n        rawTokens: totalRepoRawTokens,\n        atlasTokens,\n        tokenSavingsPct,\n        durationMs,\n      });\n    }\n\n    db.close();\n\n    const overallPrecision =\n      Math.round(\n        (taskResults.reduce((acc, t) => acc + t.precision, 0) / taskResults.length) * 100,\n      ) / 100;\n    const overallRecall =\n      Math.round((taskResults.reduce((acc, t) => acc + t.recall, 0) / taskResults.length) * 100) /\n      100;\n    const overallTokenSavingsPct = Math.round(\n      taskResults.reduce((acc, t) => acc + t.tokenSavingsPct, 0) / taskResults.length,\n    );\n    const averageLatencyMs = Math.round(\n      taskResults.reduce((acc, t) => acc + t.durationMs, 0) / taskResults.length,\n    );\n\n    return {\n      datasetName: dataset.name,\n      timestamp: new Date().toISOString(),\n      taskResults,\n      overallPrecision,\n      overallRecall,\n      overallTokenSavingsPct,\n      averageLatencyMs,\n    };\n  }\n\n  private ensureRepository(dataset: BenchmarkDataset): string {\n    const reposDir = path.join(this.baseDir, 'repos');\n    fs.mkdirSync(reposDir, { recursive: true });\n\n    const safeRepoName = dataset.name.replace(/[^a-zA-Z0-9_-]/g, '_');\n    const targetDir = path.join(reposDir, safeRepoName);\n\n    if (!fs.existsSync(path.join(targetDir, '.git'))) {\n      console.log(`📦 Cloning benchmark target [${dataset.name}] into ${targetDir}...`);\n      try {\n        execSync(\n          `git clone --depth 1 --branch ${dataset.branch} ${dataset.repositoryUrl} \"${targetDir}\"`,\n          {\n            stdio: 'inherit',\n          },\n        );\n      } catch {\n        // Fallback without branch if shallow branch fails\n        execSync(`git clone --depth 1 ${dataset.repositoryUrl} \"${targetDir}\"`, {\n          stdio: 'inherit',\n        });\n      }\n    }\n\n    return targetDir;\n  }\n}\n"],"mappings":";AAAA,OAAOA,SAAQ;AACf,OAAOC,WAAU;AACjB,OAAO,WAAW;;;ACAX,IAAM,4BAA8C;AAAA,EACzD,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,EACR,OAAO;AAAA,IACL;AAAA,MACE,IAAI;AAAA,MACJ,OACE;AAAA,MACF,aAAa;AAAA,MACb,eAAe,CAAC,sBAAsB,gBAAgB;AAAA,IACxD;AAAA,IACA;AAAA,MACE,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,aAAa;AAAA,MACb,eAAe,CAAC,sBAAsB,gBAAgB;AAAA,IACxD;AAAA,IACA;AAAA,MACE,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,aAAa;AAAA,MACb,eAAe,CAAC,iBAAiB;AAAA,IACnC;AAAA,IACA;AAAA,MACE,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,aAAa;AAAA,MACb,eAAe,CAAC,gBAAgB;AAAA,IAClC;AAAA,IACA;AAAA,MACE,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,aAAa;AAAA,MACb,eAAe,CAAC,eAAe,sBAAsB,iBAAiB;AAAA,IACxE;AAAA,EACF;AACF;AAEO,IAAM,eAAmC,CAAC,yBAAyB;;;ACzC1E,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,SAAS,gBAAgB;AAEzB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,eAAe;AACxB,SAAS,uBAAuB;AAChC,SAAS,uBAAuB;AAChC,SAAS,cAAc;AACvB,SAAS,qBAAqB;AAC9B,SAAS,kBAAkB;AAC3B,SAAS,oBAAoB;AAGtB,IAAM,kBAAN,MAAsB;AAAA,EACnB;AAAA,EACA;AAAA,EAER,YAAY,UAAkB,KAAK,QAAQ,QAAQ,IAAI,GAAG,aAAa,GAAG;AACxE,SAAK,UAAU;AACf,SAAK,eAAe,IAAI,aAAa;AAAA,EACvC;AAAA,EAEA,MAAa,IAAI,SAAqD;AACpE,UAAM,UAAU,KAAK,iBAAiB,OAAO;AAC7C,UAAM,SAAS,KAAK,KAAK,SAAS,UAAU,UAAU;AAEtD,QAAI,GAAG,WAAW,KAAK,QAAQ,MAAM,CAAC,GAAG;AACvC,SAAG,OAAO,KAAK,QAAQ,MAAM,GAAG,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAAA,IAClE;AACA,OAAG,UAAU,KAAK,QAAQ,MAAM,GAAG,EAAE,WAAW,KAAK,CAAC;AAEtD,UAAM,KAAK,IAAI,cAAc,MAAM;AACnC,kBAAc,EAAE;AAEhB,UAAM,aAAa,GAAG,IAAoB,0CAA0C,OAAO;AAC3F,QAAI,YAAY,YAAY;AAC5B,QAAI,CAAC,WAAW;AACd,YAAM,MAAM,GAAG,IAAI,mDAAmD,QAAQ,MAAM,OAAO;AAC3F,kBAAY,OAAO,IAAI,eAAe;AAAA,IACxC;AAEA,YAAQ,IAAI;AAAA,sCAAoC,QAAQ,IAAI,KAAK;AACjE,UAAM,UAAU,IAAI,QAAQ;AAAA,MAC1B,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,cAAc;AAAA,IAChB,CAAC;AACD,UAAM,QAAQ,MAAM;AAEpB,UAAM,WAAW,IAAI,eAAe,EAAE;AACtC,UAAM,WAAW,SAAS,OAAO,SAAmB;AACpD,UAAM,cAAc,IAAI,IAAsB,SAAS,IAAI,CAAC,MAAM,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC;AAGtF,QAAI,qBAAqB;AACzB,eAAW,QAAQ,UAAU;AAC3B,YAAM,WAAW,KAAK,KAAK,SAAS,KAAK,YAAY;AACrD,UAAI,GAAG,WAAW,QAAQ,GAAG;AAC3B,YAAI;AACF,gBAAM,UAAU,GAAG,aAAa,UAAU,OAAO;AACjD,gCAAsB,KAAK,aAAa,MAAM,OAAO;AAAA,QACvD,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,IACF;AAEA,UAAM,UAAU,IAAI,qBAAqB,EAAE;AAC3C,UAAM,OAAO,QAAQ,OAAO,SAAmB;AAC/C,UAAM,QAAQ,IAAI,gBAAgB;AAClC,UAAM,SAAS,IAAI;AAEnB,UAAM,aAAa,IAAI,iBAAiB,EAAE;AAC1C,UAAM,YAAY,IAAI,gBAAgB,YAAY,OAAO,WAAW;AACpE,UAAM,aAAa,IAAI,WAAW,OAAO;AACzC,UAAM,QAAQ,WAAW,SAAS;AAElC,UAAM,cAA2B;AAAA,MAC/B,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ,QAAQ,OAAO,GAAG;AAAA,MAChC,WAAW,CAAC,YAAY;AAAA,MACxB,YAAY,CAAC;AAAA,MACb,gBAAgB;AAAA,MAChB,WAAW,SAAS;AAAA,MACpB,aAAa;AAAA,MACb,iBAAiB,KAAK;AAAA,MACtB,YAAY;AAAA,MACZ,YAAY,CAAC;AAAA,IACf;AAEA,UAAM,cAA4B,CAAC;AAEnC,YAAQ,IAAI,qBAAc,QAAQ,MAAM,MAAM,4BAA4B;AAE1E,eAAW,QAAQ,QAAQ,OAAO;AAChC,YAAM,QAAQ,YAAY,IAAI;AAG9B,YAAM,kBAAkB,UAAU,SAAS,KAAK,OAAO,EAAE;AAGzD,YAAM,SAAS,IAAI,OAAO;AAAA,QACxB,SAAS;AAAA,UACP,gBAAgB;AAAA,UAChB,eAAe;AAAA,UACf,aAAa;AAAA,UACb,mBAAmB;AAAA,UACnB,eAAe;AAAA,UACf,aAAa;AAAA,UACb,gBAAgB;AAAA,QAClB;AAAA,QACA,YAAY,gBAAgB;AAAA,MAC9B,CAAC;AACD,YAAM,SAAS,OAAO,KAAK,gBAAgB,UAAU;AAGrD,YAAM,gBAAgB,IAAI,cAAc;AAAA,QACtC,aAAa;AAAA,QACb,aAAa;AAAA,QACb,gBAAgB;AAAA,MAClB,CAAC;AACD,YAAM,OAAO,cAAc,MAAM;AAAA,QAC/B,MAAM,KAAK;AAAA,QACX,SAAS;AAAA,QACT,eAAe;AAAA,QACf;AAAA,MACF,CAAC;AAED,YAAM,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;AAEvD,YAAM,iBAAiB,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,aAAa,QAAQ,OAAO,GAAG,CAAC;AAC/E,YAAM,qBAAqB,KAAK,cAAc,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAO,GAAG,CAAC;AAG9E,YAAM,OAAO,mBAAmB;AAAA,QAAO,CAAC,QACtC,eAAe,KAAK,CAAC,QAAQ,QAAQ,OAAO,IAAI,SAAS,GAAG,CAAC;AAAA,MAC/D;AAEA,YAAM,YAAY,eAAe,SAAS,IAAI,KAAK,SAAS,eAAe,SAAS;AACpF,YAAM,SAAS,mBAAmB,SAAS,IAAI,KAAK,SAAS,mBAAmB,SAAS;AAEzF,YAAM,cAAc,KAAK;AACzB,YAAM,kBACJ,qBAAqB,IACjB,KAAK,IAAI,GAAG,KAAK,OAAQ,qBAAqB,eAAe,qBAAsB,GAAG,CAAC,IACvF;AAEN,kBAAY,KAAK;AAAA,QACf,QAAQ,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,gBAAgB;AAAA,QAChB,eAAe;AAAA,QACf,WAAW,KAAK,MAAM,YAAY,GAAG,IAAI;AAAA,QACzC,QAAQ,KAAK,MAAM,SAAS,GAAG,IAAI;AAAA,QACnC,WAAW;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAEA,OAAG,MAAM;AAET,UAAM,mBACJ,KAAK;AAAA,MACF,YAAY,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,YAAY,SAAU;AAAA,IAChF,IAAI;AACN,UAAM,gBACJ,KAAK,MAAO,YAAY,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,QAAQ,CAAC,IAAI,YAAY,SAAU,GAAG,IACzF;AACF,UAAM,yBAAyB,KAAK;AAAA,MAClC,YAAY,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,iBAAiB,CAAC,IAAI,YAAY;AAAA,IAC3E;AACA,UAAM,mBAAmB,KAAK;AAAA,MAC5B,YAAY,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,YAAY,CAAC,IAAI,YAAY;AAAA,IACtE;AAEA,WAAO;AAAA,MACL,aAAa,QAAQ;AAAA,MACrB,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,iBAAiB,SAAmC;AAC1D,UAAM,WAAW,KAAK,KAAK,KAAK,SAAS,OAAO;AAChD,OAAG,UAAU,UAAU,EAAE,WAAW,KAAK,CAAC;AAE1C,UAAM,eAAe,QAAQ,KAAK,QAAQ,mBAAmB,GAAG;AAChE,UAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAElD,QAAI,CAAC,GAAG,WAAW,KAAK,KAAK,WAAW,MAAM,CAAC,GAAG;AAChD,cAAQ,IAAI,uCAAgC,QAAQ,IAAI,UAAU,SAAS,KAAK;AAChF,UAAI;AACF;AAAA,UACE,gCAAgC,QAAQ,MAAM,IAAI,QAAQ,aAAa,KAAK,SAAS;AAAA,UACrF;AAAA,YACE,OAAO;AAAA,UACT;AAAA,QACF;AAAA,MACF,QAAQ;AAEN,iBAAS,uBAAuB,QAAQ,aAAa,KAAK,SAAS,KAAK;AAAA,UACtE,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;;;AFvNA,eAAe,OAAO;AACpB,UAAQ,IAAI,MAAM,KAAK,KAAK,oDAA6C,CAAC;AAC1E,UAAQ,IAAI,MAAM,IAAI,uEAAuE,CAAC;AAE9F,QAAM,SAAS,IAAI,gBAAgB;AACnC,QAAM,UAA6B,CAAC;AAEpC,aAAW,WAAW,cAAc;AAClC,UAAM,SAAS,MAAM,OAAO,IAAI,OAAO;AACvC,YAAQ,KAAK,MAAM;AAEnB,YAAQ,IAAI,MAAM,KAAK,MAAM;AAAA,mCAA+B,OAAO,WAAW,EAAE,CAAC;AACjF,YAAQ,IAAI,SAAI,OAAO,EAAE,CAAC;AAC1B,YAAQ;AAAA,MACN,MAAM;AAAA,QACJ,GAAG,UAAU,OAAO,EAAE,CAAC,IAAI,SAAS,SAAS,CAAC,CAAC,IAAI,YAAY,SAAS,EAAE,CAAC,IAAI,eAAe,SAAS,EAAE,CAAC,IAAI,UAAU,SAAS,CAAC,CAAC,IAAI,UAAU,SAAS,CAAC,CAAC;AAAA,MAC9J;AAAA,IACF;AACA,YAAQ,IAAI,SAAI,OAAO,EAAE,CAAC;AAE1B,eAAW,QAAQ,OAAO,aAAa;AACrC,YAAM,YAAY,GAAG,KAAK,MAAM,KAAK,SAAS,GAAG,CAAC;AAClD,YAAM,UAAU,GAAG,KAAK,MAAM,KAAK,YAAY,GAAG,CAAC;AACnD,YAAM,YAAY,GAAG,KAAK,YAAY,eAAe,CAAC;AACtD,YAAM,aAAa,GAAG,KAAK,eAAe;AAC1C,YAAM,aAAa,GAAG,KAAK,UAAU;AAErC,cAAQ;AAAA,QACN,GAAG,MAAM,KAAK,KAAK,OAAO,OAAO,EAAE,CAAC,CAAC,IAAI,MAAM,MAAM,UAAU,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,OAAO,QAAQ,SAAS,EAAE,CAAC,CAAC,IAAI,MAAM,QAAQ,UAAU,SAAS,EAAE,CAAC,CAAC,IAAI,MAAM,KAAK,MAAM,WAAW,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,IAAI,WAAW,SAAS,CAAC,CAAC,CAAC;AAAA,MAC7O;AAAA,IACF;AAEA,YAAQ,IAAI,SAAI,OAAO,EAAE,CAAC;AAC1B,YAAQ;AAAA,MACN,MAAM;AAAA,QACJ,GAAG,mBAAmB,OAAO,EAAE,CAAC,IAAI,MAAM,OAAO,KAAK,MAAM,OAAO,gBAAgB,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,QAAQ,KAAK,MAAM,OAAO,mBAAmB,GAAG,IAAI,KAAK,SAAS,EAAE,CAAC,CAAC,IAAI,SAAI,SAAS,EAAE,CAAC,IAAI,MAAM,KAAK,OAAO,OAAO,yBAAyB,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,KAAK,OAAO,mBAAmB,MAAM,SAAS,CAAC,CAAC,CAAC;AAAA,MACnV;AAAA,IACF;AACA,YAAQ,IAAI,SAAI,OAAO,EAAE,CAAC;AAAA,EAC5B;AAGA,QAAM,aAAaC,MAAK,QAAQ,QAAQ,IAAI,GAAG,eAAe,SAAS;AACvE,EAAAC,IAAG,UAAU,YAAY,EAAE,WAAW,KAAK,CAAC;AAE5C,EAAAA,IAAG,cAAcD,MAAK,KAAK,YAAY,aAAa,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,GAAG,OAAO;AAGhG,MAAI,KAAK;AACT,QACE;AAEF,aAAW,UAAU,SAAS;AAC5B,UAAM,2BAAoB,OAAO,WAAW;AAAA;AAAA;AAC5C,UAAM;AAAA;AACN,UAAM;AAAA;AAEN,eAAW,QAAQ,OAAO,aAAa;AACrC,YAAM,OAAO,KAAK,MAAM,UAAU,KAAK,MAAM,KAAK,SAAS,GAAG,CAAC,SAAS,KAAK,MAAM,KAAK,YAAY,GAAG,CAAC,OAAO,KAAK,YAAY,eAAe,CAAC,QAAQ,KAAK,eAAe,SAAS,KAAK,UAAU;AAAA;AAAA,IACtM;AAEA,UAAM,6BAA6B,KAAK,MAAM,OAAO,gBAAgB,GAAG,CAAC,WAAW,KAAK,MAAM,OAAO,mBAAmB,GAAG,CAAC,oBAAe,OAAO,sBAAsB,WAAW,OAAO,gBAAgB;AAAA;AAAA;AAAA,EAC7M;AAEA,EAAAC,IAAG,cAAcD,MAAK,KAAK,YAAY,gBAAgB,GAAG,IAAI,OAAO;AAErE,UAAQ;AAAA,IACN,MAAM,KAAK;AAAA,MACT;AAAA;AAAA;AAAA,IACF;AAAA,EACF;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,MAAM,IAAI,mBAAmB,GAAG,GAAG;AACjD,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["fs","path","path","fs"]}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@codeatlas-ai/benchmark",
+  "version": "0.4.0",
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "chalk": "^5.3.0",
+    "@codeatlas-ai/core": "0.4.0",
+    "@codeatlas-ai/shared": "0.4.0",
+    "@codeatlas-ai/storage": "0.4.0",
+    "@codeatlas-ai/indexer": "0.4.0",
+    "@codeatlas-ai/graph": "0.4.0",
+    "@codeatlas-ai/retrieval": "0.4.0",
+    "@codeatlas-ai/ranking": "0.4.0",
+    "@codeatlas-ai/context": "0.4.0",
+    "@codeatlas-ai/rules": "0.4.0",
+    "@codeatlas-ai/token-counter": "0.4.0"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/shditz/codeatlas.git",
+    "directory": "packages/benchmark"
+  },
+  "scripts": {
+    "build": "tsup",
+    "start": "node dist/index.js",
+    "dev": "tsup --watch",
+    "typecheck": "tsc --noEmit",
+    "clean": "rimraf dist"
+  }
+}