codesight 1.3.2 → 1.5.0

package/dist/telemetry.js

@@ -0,0 +1,257 @@
+/**
+ * Token telemetry: measures real before/after token usage by simulating
+ * what an AI agent would do with and without codesight context.
+ *
+ * Approach: for each standard task (explain architecture, add route, review diff),
+ * measure the actual bytes of context that would be consumed.
+ *
+ * "Without codesight": count tokens from the files an AI would need to read
+ * to discover routes, schema, components, config, etc.
+ *
+ * "With codesight": count tokens from the CODESIGHT.md output.
+ */
+import { readFile } from "node:fs/promises";
+import { join, relative } from "node:path";
+function countTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+async function readFileSafe(path) {
+    try {
+        return await readFile(path, "utf-8");
+    }
+    catch {
+        return "";
+    }
+}
+/**
+ * Task 1: "Explain the architecture"
+ * Without codesight: AI reads package.json, scans dirs, reads route files,
+ * schema files, config files, middleware files — typically 15-25 file reads.
+ */
+async function measureExplainArchitecture(root, result, codesightTokens) {
+    const filesToRead = new Set();
+    // AI would read package.json first
+    filesToRead.add(join(root, "package.json"));
+    // Then scan for route files
+    for (const route of result.routes) {
+        filesToRead.add(join(root, route.file));
+    }
+    // Schema files
+    for (const _schema of result.schemas) {
+        // Find the file containing this schema from routes or libs
+        for (const lib of result.libs) {
+            if (lib.file.includes("schema") || lib.file.includes("model") || lib.file.includes("db")) {
+                filesToRead.add(join(root, lib.file));
+            }
+        }
+    }
+    // Config files
+    for (const cf of result.config.configFiles) {
+        filesToRead.add(join(root, cf));
+    }
+    // Middleware files
+    for (const mw of result.middleware) {
+        filesToRead.add(join(root, mw.file));
+    }
+    // Hot files (AI would discover these during exploration)
+    for (const hf of result.graph.hotFiles.slice(0, 10)) {
+        filesToRead.add(join(root, hf.file));
+    }
+    // Read all files and count tokens
+    let totalTokens = 0;
+    const readFiles = [];
+    for (const f of filesToRead) {
+        const content = await readFileSafe(f);
+        if (content) {
+            totalTokens += countTokens(content);
+            readFiles.push(relative(root, f));
+        }
+    }
+    // Add overhead for glob/grep tool calls (each costs ~50-100 tokens for command + results)
+    const toolCalls = Math.max(10, Math.ceil(filesToRead.size * 0.8));
+    totalTokens += toolCalls * 75; // average 75 tokens per tool call overhead
+    const reduction = totalTokens > 0 ? Math.round((totalTokens / codesightTokens) * 10) / 10 : 1;
+    return {
+        name: "Explain architecture",
+        description: "Understand project stack, routes, schema, and dependencies",
+        filesRead: readFiles,
+        toolCalls,
+        tokensWithout: totalTokens,
+        tokensWith: codesightTokens,
+        reduction,
+    };
+}
+/**
+ * Task 2: "Add a new API route"
+ * Without codesight: AI needs to find existing routes to match patterns,
+ * read schema for related models, check middleware, check config.
+ */
+async function measureAddRoute(root, result, codesightTokens) {
+    const filesToRead = new Set();
+    // AI would grep for existing route patterns — reads 3-5 route files
+    const routeFiles = [...new Set(result.routes.map((r) => r.file))];
+    for (const f of routeFiles.slice(0, 5)) {
+        filesToRead.add(join(root, f));
+    }
+    // Read schema to understand models
+    for (const lib of result.libs) {
+        if (lib.file.includes("schema") || lib.file.includes("model") || lib.file.includes("db")) {
+            filesToRead.add(join(root, lib.file));
+        }
+    }
+    // Check middleware to know what to apply
+    for (const mw of result.middleware) {
+        filesToRead.add(join(root, mw.file));
+    }
+    let totalTokens = 0;
+    const readFiles = [];
+    for (const f of filesToRead) {
+        const content = await readFileSafe(f);
+        if (content) {
+            totalTokens += countTokens(content);
+            readFiles.push(relative(root, f));
+        }
+    }
+    const toolCalls = Math.max(6, Math.ceil(filesToRead.size * 0.7));
+    totalTokens += toolCalls * 75;
+    // With codesight, AI only reads the routes + schema sections (~40% of output)
+    const withTokens = Math.ceil(codesightTokens * 0.4);
+    const reduction = totalTokens > 0 ? Math.round((totalTokens / withTokens) * 10) / 10 : 1;
+    return {
+        name: "Add new API route",
+        description: "Find route patterns, check schema, apply middleware",
+        filesRead: readFiles,
+        toolCalls,
+        tokensWithout: totalTokens,
+        tokensWith: withTokens,
+        reduction,
+    };
+}
+/**
+ * Task 3: "Review a diff / understand blast radius"
+ * Without codesight: AI needs to trace imports, find dependents, check what routes
+ * and models are affected by a file change.
+ */
+async function measureReviewDiff(root, result, codesightTokens) {
+    const filesToRead = new Set();
+    // AI would read the changed file + all its importers
+    // Simulate: pick the hottest file and trace its dependents
+    if (result.graph.hotFiles.length > 0) {
+        const hotFile = result.graph.hotFiles[0];
+        filesToRead.add(join(root, hotFile.file));
+        // Read files that import it
+        for (const edge of result.graph.edges) {
+            if (edge.to === hotFile.file) {
+                filesToRead.add(join(root, edge.from));
+            }
+        }
+    }
+    // Also read some route files to check impact
+    const routeFiles = [...new Set(result.routes.map((r) => r.file))];
+    for (const f of routeFiles.slice(0, 3)) {
+        filesToRead.add(join(root, f));
+    }
+    let totalTokens = 0;
+    const readFiles = [];
+    for (const f of filesToRead) {
+        const content = await readFileSafe(f);
+        if (content) {
+            totalTokens += countTokens(content);
+            readFiles.push(relative(root, f));
+        }
+    }
+    const toolCalls = Math.max(8, Math.ceil(filesToRead.size * 0.6));
+    totalTokens += toolCalls * 75;
+    // With codesight, AI reads graph section + routes (~50% of output)
+    const withTokens = Math.ceil(codesightTokens * 0.5);
+    const reduction = totalTokens > 0 ? Math.round((totalTokens / withTokens) * 10) / 10 : 1;
+    return {
+        name: "Review diff / blast radius",
+        description: "Trace imports, find affected routes and models",
+        filesRead: readFiles,
+        toolCalls,
+        tokensWithout: totalTokens,
+        tokensWith: withTokens,
+        reduction,
+    };
+}
+export async function runTelemetry(root, result, outputDir) {
+    // Read the codesight output to get real token count
+    const codesightContent = await readFileSafe(join(outputDir, "CODESIGHT.md"));
+    const codesightTokens = countTokens(codesightContent);
+    const tasks = await Promise.all([
+        measureExplainArchitecture(root, result, codesightTokens),
+        measureAddRoute(root, result, codesightTokens),
+        measureReviewDiff(root, result, codesightTokens),
+    ]);
+    const totalWithout = tasks.reduce((s, t) => s + t.tokensWithout, 0);
+    const totalWith = tasks.reduce((s, t) => s + t.tokensWith, 0);
+    const totalToolCalls = tasks.reduce((s, t) => s + t.toolCalls, 0);
+    const report = {
+        project: result.project.name,
+        tasks,
+        summary: {
+            totalTokensWithout: totalWithout,
+            totalTokensWith: totalWith,
+            averageReduction: totalWith > 0 ? Math.round((totalWithout / totalWith) * 10) / 10 : 1,
+            totalToolCallsSaved: totalToolCalls,
+        },
+    };
+    // Write telemetry report
+    const reportLines = [
+        `# Token Telemetry: ${result.project.name}`,
+        "",
+        `> Measured by reading the actual files an AI agent would need for each task,`,
+        `> then comparing against the codesight output (~${codesightTokens.toLocaleString()} tokens).`,
+        "",
+        "## Tasks",
+        "",
+    ];
+    for (const task of tasks) {
+        reportLines.push(`### ${task.name}`);
+        reportLines.push(`_${task.description}_`);
+        reportLines.push("");
+        reportLines.push(`| Metric | Value |`);
+        reportLines.push(`|---|---|`);
+        reportLines.push(`| Files AI would read | ${task.filesRead.length} |`);
+        reportLines.push(`| Tool calls (glob/grep/read) | ${task.toolCalls} |`);
+        reportLines.push(`| Tokens without codesight | ~${task.tokensWithout.toLocaleString()} |`);
+        reportLines.push(`| Tokens with codesight | ~${task.tokensWith.toLocaleString()} |`);
+        reportLines.push(`| **Reduction** | **${task.reduction}x** |`);
+        reportLines.push("");
+        if (task.filesRead.length > 0) {
+            reportLines.push("<details>");
+            reportLines.push(`<summary>Files read (${task.filesRead.length})</summary>`);
+            reportLines.push("");
+            for (const f of task.filesRead) {
+                reportLines.push(`- \`${f}\``);
+            }
+            reportLines.push("");
+            reportLines.push("</details>");
+            reportLines.push("");
+        }
+    }
+    reportLines.push("## Summary");
+    reportLines.push("");
+    reportLines.push(`| Metric | Value |`);
+    reportLines.push(`|---|---|`);
+    reportLines.push(`| Total tokens without codesight | ~${report.summary.totalTokensWithout.toLocaleString()} |`);
+    reportLines.push(`| Total tokens with codesight | ~${report.summary.totalTokensWith.toLocaleString()} |`);
+    reportLines.push(`| **Average reduction** | **${report.summary.averageReduction}x** |`);
+    reportLines.push(`| Tool calls saved | ${report.summary.totalToolCallsSaved} |`);
+    reportLines.push("");
+    reportLines.push("## Methodology");
+    reportLines.push("");
+    reportLines.push("Token counts are calculated by reading the actual source files an AI agent would");
+    reportLines.push("need to explore for each task, using the ~4 chars/token heuristic (standard for");
+    reportLines.push("GPT/Claude tokenizers). Tool call overhead is estimated at ~75 tokens per call");
+    reportLines.push("(command text + result formatting). The \"with codesight\" count uses the real");
+    reportLines.push("CODESIGHT.md output size, proportioned to the sections relevant to each task.");
+    reportLines.push("");
+    reportLines.push(`_Generated by codesight --telemetry_`);
+    const { writeFile: wf } = await import("node:fs/promises");
+    const { mkdir } = await import("node:fs/promises");
+    await mkdir(outputDir, { recursive: true });
+    await wf(join(outputDir, "telemetry.md"), reportLines.join("\n"));
+    return report;
+}

package/dist/types.d.ts

@@ -95,12 +95,47 @@ export interface BlastRadiusResult {
     depth: number;
 }
 export interface CodesightConfig {
+    /** Disable specific detectors: "routes", "schema", "components", "libs", "config", "middleware", "graph" */
     disableDetectors?: string[];
+    /** Custom route tags: { "billing": ["stripe", "payment"] } */
     customTags?: Record<string, string[]>;
+    /** Max directory depth (default: 10) */
     maxDepth?: number;
+    /** Output directory name (default: ".codesight") */
     outputDir?: string;
+    /** AI tool profile */
     profile?: "claude-code" | "cursor" | "codex" | "copilot" | "windsurf" | "generic";
+    /** Additional ignore patterns (glob-style) */
     ignorePatterns?: string[];
+    /** Custom route patterns: [{ pattern: "router\\.handle\\(", method: "ALL" }] */
+    customRoutePatterns?: {
+        pattern: string;
+        method?: string;
+    }[];
+    /** Blast radius max BFS depth (default: 5) */
+    blastRadiusDepth?: number;
+    /** Hot file threshold: min imports to be "hot" (default: 3) */
+    hotFileThreshold?: number;
+    /** Plugin hooks */
+    plugins?: CodesightPlugin[];
+}
+export interface CodesightPlugin {
+    /** Plugin name for identification */
+    name: string;
+    /** Custom detector: runs after built-in detectors */
+    detector?: (files: string[], project: ProjectInfo) => Promise<PluginDetectorResult>;
+    /** Post-processor: transforms the final ScanResult */
+    postProcessor?: (result: ScanResult) => Promise<ScanResult>;
+}
+export interface PluginDetectorResult {
+    /** Additional routes to merge */
+    routes?: RouteInfo[];
+    /** Additional schema models to merge */
+    schemas?: SchemaModel[];
+    /** Additional components to merge */
+    components?: ComponentInfo[];
+    /** Additional middleware to merge */
+    middleware?: MiddlewareInfo[];
 }
 export interface ScanResult {
     project: ProjectInfo;

package/eval/README.md

@@ -0,0 +1,36 @@
+# codesight Evaluation Suite
+
+Reproducible accuracy benchmarks for codesight detectors.
+
+## How It Works
+
+Each fixture in `fixtures/` contains:
+- `repo.json` — describes the repo structure (files with inline content)
+- `ground-truth.json` — expected detection results (routes, models, env vars, blast radius)
+
+Running `npx codesight --eval` will:
+1. Create temporary directories from each fixture
+2. Run codesight detectors on them
+3. Compare results against ground truth
+4. Print precision, recall, F1 score, and runtime per fixture
+
+## Fixtures
+
+| Fixture | Stack | What it tests |
+|---|---|---|
+| `nextjs-drizzle` | Next.js App Router + Drizzle ORM | Routes, schema, components, env vars |
+| `express-prisma` | Express + Prisma | Route detection, schema parsing, middleware |
+| `fastapi-sqlalchemy` | FastAPI + SQLAlchemy | Python routes, Python ORM, config |
+| `hono-monorepo` | Hono + Drizzle (pnpm monorepo) | Monorepo detection, workspace routes, schema |
+
+## Adding a Fixture
+
+1. Create a folder in `fixtures/` with `repo.json` and `ground-truth.json`
+2. Follow the JSON schema used by existing fixtures
+3. Run `npx codesight --eval` to verify
+
+## Metrics
+
+- **Precision**: of all items codesight detected, how many are correct?
+- **Recall**: of all items that exist, how many did codesight find?
+- **F1**: harmonic mean of precision and recall

package/eval/fixtures/express-prisma/ground-truth.json

@@ -0,0 +1,31 @@
+{
+  "routes": [
+    { "method": "POST", "path": "/login" },
+    { "method": "POST", "path": "/register" },
+    { "method": "GET", "path": "/" },
+    { "method": "GET", "path": "/:id" },
+    { "method": "PUT", "path": "/:id" },
+    { "method": "DELETE", "path": "/:id" },
+    { "method": "POST", "path": "/" }
+  ],
+  "models": [
+    {
+      "name": "User",
+      "fields": ["id", "email", "password", "name", "role", "posts", "createdAt"]
+    },
+    {
+      "name": "Post",
+      "fields": ["id", "title", "content", "published", "author", "authorId", "tags", "createdAt"]
+    },
+    {
+      "name": "Tag",
+      "fields": ["id", "name", "posts"]
+    },
+    {
+      "name": "enum:Role",
+      "fields": ["USER", "ADMIN"]
+    }
+  ],
+  "envVars": ["DATABASE_URL", "JWT_SECRET", "PORT", "REDIS_URL", "CORS_ORIGIN"],
+  "middleware": ["auth", "error", "rate-limit", "cors"]
+}

package/eval/fixtures/express-prisma/repo.json

@@ -0,0 +1,19 @@
+{
+  "name": "express-prisma-api",
+  "description": "Express.js REST API with Prisma ORM and middleware",
+  "files": {
+    "package.json": "{\"name\":\"api-server\",\"dependencies\":{\"express\":\"4.18.0\",\"@prisma/client\":\"5.0.0\",\"cors\":\"2.8.5\",\"helmet\":\"7.0.0\",\"express-rate-limit\":\"7.0.0\",\"jsonwebtoken\":\"9.0.0\"},\"devDependencies\":{\"typescript\":\"5.3.0\",\"@types/express\":\"4.17.0\",\"@types/node\":\"20.0.0\",\"prisma\":\"5.0.0\"}}",
+    "tsconfig.json": "{\"compilerOptions\":{\"target\":\"es2017\",\"module\":\"commonjs\",\"outDir\":\"dist\"}}",
+    ".env.example": "DATABASE_URL=postgresql://localhost:5432/api\nJWT_SECRET=changeme\nPORT=3000\nREDIS_URL=redis://localhost:6379\nCORS_ORIGIN=http://localhost:3001",
+    "prisma/schema.prisma": "datasource db {\n  provider = \"postgresql\"\n  url      = env(\"DATABASE_URL\")\n}\n\ngenerator client {\n  provider = \"prisma-client-js\"\n}\n\nmodel User {\n  id        Int      @id @default(autoincrement())\n  email     String   @unique\n  password  String\n  name      String?\n  role      Role     @default(USER)\n  posts     Post[]\n  createdAt DateTime @default(now())\n}\n\nmodel Post {\n  id        Int       @id @default(autoincrement())\n  title     String\n  content   String?\n  published Boolean   @default(false)\n  author    User      @relation(fields: [authorId], references: [id])\n  authorId  Int\n  tags      Tag[]\n  createdAt DateTime  @default(now())\n}\n\nmodel Tag {\n  id    Int    @id @default(autoincrement())\n  name  String @unique\n  posts Post[]\n}\n\nenum Role {\n  USER\n  ADMIN\n}",
+    "src/index.ts": "import express from 'express';\nimport cors from 'cors';\nimport helmet from 'helmet';\nimport { authRouter } from './routes/auth';\nimport { usersRouter } from './routes/users';\nimport { postsRouter } from './routes/posts';\nimport { errorHandler } from './middleware/error';\nimport { rateLimiter } from './middleware/rate-limit';\n\nconst app = express();\n\napp.use(cors());\napp.use(helmet());\napp.use(express.json());\napp.use(rateLimiter);\n\napp.use('/api/auth', authRouter);\napp.use('/api/users', usersRouter);\napp.use('/api/posts', postsRouter);\n\napp.use(errorHandler);\n\napp.listen(process.env.PORT || 3000);",
+    "src/routes/auth.ts": "import { Router } from 'express';\nimport { prisma } from '../lib/prisma';\nimport { generateToken } from '../lib/jwt';\n\nexport const authRouter = Router();\n\nauthRouter.post('/login', async (req, res) => {\n  const { email, password } = req.body;\n  const user = await prisma.user.findUnique({ where: { email } });\n  if (!user) return res.status(401).json({ error: 'Invalid credentials' });\n  const token = generateToken(user.id);\n  res.json({ token });\n});\n\nauthRouter.post('/register', async (req, res) => {\n  const { email, password, name } = req.body;\n  const user = await prisma.user.create({ data: { email, password, name } });\n  const token = generateToken(user.id);\n  res.status(201).json({ token });\n});",
+    "src/routes/users.ts": "import { Router } from 'express';\nimport { prisma } from '../lib/prisma';\nimport { authenticate } from '../middleware/auth';\n\nexport const usersRouter = Router();\n\nusersRouter.get('/', authenticate, async (req, res) => {\n  const users = await prisma.user.findMany();\n  res.json(users);\n});\n\nusersRouter.get('/:id', authenticate, async (req, res) => {\n  const user = await prisma.user.findUnique({ where: { id: parseInt(req.params.id) } });\n  res.json(user);\n});\n\nusersRouter.put('/:id', authenticate, async (req, res) => {\n  const user = await prisma.user.update({ where: { id: parseInt(req.params.id) }, data: req.body });\n  res.json(user);\n});\n\nusersRouter.delete('/:id', authenticate, async (req, res) => {\n  await prisma.user.delete({ where: { id: parseInt(req.params.id) } });\n  res.json({ deleted: true });\n});",
+    "src/routes/posts.ts": "import { Router } from 'express';\nimport { prisma } from '../lib/prisma';\nimport { authenticate } from '../middleware/auth';\n\nexport const postsRouter = Router();\n\npostsRouter.get('/', async (req, res) => {\n  const posts = await prisma.post.findMany({ include: { author: true, tags: true } });\n  res.json(posts);\n});\n\npostsRouter.get('/:id', async (req, res) => {\n  const post = await prisma.post.findUnique({ where: { id: parseInt(req.params.id) }, include: { author: true, tags: true } });\n  res.json(post);\n});\n\npostsRouter.post('/', authenticate, async (req, res) => {\n  const post = await prisma.post.create({ data: { ...req.body, authorId: req.userId } });\n  res.status(201).json(post);\n});\n\npostsRouter.put('/:id', authenticate, async (req, res) => {\n  const post = await prisma.post.update({ where: { id: parseInt(req.params.id) }, data: req.body });\n  res.json(post);\n});\n\npostsRouter.delete('/:id', authenticate, async (req, res) => {\n  await prisma.post.delete({ where: { id: parseInt(req.params.id) } });\n  res.json({ deleted: true });\n});",
+    "src/middleware/auth.ts": "import { Request, Response, NextFunction } from 'express';\nimport { verifyToken } from '../lib/jwt';\n\nexport function authenticate(req: Request, res: Response, next: NextFunction) {\n  const token = req.headers.authorization?.replace('Bearer ', '');\n  if (!token) return res.status(401).json({ error: 'No token' });\n  try {\n    const payload = verifyToken(token);\n    (req as any).userId = payload.userId;\n    next();\n  } catch {\n    res.status(401).json({ error: 'Invalid token' });\n  }\n}",
+    "src/middleware/error.ts": "import { Request, Response, NextFunction } from 'express';\n\nexport function errorHandler(err: Error, req: Request, res: Response, next: NextFunction) {\n  console.error(err.stack);\n  res.status(500).json({ error: 'Internal server error' });\n}",
+    "src/middleware/rate-limit.ts": "import rateLimit from 'express-rate-limit';\n\nexport const rateLimiter = rateLimit({\n  windowMs: 15 * 60 * 1000,\n  max: 100,\n  message: { error: 'Too many requests' }\n});",
+    "src/lib/prisma.ts": "import { PrismaClient } from '@prisma/client';\n\nexport const prisma = new PrismaClient();",
+    "src/lib/jwt.ts": "import jwt from 'jsonwebtoken';\n\nconst SECRET = process.env.JWT_SECRET || 'dev-secret';\n\nexport function generateToken(userId: number): string {\n  return jwt.sign({ userId }, SECRET, { expiresIn: '24h' });\n}\n\nexport function verifyToken(token: string): { userId: number } {\n  return jwt.verify(token, SECRET) as { userId: number };\n}"
+  }
+}

package/eval/fixtures/fastapi-sqlalchemy/ground-truth.json

@@ -0,0 +1,29 @@
+{
+  "routes": [
+    { "method": "GET", "path": "/health" },
+    { "method": "GET", "path": "/" },
+    { "method": "GET", "path": "/{user_id}" },
+    { "method": "POST", "path": "/" },
+    { "method": "DELETE", "path": "/{user_id}" },
+    { "method": "GET", "path": "/{item_id}" },
+    { "method": "POST", "path": "/" },
+    { "method": "PUT", "path": "/{item_id}" },
+    { "method": "POST", "path": "/login" },
+    { "method": "POST", "path": "/register" }
+  ],
+  "models": [
+    {
+      "name": "User",
+      "fields": ["id", "email", "password", "name", "is_active", "items", "created_at"]
+    },
+    {
+      "name": "Item",
+      "fields": ["id", "title", "description", "price", "owner_id", "owner", "created_at"]
+    },
+    {
+      "name": "Category",
+      "fields": ["id", "name", "description"]
+    }
+  ],
+  "envVars": ["DATABASE_URL", "SECRET_KEY", "DEBUG", "ALLOWED_ORIGINS"]
+}

package/eval/fixtures/fastapi-sqlalchemy/repo.json

@@ -0,0 +1,16 @@
+{
+  "name": "fastapi-sqlalchemy-api",
+  "description": "FastAPI with SQLAlchemy models",
+  "files": {
+    "requirements.txt": "fastapi==0.104.0\nuvicorn==0.24.0\nsqlalchemy==2.0.23\nalembic==1.12.0\npydantic==2.5.0\npython-dotenv==1.0.0",
+    ".env.example": "DATABASE_URL=postgresql://localhost:5432/fastapi_db\nSECRET_KEY=changeme\nDEBUG=true\nALLOWED_ORIGINS=http://localhost:3000",
+    "app/__init__.py": "",
+    "app/main.py": "from fastapi import FastAPI\nfrom fastapi.middleware.cors import CORSMiddleware\nfrom app.routes import users, items, auth\n\napp = FastAPI(title=\"My API\")\n\napp.add_middleware(CORSMiddleware, allow_origins=[\"*\"])\n\napp.include_router(users.router, prefix=\"/api/users\", tags=[\"users\"])\napp.include_router(items.router, prefix=\"/api/items\", tags=[\"items\"])\napp.include_router(auth.router, prefix=\"/api/auth\", tags=[\"auth\"])\n\n@app.get(\"/health\")\ndef health():\n    return {\"status\": \"ok\"}",
+    "app/routes/__init__.py": "",
+    "app/routes/users.py": "from fastapi import APIRouter, Depends, HTTPException\nfrom sqlalchemy.orm import Session\nfrom app.db import get_db\nfrom app.models import User\n\nrouter = APIRouter()\n\n@router.get(\"/\")\ndef list_users(db: Session = Depends(get_db)):\n    return db.query(User).all()\n\n@router.get(\"/{user_id}\")\ndef get_user(user_id: int, db: Session = Depends(get_db)):\n    user = db.query(User).filter(User.id == user_id).first()\n    if not user:\n        raise HTTPException(status_code=404)\n    return user\n\n@router.post(\"/\")\ndef create_user(data: dict, db: Session = Depends(get_db)):\n    user = User(**data)\n    db.add(user)\n    db.commit()\n    return user\n\n@router.delete(\"/{user_id}\")\ndef delete_user(user_id: int, db: Session = Depends(get_db)):\n    db.query(User).filter(User.id == user_id).delete()\n    db.commit()\n    return {\"deleted\": True}",
+    "app/routes/items.py": "from fastapi import APIRouter, Depends\nfrom sqlalchemy.orm import Session\nfrom app.db import get_db\nfrom app.models import Item\n\nrouter = APIRouter()\n\n@router.get(\"/\")\ndef list_items(db: Session = Depends(get_db)):\n    return db.query(Item).all()\n\n@router.get(\"/{item_id}\")\ndef get_item(item_id: int, db: Session = Depends(get_db)):\n    return db.query(Item).filter(Item.id == item_id).first()\n\n@router.post(\"/\")\ndef create_item(data: dict, db: Session = Depends(get_db)):\n    item = Item(**data)\n    db.add(item)\n    db.commit()\n    return item\n\n@router.put(\"/{item_id}\")\ndef update_item(item_id: int, data: dict, db: Session = Depends(get_db)):\n    item = db.query(Item).filter(Item.id == item_id).first()\n    for k, v in data.items():\n        setattr(item, k, v)\n    db.commit()\n    return item",
+    "app/routes/auth.py": "from fastapi import APIRouter\n\nrouter = APIRouter()\n\n@router.post(\"/login\")\ndef login(data: dict):\n    return {\"token\": \"fake-token\"}\n\n@router.post(\"/register\")\ndef register(data: dict):\n    return {\"user\": data}",
+    "app/models.py": "from sqlalchemy import Column, Integer, String, Boolean, ForeignKey, DateTime, Float\nfrom sqlalchemy.orm import relationship\nfrom datetime import datetime\nfrom app.db import Base\n\nclass User(Base):\n    __tablename__ = 'users'\n    id = Column(Integer, primary_key=True)\n    email = Column(String, unique=True, nullable=False)\n    password = Column(String, nullable=False)\n    name = Column(String)\n    is_active = Column(Boolean, default=True)\n    items = relationship('Item', back_populates='owner')\n    created_at = Column(DateTime, default=datetime.utcnow)\n\nclass Item(Base):\n    __tablename__ = 'items'\n    id = Column(Integer, primary_key=True)\n    title = Column(String, nullable=False)\n    description = Column(String)\n    price = Column(Float)\n    owner_id = Column(Integer, ForeignKey('users.id'))\n    owner = relationship('User', back_populates='items')\n    created_at = Column(DateTime, default=datetime.utcnow)\n\nclass Category(Base):\n    __tablename__ = 'categories'\n    id = Column(Integer, primary_key=True)\n    name = Column(String, unique=True, nullable=False)\n    description = Column(String)",
+    "app/db.py": "from sqlalchemy import create_engine\nfrom sqlalchemy.orm import sessionmaker, declarative_base\nimport os\n\nDATABASE_URL = os.getenv('DATABASE_URL', 'sqlite:///./test.db')\nengine = create_engine(DATABASE_URL)\nSessionLocal = sessionmaker(bind=engine)\nBase = declarative_base()\n\ndef get_db():\n    db = SessionLocal()\n    try:\n        yield db\n    finally:\n        db.close()"
+  }
+}

package/eval/fixtures/gin-gorm/ground-truth.json

@@ -0,0 +1,25 @@
+{
+  "routes": [
+    { "method": "GET", "path": "/health" },
+    { "method": "GET", "path": "/api/users" },
+    { "method": "GET", "path": "/api/users/:id" },
+    { "method": "POST", "path": "/api/users" },
+    { "method": "PUT", "path": "/api/users/:id" },
+    { "method": "DELETE", "path": "/api/users/:id" },
+    { "method": "GET", "path": "/api/projects" },
+    { "method": "GET", "path": "/api/projects/:id" },
+    { "method": "POST", "path": "/api/projects" }
+  ],
+  "models": [
+    {
+      "name": "User",
+      "fields": ["ID", "CreatedAt", "UpdatedAt", "DeletedAt", "Email", "Password", "Name", "Role"]
+    },
+    {
+      "name": "Project",
+      "fields": ["ID", "CreatedAt", "UpdatedAt", "DeletedAt", "Name", "Description", "IsPublic", "OwnerID"]
+    }
+  ],
+  "envVars": ["DATABASE_URL", "JWT_SECRET", "PORT", "GIN_MODE"],
+  "middleware": ["auth"]
+}

package/eval/fixtures/gin-gorm/repo.json

@@ -0,0 +1,16 @@
+{
+  "name": "gin-gorm-api",
+  "description": "Gin REST API with GORM models and route groups",
+  "files": {
+    "go.mod": "module github.com/example/api\n\ngo 1.22\n\nrequire (\n\tgithub.com/gin-gonic/gin v1.9.1\n\tgorm.io/gorm v1.25.5\n\tgorm.io/driver/postgres v1.5.4\n)",
+    ".env.example": "DATABASE_URL=postgres://localhost:5432/ginapi\nJWT_SECRET=changeme\nPORT=8080\nGIN_MODE=release",
+    "main.go": "package main\n\nimport (\n\t\"github.com/gin-gonic/gin\"\n\t\"github.com/example/api/handlers\"\n\t\"github.com/example/api/middleware\"\n)\n\nfunc main() {\n\tr := gin.Default()\n\n\tr.GET(\"/health\", handlers.Health)\n\n\tapi := r.Group(\"/api\")\n\tapi.Use(middleware.Auth())\n\n\tusers := api.Group(\"/users\")\n\tusers.GET(\"\", handlers.ListUsers)\n\tusers.GET(\"/:id\", handlers.GetUser)\n\tusers.POST(\"\", handlers.CreateUser)\n\tusers.PUT(\"/:id\", handlers.UpdateUser)\n\tusers.DELETE(\"/:id\", handlers.DeleteUser)\n\n\tprojects := api.Group(\"/projects\")\n\tprojects.GET(\"\", handlers.ListProjects)\n\tprojects.GET(\"/:id\", handlers.GetProject)\n\tprojects.POST(\"\", handlers.CreateProject)\n\n\tr.Run(\":8080\")\n}",
+    "models/user.go": "package models\n\nimport (\n\t\"gorm.io/gorm\"\n\t\"time\"\n)\n\ntype User struct {\n\tgorm.Model\n\tEmail     string    `gorm:\"uniqueIndex;not null\" json:\"email\"`\n\tPassword  string    `gorm:\"not null\" json:\"-\"`\n\tName      string    `json:\"name\"`\n\tRole      string    `gorm:\"default:user\" json:\"role\"`\n\tProjects  []Project `gorm:\"foreignKey:OwnerID\" json:\"projects,omitempty\"`\n}",
+    "models/project.go": "package models\n\nimport (\n\t\"gorm.io/gorm\"\n)\n\ntype Project struct {\n\tgorm.Model\n\tName        string `gorm:\"not null\" json:\"name\"`\n\tDescription string `json:\"description\"`\n\tIsPublic    bool   `gorm:\"default:true\" json:\"is_public\"`\n\tOwnerID     uint   `gorm:\"not null;index\" json:\"owner_id\"`\n\tOwner       *User  `gorm:\"foreignKey:OwnerID\" json:\"owner,omitempty\"`\n}",
+    "handlers/health.go": "package handlers\n\nimport \"github.com/gin-gonic/gin\"\n\nfunc Health(c *gin.Context) {\n\tc.JSON(200, gin.H{\"status\": \"ok\"})\n}",
+    "handlers/users.go": "package handlers\n\nimport (\n\t\"github.com/gin-gonic/gin\"\n\t\"github.com/example/api/models\"\n)\n\nfunc ListUsers(c *gin.Context) {\n\tvar users []models.User\n\tc.JSON(200, users)\n}\n\nfunc GetUser(c *gin.Context) {\n\tc.JSON(200, gin.H{})\n}\n\nfunc CreateUser(c *gin.Context) {\n\tc.JSON(201, gin.H{})\n}\n\nfunc UpdateUser(c *gin.Context) {\n\tc.JSON(200, gin.H{})\n}\n\nfunc DeleteUser(c *gin.Context) {\n\tc.JSON(200, gin.H{\"deleted\": true})\n}",
+    "handlers/projects.go": "package handlers\n\nimport (\n\t\"github.com/gin-gonic/gin\"\n)\n\nfunc ListProjects(c *gin.Context) {\n\tc.JSON(200, gin.H{})\n}\n\nfunc GetProject(c *gin.Context) {\n\tc.JSON(200, gin.H{})\n}\n\nfunc CreateProject(c *gin.Context) {\n\tc.JSON(201, gin.H{})\n}",
+    "middleware/auth.go": "package middleware\n\nimport \"github.com/gin-gonic/gin\"\n\nfunc Auth() gin.HandlerFunc {\n\treturn func(c *gin.Context) {\n\t\ttoken := c.GetHeader(\"Authorization\")\n\t\tif token == \"\" {\n\t\t\tc.AbortWithStatusJSON(401, gin.H{\"error\": \"unauthorized\"})\n\t\t\treturn\n\t\t}\n\t\tc.Next()\n\t}\n}",
+    "db/db.go": "package db\n\nimport (\n\t\"gorm.io/driver/postgres\"\n\t\"gorm.io/gorm\"\n\t\"os\"\n)\n\nvar DB *gorm.DB\n\nfunc Init() {\n\tdsn := os.Getenv(\"DATABASE_URL\")\n\tdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tDB = db\n}"
+  }
+}

package/eval/fixtures/hono-monorepo/ground-truth.json

@@ -0,0 +1,33 @@
+{
+  "routes": [
+    { "method": "GET", "path": "/health" },
+    { "method": "POST", "path": "/login" },
+    { "method": "POST", "path": "/register" },
+    { "method": "POST", "path": "/refresh" },
+    { "method": "GET", "path": "/" },
+    { "method": "GET", "path": "/:id" },
+    { "method": "PUT", "path": "/:id" },
+    { "method": "DELETE", "path": "/:id" },
+    { "method": "GET", "path": "/" },
+    { "method": "GET", "path": "/:id" },
+    { "method": "POST", "path": "/" }
+  ],
+  "models": [
+    {
+      "name": "users",
+      "fields": ["id", "email", "name", "role", "createdAt"],
+      "relations": ["projects"]
+    },
+    {
+      "name": "projects",
+      "fields": ["id", "name", "description", "ownerId", "isPublic", "createdAt"],
+      "relations": ["owner"]
+    }
+  ],
+  "components": [
+    { "name": "ProjectCard", "props": ["name", "description", "isPublic"] },
+    { "name": "UserAvatar", "props": ["name", "size"] }
+  ],
+  "envVars": ["DATABASE_URL", "JWT_SECRET", "PORT"],
+  "middleware": ["auth"]
+}

package/eval/fixtures/hono-monorepo/repo.json

@@ -0,0 +1,20 @@
+{
+  "name": "hono-monorepo",
+  "description": "Hono API + React frontend in pnpm monorepo with Drizzle",
+  "files": {
+    "package.json": "{\"name\":\"my-monorepo\",\"private\":true}",
+    "pnpm-workspace.yaml": "packages:\n  - 'apps/*'\n  - 'packages/*'",
+    "apps/api/package.json": "{\"name\":\"@mono/api\",\"dependencies\":{\"hono\":\"4.0.0\",\"drizzle-orm\":\"0.30.0\",\"pg\":\"8.11.0\",\"zod\":\"3.22.0\"},\"devDependencies\":{\"typescript\":\"5.3.0\"}}",
+    "apps/api/.env.example": "DATABASE_URL=postgres://localhost:5432/mono\nJWT_SECRET=secret\nPORT=4000",
+    "apps/api/src/index.ts": "import { Hono } from 'hono';\nimport { cors } from 'hono/cors';\nimport { logger } from 'hono/logger';\nimport { authRoutes } from './routes/auth';\nimport { userRoutes } from './routes/users';\nimport { projectRoutes } from './routes/projects';\n\nconst app = new Hono();\n\napp.use('*', cors());\napp.use('*', logger());\n\napp.get('/health', (c) => c.json({ ok: true }));\n\napp.route('/api/auth', authRoutes);\napp.route('/api/users', userRoutes);\napp.route('/api/projects', projectRoutes);\n\nexport default app;",
+    "apps/api/src/routes/auth.ts": "import { Hono } from 'hono';\nimport { zValidator } from '@hono/zod-validator';\nimport { z } from 'zod';\n\nexport const authRoutes = new Hono();\n\nconst loginSchema = z.object({ email: z.string().email(), password: z.string() });\n\nauthRoutes.post('/login', zValidator('json', loginSchema), async (c) => {\n  const { email, password } = c.req.valid('json');\n  return c.json({ token: 'jwt-token' });\n});\n\nauthRoutes.post('/register', async (c) => {\n  const body = await c.req.json();\n  return c.json({ user: body }, 201);\n});\n\nauthRoutes.post('/refresh', async (c) => {\n  return c.json({ token: 'new-token' });\n});",
+    "apps/api/src/routes/users.ts": "import { Hono } from 'hono';\nimport { db } from '../db';\nimport { users } from '../db/schema';\nimport { eq } from 'drizzle-orm';\n\nexport const userRoutes = new Hono();\n\nuserRoutes.get('/', async (c) => {\n  const all = await db.select().from(users);\n  return c.json(all);\n});\n\nuserRoutes.get('/:id', async (c) => {\n  const id = parseInt(c.req.param('id'));\n  const user = await db.select().from(users).where(eq(users.id, id));\n  return c.json(user[0]);\n});\n\nuserRoutes.put('/:id', async (c) => {\n  const id = parseInt(c.req.param('id'));\n  const body = await c.req.json();\n  const updated = await db.update(users).set(body).where(eq(users.id, id)).returning();\n  return c.json(updated[0]);\n});\n\nuserRoutes.delete('/:id', async (c) => {\n  const id = parseInt(c.req.param('id'));\n  await db.delete(users).where(eq(users.id, id));\n  return c.json({ deleted: true });\n});",
+    "apps/api/src/routes/projects.ts": "import { Hono } from 'hono';\nimport { db } from '../db';\nimport { projects } from '../db/schema';\nimport { eq } from 'drizzle-orm';\n\nexport const projectRoutes = new Hono();\n\nprojectRoutes.get('/', async (c) => {\n  const all = await db.select().from(projects);\n  return c.json(all);\n});\n\nprojectRoutes.get('/:id', async (c) => {\n  const id = parseInt(c.req.param('id'));\n  const project = await db.select().from(projects).where(eq(projects.id, id));\n  return c.json(project[0]);\n});\n\nprojectRoutes.post('/', async (c) => {\n  const body = await c.req.json();\n  const created = await db.insert(projects).values(body).returning();\n  return c.json(created[0], 201);\n});",
+    "apps/api/src/db/index.ts": "import { drizzle } from 'drizzle-orm/node-postgres';\nimport { Pool } from 'pg';\n\nconst pool = new Pool({ connectionString: process.env.DATABASE_URL });\nexport const db = drizzle(pool);",
+    "apps/api/src/db/schema.ts": "import { pgTable, serial, text, timestamp, integer, boolean } from 'drizzle-orm/pg-core';\nimport { relations } from 'drizzle-orm';\n\nexport const users = pgTable('users', {\n  id: serial('id').primaryKey(),\n  email: text('email').notNull().unique(),\n  name: text('name'),\n  role: text('role').default('user'),\n  createdAt: timestamp('created_at').defaultNow()\n});\n\nexport const projects = pgTable('projects', {\n  id: serial('id').primaryKey(),\n  name: text('name').notNull(),\n  description: text('description'),\n  ownerId: integer('owner_id').notNull().references(() => users.id),\n  isPublic: boolean('is_public').default(true),\n  createdAt: timestamp('created_at').defaultNow()\n});\n\nexport const usersRelations = relations(users, ({ many }) => ({\n  projects: many(projects)\n}));\n\nexport const projectsRelations = relations(projects, ({ one }) => ({\n  owner: one(users, { fields: [projects.ownerId], references: [users.id] })\n}));",
+    "apps/api/src/middleware/auth.ts": "import { Context, Next } from 'hono';\n\nexport async function authMiddleware(c: Context, next: Next) {\n  const token = c.req.header('Authorization');\n  if (!token) return c.json({ error: 'Unauthorized' }, 401);\n  await next();\n}",
+    "apps/web/package.json": "{\"name\":\"@mono/web\",\"dependencies\":{\"react\":\"18.0.0\",\"react-dom\":\"18.0.0\"},\"devDependencies\":{\"typescript\":\"5.3.0\",\"vite\":\"5.0.0\"}}",
+    "apps/web/src/components/ProjectCard.tsx": "import React from 'react';\n\ninterface ProjectCardProps {\n  name: string;\n  description: string;\n  isPublic: boolean;\n}\n\nexport function ProjectCard({ name, description, isPublic }: ProjectCardProps) {\n  return <div><h3>{name}</h3><p>{description}</p>{isPublic && <span>Public</span>}</div>;\n}",
+    "apps/web/src/components/UserAvatar.tsx": "'use client';\nimport React from 'react';\n\ninterface UserAvatarProps {\n  name: string;\n  size?: number;\n}\n\nexport function UserAvatar({ name, size = 40 }: UserAvatarProps) {\n  return <div style={{ width: size, height: size }}>{name[0]}</div>;\n}"
+  }
+}

package/eval/fixtures/nextjs-drizzle/ground-truth.json

@@ -0,0 +1,37 @@
+{
+  "routes": [
+    { "method": "GET", "path": "/api/users" },
+    { "method": "POST", "path": "/api/users" },
+    { "method": "GET", "path": "/api/users/[id]" },
+    { "method": "PUT", "path": "/api/users/[id]" },
+    { "method": "DELETE", "path": "/api/users/[id]" },
+    { "method": "GET", "path": "/api/posts" },
+    { "method": "POST", "path": "/api/posts" },
+    { "method": "GET", "path": "/api/posts/[id]/comments" },
+    { "method": "POST", "path": "/api/posts/[id]/comments" }
+  ],
+  "models": [
+    {
+      "name": "users",
+      "fields": ["id", "email", "name", "createdAt"],
+      "relations": ["posts", "comments"]
+    },
+    {
+      "name": "posts",
+      "fields": ["id", "title", "content", "published", "authorId", "createdAt"],
+      "relations": ["author", "comments"]
+    },
+    {
+      "name": "comments",
+      "fields": ["id", "body", "postId", "authorId", "createdAt"],
+      "relations": []
+    }
+  ],
+  "components": [
+    { "name": "UserCard", "props": ["name", "email", "avatar"] },
+    { "name": "PostList", "props": ["posts", "onSelect"] },
+    { "name": "CommentForm", "props": ["postId", "onSubmit"] }
+  ],
+  "envVars": ["DATABASE_URL", "NEXTAUTH_SECRET", "NEXT_PUBLIC_API_URL"],
+  "middleware": ["middleware", "auth"]
+}