@aionlabsai/aion 0.2.12 → 0.2.13

package/dist/cli/commands/eval.d.ts

@@ -0,0 +1,3 @@
+import type { Command } from 'commander';
+export declare function registerEval(program: Command): void;
+//# sourceMappingURL=eval.d.ts.map

package/dist/cli/commands/eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiEzC,wBAAgB,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CA4JnD"}

package/dist/cli/commands/eval.js

@@ -0,0 +1,160 @@
+import chalk from 'chalk';
+import ora from 'ora';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { embedTextRemote, embedText } from '../../infra/embeddings.js';
+import { createVectorStore } from '../../infra/vector-store.js';
+import { rerankWithLLM, rerankLocal } from '../../infra/reranker.js';
+import { BM25Index, rrfScore } from '../../infra/bm25.js';
+function loadGoldenSet(path) {
+    try {
+        return JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch {
+        return null;
+    }
+}
+function scaffoldGoldenSet(path) {
+    mkdirSync(join(path, '..'), { recursive: true });
+    const example = {
+        queries: [
+            { query: 'embedding vector similarity', expected: ['src/infra/embeddings.ts'], note: 'example — edit expected files' },
+            { query: 'BM25 ranking score', expected: ['src/infra/bm25.ts'] },
+            { query: 'audit pipeline scanner agents', expected: ['src/core/pipelines/audit-pipeline.ts'] },
+        ],
+    };
+    writeFileSync(path, JSON.stringify(example, null, 2), 'utf8');
+}
+export function registerEval(program) {
+    const evalCmd = program
+        .command('eval')
+        .description('Evaluate retrieval quality and agent output quality');
+    // ── aion eval retrieval ───────────────────────────────────────────────────
+    evalCmd
+        .command('retrieval')
+        .description('Measure recall@3, recall@5, MRR of memory search against a golden set')
+        .option('--rerank <mode>', 'none | local | llm — apply re-ranking after retrieval (default: none)', 'none')
+        .option('--top-k <n>', 'candidates to retrieve before re-ranking', '10')
+        .option('--scaffold', 'create example golden set at .ai-memory/eval/retrieval.json and exit')
+        .option('--json', 'output results as JSON')
+        .action(async (options) => {
+        const cwd = process.cwd();
+        const goldenPath = join(cwd, '.ai-memory', 'eval', 'retrieval.json');
+        if (options.scaffold) {
+            scaffoldGoldenSet(goldenPath);
+            console.log(chalk.green(`Created: ${goldenPath}`));
+            console.log(chalk.gray('Edit the file with your project-specific queries and expected files, then run `aion eval retrieval`.'));
+            return;
+        }
+        if (!existsSync(goldenPath)) {
+            console.error(chalk.yellow(`No golden set found at ${goldenPath}`));
+            console.error(chalk.gray('Run `aion eval retrieval --scaffold` to create an example.'));
+            process.exit(1);
+        }
+        const golden = loadGoldenSet(goldenPath);
+        if (!golden || golden.queries.length === 0) {
+            console.error(chalk.red('Golden set is empty or invalid.'));
+            process.exit(1);
+        }
+        const topK = Math.max(5, parseInt(options.topK, 10) || 10);
+        const spinner = ora(`Evaluating ${golden.queries.length} queries (rerank: ${options.rerank})...`).start();
+        const vectorStore = createVectorStore(cwd);
+        if (vectorStore.size() === 0) {
+            spinner.fail(chalk.yellow('No vector index. Run `aion memory build` first.'));
+            process.exit(1);
+        }
+        const results = [];
+        for (const { query, expected } of golden.queries) {
+            // 1. Embed query
+            const remote = await embedTextRemote(query);
+            const queryVec = remote ? remote : Array.from(embedText(query, 500));
+            // 2. Vector search → top-K candidates
+            const vecResults = await vectorStore.search(queryVec, topK);
+            // 3. BM25 over the same candidates (in-memory)
+            const bm25 = new BM25Index();
+            for (const r of vecResults) {
+                bm25.add(r.id, String(r.payload['preview'] ?? r.id));
+            }
+            const bm25Results = bm25.score(query);
+            // 4. RRF fusion
+            const fused = rrfScore(vecResults.map((r, i) => ({ id: r.id, score: vecResults.length - i })), bm25Results);
+            // 5. Optional re-ranking
+            let finalIds;
+            if (options.rerank === 'llm') {
+                const candidates = fused.slice(0, topK).map((r) => {
+                    const hit = vecResults.find((v) => v.id === r.id);
+                    return { id: r.id, content: String(hit?.payload['preview'] ?? r.id) };
+                });
+                const reranked = await rerankWithLLM(query, candidates, 5);
+                finalIds = reranked.map((r) => r.id);
+            }
+            else if (options.rerank === 'local') {
+                const candidates = fused.slice(0, topK).map((r) => {
+                    const hit = vecResults.find((v) => v.id === r.id);
+                    return { id: r.id, content: String(hit?.payload['preview'] ?? r.id) };
+                });
+                finalIds = rerankLocal(query, candidates, 5).map((r) => r.id);
+            }
+            else {
+                finalIds = fused.slice(0, 5).map((r) => r.id);
+            }
+            // 6. Score: match by file path substring
+            const matchesExpected = (id) => expected.some((exp) => id.includes(exp) || exp.includes(id.split(':')[0] ?? ''));
+            const rankOfFirst = finalIds.findIndex(matchesExpected);
+            results.push({
+                query,
+                expected,
+                retrieved: finalIds,
+                hitAt3: rankOfFirst >= 0 && rankOfFirst < 3,
+                hitAt5: rankOfFirst >= 0 && rankOfFirst < 5,
+                rr: rankOfFirst >= 0 ? 1 / (rankOfFirst + 1) : 0,
+            });
+        }
+        spinner.stop();
+        const n = results.length;
+        const recall3 = results.filter((r) => r.hitAt3).length / n;
+        const recall5 = results.filter((r) => r.hitAt5).length / n;
+        const mrr = results.reduce((s, r) => s + r.rr, 0) / n;
+        const { embeddingProvider } = await import('../../infra/embeddings.js');
+        const { vectorStoreBackend } = await import('../../infra/vector-store.js');
+        const report = {
+            provider: embeddingProvider(),
+            backend: vectorStoreBackend(),
+            rerankMode: options.rerank,
+            queries: n,
+            recall3: Math.round(recall3 * 1000) / 1000,
+            recall5: Math.round(recall5 * 1000) / 1000,
+            mrr: Math.round(mrr * 1000) / 1000,
+            results,
+            ranAt: new Date().toISOString(),
+        };
+        if (options.json) {
+            console.log(JSON.stringify(report, null, 2));
+            return;
+        }
+        // Human-readable output
+        console.log(chalk.bold(`\nRetrieval Evaluation — ${n} queries\n`));
+        console.log(`  Provider:  ${chalk.cyan(report.provider)}`);
+        console.log(`  Backend:   ${chalk.cyan(report.backend)}`);
+        console.log(`  Re-rank:   ${chalk.cyan(options.rerank)}\n`);
+        const pct = (v) => chalk.bold(`${(v * 100).toFixed(1)}%`);
+        console.log(`  recall@3   ${pct(recall3)}`);
+        console.log(`  recall@5   ${pct(recall5)}`);
+        console.log(`  MRR        ${pct(mrr)}\n`);
+        for (const r of results) {
+            const status = r.hitAt3 ? chalk.green('✓') : r.hitAt5 ? chalk.yellow('~') : chalk.red('✗');
+            console.log(`  ${status}  ${chalk.gray(r.query.slice(0, 60))}`);
+            if (!r.hitAt5) {
+                console.log(chalk.gray(`       expected: ${r.expected.join(', ')}`));
+                console.log(chalk.gray(`       got:      ${r.retrieved.slice(0, 3).join(', ')}`));
+            }
+        }
+        // Save report
+        const outDir = join(cwd, '.ai-runtime', 'eval');
+        mkdirSync(outDir, { recursive: true });
+        const outPath = join(outDir, `retrieval-${Date.now()}.json`);
+        writeFileSync(outPath, JSON.stringify(report, null, 2), 'utf8');
+        console.log(chalk.gray(`\n  Report saved: ${outPath}`));
+    });
+}
+//# sourceMappingURL=eval.js.map

package/dist/cli/commands/eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/cli/commands/eval.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAqC1D,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAc,CAAC;IAC7D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,MAAM,OAAO,GAAc;QACzB,OAAO,EAAE;YACP,EAAE,KAAK,EAAE,6BAA6B,EAAE,QAAQ,EAAE,CAAC,yBAAyB,CAAC,EAAE,IAAI,EAAE,+BAA+B,EAAE;YACtH,EAAE,KAAK,EAAE,oBAAoB,EAAE,QAAQ,EAAE,CAAC,mBAAmB,CAAC,EAAE;YAChE,EAAE,KAAK,EAAE,+BAA+B,EAAE,QAAQ,EAAE,CAAC,sCAAsC,CAAC,EAAE;SAC/F;KACF,CAAC;IACF,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAgB;IAC3C,MAAM,OAAO,GAAG,OAAO;SACpB,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,qDAAqD,CAAC,CAAC;IAEtE,6EAA6E;IAC7E,OAAO;SACJ,OAAO,CAAC,WAAW,CAAC;SACpB,WAAW,CAAC,uEAAuE,CAAC;SACpF,MAAM,CAAC,iBAAiB,EAAE,uEAAuE,EAAE,MAAM,CAAC;SAC1G,MAAM,CAAC,aAAa,EAAE,0CAA0C,EAAE,IAAI,CAAC;SACvE,MAAM,CAAC,YAAY,EAAE,sEAAsE,CAAC;SAC5F,MAAM,CAAC,QAAQ,EAAE,wBAAwB,CAAC;SAC1C,MAAM,CAAC,KAAK,EAAE,OAA6E,EAAE,EAAE;QAC9F,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAErE,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,iBAAiB,CAAC,UAAU,CAAC,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,UAAU,EAAE,CAAC,CAAC,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sGAAsG,CAAC,CAAC,CAAC;YAChI,OAAO;QACT,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,0BAA0B,UAAU,EAAE,CAAC,CAAC,CAAC;YACpE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC,CAAC;YACxF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC,CAAC;YAC5D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,GAAG,CAAC,cAAc,MAAM,CAAC,OAAO,CAAC,MAAM,qBAAqB,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;QAE1G,MAAM,WAAW,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,iDAAiD,CAAC,CAAC,CAAC;YAC9E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,OAAO,GAAiB,EAAE,CAAC;QAEjC,KAAK,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACjD,iBAAiB;YACjB,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YAErE,sCAAsC;YACtC,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAE5D,+CAA+C;YAC/C,MAAM,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC7B,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;gBAC3B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvD,CAAC;YACD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAEtC,gBAAgB;YAChB,MAAM,KAAK,GAAG,QAAQ,CACpB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC,EACtE,WAAW,CACZ,CAAC;YAEF,yBAAyB;YACzB,IAAI,QAAkB,CAAC;YACvB,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;gBAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBAChD,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;oBAClD,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;gBACxE,CAAC,CAAC,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;gBAC3D,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACvC,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;gBACtC,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBAChD,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;oBAClD,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;gBACxE,CAAC,CAAC,CAAC;gBACH,QAAQ,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChE,CAAC;iBAAM,CAAC;gBACN,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChD,CAAC;YAED,yCAAyC;YACzC,MAAM,eAAe,GAAG,CAAC,EAAU,EAAE,EAAE,CACrC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAEnF,MAAM,WAAW,GAAG,QAAQ,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YACxD,OAAO,CAAC,IAAI,CAAC;gBACX,KAAK;gBACL,QAAQ;gBACR,SAAS,EAAE,QAAQ;gBACnB,MAAM,EAAE,WAAW,IAAI,CAAC,IAAI,WAAW,GAAG,CAAC;gBAC3C,MAAM,EAAE,WAAW,IAAI,CAAC,IAAI,WAAW,GAAG,CAAC;gBAC3C,EAAE,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aACjD,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,IAAI,EAAE,CAAC;QAEf,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAC3D,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QAEtD,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACxE,MAAM,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAE3E,MAAM,MAAM,GAAe;YACzB,QAAQ,EAAE,iBAAiB,EAAE;YAC7B,OAAO,EAAE,kBAAkB,EAAE;YAC7B,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,IAAI;YAC1C,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,IAAI;YAC1C,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI;YAClC,OAAO;YACP,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SAChC,CAAC;QAEF,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QAED,wBAAwB;QACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,YAAY,CAAC,CAAC,CAAC;QACnE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAE5D,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,gBAAgB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE1C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3F,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;gBACrE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACpF,CAAC;QACH,CAAC;QAED,cAAc;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;QAChD,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,aAAa,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC7D,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,qBAAqB,OAAO,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/index.js

@@ -25,6 +25,7 @@ import { registerSearch } from './cli/commands/search.js';
 import { registerTree } from './cli/commands/tree.js';
 import { registerNext } from './cli/commands/next.js';
 import { registerCi } from './cli/commands/ci.js';
+import { registerEval } from './cli/commands/eval.js';
 import { runNaturalLanguage } from './cli/interactive.js';
 import { runMenu } from './cli/menu.js';
 runMigrations();
@@ -84,6 +85,7 @@ registerSearch(program);
 registerTree(program);
 registerNext(program);
 registerCi(program);
+registerEval(program);
 // Explicit menu command
 program
     .command('menu')

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5E,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAkB,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,aAAa,EAAE,CAAC;AAChB,MAAM,cAAc,EAAE,CAAC;AAEvB,OAAO;KACJ,IAAI,CAAC,IAAI,CAAC;KACV,WAAW,CAAC,oCAAoC,CAAC;KACjD,OAAO,CAAC,iBAAiB,EAAE,CAAC;KAC5B,MAAM,CAAC,kBAAkB,EAAE,mDAAmD,CAAC;KAC/E,QAAQ,CAAC,cAAc,EAAE,0DAA0D,CAAC;KACpF,MAAM,CAAC,KAAK,EAAE,YAAsB,EAAE,EAAE;IACvC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAsB,CAAC;IAChD,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACrD,OAAO,CAAC,KAAK,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,mEAAmE;QACnE,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC/B,CAAC;SAAM,CAAC;QACN,MAAM,kBAAkB,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAClE,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,WAAW,EAAE,EAAE;IAC5C,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAsB,CAAC;IACpD,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACrD,OAAO,CAAC,KAAK,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,eAAe,CAAC,OAAO,CAAC,CAAC;AACzB,WAAW,CAAC,OAAO,CAAC,CAAC;AACrB,cAAc,CAAC,OAAO,CAAC,CAAC;AACxB,cAAc,CAAC,OAAO,CAAC,CAAC;AACxB,aAAa,CAAC,OAAO,CAAC,CAAC;AACvB,aAAa,CAAC,OAAO,CAAC,CAAC;AACvB,aAAa,CAAC,OAAO,CAAC,CAAC;AACvB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,gBAAgB,CAAC,OAAO,CAAC,CAAC;AAC1B,cAAc,CAAC,OAAO,CAAC,CAAC;AACxB,cAAc,CAAC,OAAO,CAAC,CAAC;AACxB,eAAe,CAAC,OAAO,CAAC,CAAC;AACzB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,eAAe,CAAC,OAAO,CAAC,CAAC;AACzB,cAAc,CAAC,OAAO,CAAC,CAAC;AACxB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,UAAU,CAAC,OAAO,CAAC,CAAC;AAEpB,wBAAwB;AACxB,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,kDAAkD,CAAC;KAC/D,MAAM,CAAC,KAAK,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEzD,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5E,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAkB,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,aAAa,EAAE,CAAC;AAChB,MAAM,cAAc,EAAE,CAAC;AAEvB,OAAO;KACJ,IAAI,CAAC,IAAI,CAAC;KACV,WAAW,CAAC,oCAAoC,CAAC;KACjD,OAAO,CAAC,iBAAiB,EAAE,CAAC;KAC5B,MAAM,CAAC,kBAAkB,EAAE,mDAAmD,CAAC;KAC/E,QAAQ,CAAC,cAAc,EAAE,0DAA0D,CAAC;KACpF,MAAM,CAAC,KAAK,EAAE,YAAsB,EAAE,EAAE;IACvC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAsB,CAAC;IAChD,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACrD,OAAO,CAAC,KAAK,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,mEAAmE;QACnE,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC/B,CAAC;SAAM,CAAC;QACN,MAAM,kBAAkB,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAClE,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,WAAW,EAAE,EAAE;IAC5C,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAsB,CAAC;IACpD,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACrD,OAAO,CAAC,KAAK,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,eAAe,CAAC,OAAO,CAAC,CAAC;AACzB,WAAW,CAAC,OAAO,CAAC,CAAC;AACrB,cAAc,CAAC,OAAO,CAAC,CAAC;AACxB,cAAc,CAAC,OAAO,CAAC,CAAC;AACxB,aAAa,CAAC,OAAO,CAAC,CAAC;AACvB,aAAa,CAAC,OAAO,CAAC,CAAC;AACvB,aAAa,CAAC,OAAO,CAAC,CAAC;AACvB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,gBAAgB,CAAC,OAAO,CAAC,CAAC;AAC1B,cAAc,CAAC,OAAO,CAAC,CAAC;AACxB,cAAc,CAAC,OAAO,CAAC,CAAC;AACxB,eAAe,CAAC,OAAO,CAAC,CAAC;AACzB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,eAAe,CAAC,OAAO,CAAC,CAAC;AACzB,cAAc,CAAC,OAAO,CAAC,CAAC;AACxB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,YAAY,CAAC,OAAO,CAAC,CAAC;AACtB,UAAU,CAAC,OAAO,CAAC,CAAC;AACpB,YAAY,CAAC,OAAO,CAAC,CAAC;AAEtB,wBAAwB;AACxB,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,kDAAkD,CAAC;KAC/D,MAAM,CAAC,KAAK,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEzD,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}

package/dist/infra/reranker.d.ts

@@ -0,0 +1,21 @@
+export interface RerankCandidate {
+    id: string;
+    content: string;
+}
+export interface RerankResult {
+    id: string;
+    rank: number;
+    reason?: string;
+}
+/**
+ * Re-rank candidates using Claude as a cross-encoder.
+ * Retrieves top-20, re-ranks to top-K. Cost: ~500 tokens per call.
+ * Falls back to original order if API unavailable.
+ */
+export declare function rerankWithLLM(query: string, candidates: RerankCandidate[], topK?: number): Promise<RerankResult[]>;
+/**
+ * Lightweight re-rank using keyword overlap score — zero cost, no API.
+ * Useful as fallback or for testing without spending tokens.
+ */
+export declare function rerankLocal(query: string, candidates: RerankCandidate[], topK?: number): RerankResult[];
+//# sourceMappingURL=reranker.d.ts.map

package/dist/infra/reranker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.d.ts","sourceRoot":"","sources":["../../src/infra/reranker.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,eAAe,EAAE,EAC7B,IAAI,SAAI,GACP,OAAO,CAAC,YAAY,EAAE,CAAC,CAgDzB;AAED;;;GAGG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,eAAe,EAAE,EAC7B,IAAI,SAAI,GACP,YAAY,EAAE,CAahB"}

package/dist/infra/reranker.js

@@ -0,0 +1,68 @@
+import Anthropic from '@anthropic-ai/sdk';
+/**
+ * Re-rank candidates using Claude as a cross-encoder.
+ * Retrieves top-20, re-ranks to top-K. Cost: ~500 tokens per call.
+ * Falls back to original order if API unavailable.
+ */
+export async function rerankWithLLM(query, candidates, topK = 5) {
+    if (candidates.length <= 1) {
+        return candidates.slice(0, topK).map((c, i) => ({ id: c.id, rank: i }));
+    }
+    const apiKey = process.env.ANTHROPIC_API_KEY;
+    if (!apiKey) {
+        return candidates.slice(0, topK).map((c, i) => ({ id: c.id, rank: i }));
+    }
+    const numbered = candidates
+        .slice(0, 20) // re-rank at most 20 candidates
+        .map((c, i) => `[${i + 1}] ${c.content.slice(0, 300)}`)
+        .join('\n\n');
+    const prompt = `You are a code search relevance judge.
+
+Query: "${query}"
+
+Rank these code chunks from most to least relevant. Return ONLY a JSON array of numbers representing the ranking (1-indexed), most relevant first.
+Example: [3, 1, 7, 2, 5] means chunk 3 is most relevant, then 1, then 7, etc.
+
+Chunks:
+${numbered}
+
+JSON array (top ${topK} only):`;
+    try {
+        const client = new Anthropic({ apiKey });
+        const response = await client.messages.create({
+            model: 'claude-haiku-4-5-20251001',
+            max_tokens: 100,
+            messages: [{ role: 'user', content: prompt }],
+        });
+        const text = response.content[0]?.type === 'text' ? response.content[0].text.trim() : '';
+        const match = /\[[\d,\s]+\]/.exec(text);
+        if (!match)
+            throw new Error('No JSON array in response');
+        const ranks = JSON.parse(match[0]);
+        return ranks
+            .filter((r) => r >= 1 && r <= candidates.length)
+            .slice(0, topK)
+            .map((r, i) => ({ id: candidates[r - 1].id, rank: i }));
+    }
+    catch {
+        // Fallback: return original order
+        return candidates.slice(0, topK).map((c, i) => ({ id: c.id, rank: i }));
+    }
+}
+/**
+ * Lightweight re-rank using keyword overlap score — zero cost, no API.
+ * Useful as fallback or for testing without spending tokens.
+ */
+export function rerankLocal(query, candidates, topK = 5) {
+    const queryTerms = query.toLowerCase().match(/[a-z0-9_]{2,}/g) ?? [];
+    const scored = candidates.map((c, i) => {
+        const text = c.content.toLowerCase();
+        const score = queryTerms.reduce((s, t) => s + (text.includes(t) ? 1 : 0), 0);
+        return { id: c.id, originalRank: i, score };
+    });
+    return scored
+        .sort((a, b) => b.score - a.score || a.originalRank - b.originalRank)
+        .slice(0, topK)
+        .map((r, i) => ({ id: r.id, rank: i }));
+}
+//# sourceMappingURL=reranker.js.map

package/dist/infra/reranker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.js","sourceRoot":"","sources":["../../src/infra/reranker.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAa1C;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAa,EACb,UAA6B,EAC7B,IAAI,GAAG,CAAC;IAER,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC3B,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,QAAQ,GAAG,UAAU;SACxB,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,gCAAgC;SAC7C,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;SACtD,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,MAAM,MAAM,GAAG;;UAEP,KAAK;;;;;;EAMb,QAAQ;;kBAEQ,IAAI,SAAS,CAAC;IAE9B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC5C,KAAK,EAAE,2BAA2B;YAClC,UAAU,EAAE,GAAG;YACf,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SAC9C,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzF,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAEzD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAa,CAAC;QAC/C,OAAO,KAAK;aACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC;aAC/C,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;aACd,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC;IAAC,MAAM,CAAC;QACP,kCAAkC;QAClC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,KAAa,EACb,UAA6B,EAC7B,IAAI,GAAG,CAAC;IAER,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;IAErE,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrC,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM;SACV,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC;SACpE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;SACd,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5C,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aionlabsai/aion",
-  "version": "0.2.12",
+  "version": "0.2.13",
   "description": "Multi-agent AI engineering runtime — audit, fix, graph, analyze",
   "type": "module",
   "bin": {