@code-rag/cli 0.1.2 → 0.1.4

package/dist/commands/index-cmd.js

@@ -1,10 +1,10 @@
 import chalk from 'chalk';
 import ora from 'ora';
 import { writeFile, readFile, mkdir, appendFile, unlink } from 'node:fs/promises';
-import { join, resolve, sep } from 'node:path';
+import { basename, join, resolve, sep } from 'node:path';
 import { createHash } from 'node:crypto';
 import { existsSync } from 'node:fs';
-import { loadConfig, createIgnoreFilter, FileScanner, TreeSitterParser, MarkdownParser, ASTChunker, OllamaClient, NLEnricher, OllamaEmbeddingProvider, OpenAICompatibleEmbeddingProvider, ModelLifecycleManager, LanceDBStore, BM25Index, GraphBuilder, DependencyGraph, scanForABReferences, IndexState, MultiRepoIndexer, AzureDevOpsProvider, JiraProvider, ClickUpProvider, } from '@code-rag/core';
+import { loadConfig, createIgnoreFilter, FileScanner, TreeSitterParser, MarkdownParser, ASTChunker, OllamaClient, NLEnricher, OllamaEmbeddingProvider, OpenAICompatibleEmbeddingProvider, ModelLifecycleManager, LanceDBStore, BM25Index, GraphBuilder, DependencyGraph, scanForABReferences, IndexState, AzureDevOpsProvider, JiraProvider, ClickUpProvider, } from '@code-rag/core';
 // ---------------------------------------------------------------------------
 // Simple embedding provider factory — dispatches based on provider name
 // (Used for non-lifecycle providers like openai-compatible and direct ollama)
@@ -870,7 +870,7 @@ export function registerIndexCommand(program) {
             try {
                 // Multi-repo path: if repos are configured, index each independently
                 if (config.repos && config.repos.length > 0) {
-                    await indexMultiRepo(config, storagePath, options, logger, startTime);
+                    await indexMultiRepo(config, storagePath, options, logger, startTime, managed.provider);
                     return;
                 }
                 // Single-repo path
@@ -972,7 +972,7 @@ export function registerIndexCommand(program) {
  * Multi-repo indexing: iterate configured repos, index each with separate
  * progress reporting and per-repo storage directories.
  */
-async function indexMultiRepo(config, storagePath, options, logger, startTime) {
+async function indexMultiRepo(config, storagePath, options, logger, startTime, embeddingProvider) {
     const repos = config.repos;
     // eslint-disable-next-line no-console
     console.log('');
@@ -980,41 +980,347 @@ async function indexMultiRepo(config, storagePath, options, logger, startTime) {
     console.log(chalk.bold(`Indexing ${repos.length} repo(s)...`));
     // eslint-disable-next-line no-console
     console.log('');
-    const multiRepoIndexer = new MultiRepoIndexer(repos, storagePath);
+    logger.start('Starting multi-repo indexing...');
+    const parser = new TreeSitterParser();
+    const initResult = await parser.initialize();
+    if (initResult.isErr()) {
+        throw new Error(`Parser init failed: ${initResult.error.message}`);
+    }
+    const mdParser = new MarkdownParser({ maxTokensPerChunk: config.ingestion.maxTokensPerChunk });
+    const chunker = new ASTChunker({ maxTokensPerChunk: config.ingestion.maxTokensPerChunk });
+    const repoResults = [];
     let totalFiles = 0;
     let totalChunks = 0;
     let totalErrors = 0;
-    logger.start('Starting multi-repo indexing...');
-    const result = await multiRepoIndexer.indexAll({
-        full: options.full,
-        onProgress: (repoName, status) => {
-            void logger.info(`[${repoName}] ${status}`);
-        },
-    });
-    if (result.isErr()) {
-        await logger.fail(`Multi-repo indexing failed: ${result.error.message}`);
-        process.exit(1);
-    }
-    // Per-repo summary
-    for (const repoResult of result.value.repoResults) {
-        totalFiles += repoResult.filesProcessed;
-        totalChunks += repoResult.chunksCreated;
-        if (repoResult.errors.length > 0) {
-            totalErrors += repoResult.errors.length;
-            await logger.fail(`[${repoResult.repoName}] Failed`);
-            for (const error of repoResult.errors) {
+    for (const repo of repos) {
+        const repoName = repo.name ?? basename(repo.path);
+        const repoPath = resolve(repo.path);
+        const repoStoragePath = join(storagePath, repoName);
+        await mkdir(repoStoragePath, { recursive: true });
+        // Load or create index state
+        let indexState = new IndexState();
+        const indexStatePath = join(repoStoragePath, 'index-state.json');
+        if (!options.full) {
+            try {
+                const stateData = await readFile(indexStatePath, 'utf-8');
+                indexState = IndexState.fromJSON(JSON.parse(stateData));
+            }
+            catch {
+                // No saved state, start fresh
+            }
+        }
+        // Scan files
+        await logger.info(`[${repoName}] Scanning files...`);
+        const ignoreFilter = createIgnoreFilter(repoPath);
+        const scanner = new FileScanner(repoPath, ignoreFilter);
+        const scanResult = await scanner.scanFiles();
+        if (scanResult.isErr()) {
+            totalErrors++;
+            await logger.fail(`[${repoName}] Scan failed: ${scanResult.error.message}`);
+            continue;
+        }
+        const scannedFiles = scanResult.value;
+        let filesToProcess = scannedFiles;
+        if (!options.full) {
+            filesToProcess = scannedFiles.filter((f) => indexState.isDirty(f.filePath, f.contentHash));
+            if (filesToProcess.length === 0) {
+                repoResults.push({
+                    repoName, repoPath, repoStoragePath, filesToProcess: [], chunks: [],
+                    parsedFiles: [], indexState, indexStatePath, parseErrors: 0, skippedFiles: 0, parseErrorDetails: [],
+                });
+                await logger.info(`[${repoName}] Up to date`);
+                continue;
+            }
+        }
+        // Parse & chunk
+        const repoChunks = [];
+        const repoParsedFiles = [];
+        let parseErrors = 0;
+        let skippedFiles = 0;
+        const parseErrorDetails = [];
+        for (const file of filesToProcess) {
+            if (MarkdownParser.isMarkdownFile(file.filePath)) {
+                const mdResult = mdParser.parse(file.filePath, file.content);
+                if (mdResult.isErr()) {
+                    parseErrors++;
+                    parseErrorDetails.push({ file: file.filePath, reason: mdResult.error.message });
+                    continue;
+                }
+                repoChunks.push(...mdResult.value.chunks);
+                continue;
+            }
+            const parseResult = await parser.parse(file.filePath, file.content);
+            if (parseResult.isErr()) {
+                if (parseResult.error.message.startsWith('Unsupported file type:')) {
+                    skippedFiles++;
+                    continue;
+                }
+                parseErrors++;
+                parseErrorDetails.push({ file: file.filePath, reason: parseResult.error.message });
+                continue;
+            }
+            const parsed = parseResult.value;
+            repoParsedFiles.push(parsed);
+            const chunkResult = await chunker.chunk(parsed);
+            if (chunkResult.isErr()) {
+                parseErrors++;
+                parseErrorDetails.push({ file: file.filePath, reason: chunkResult.error.message });
+                continue;
+            }
+            repoChunks.push(...chunkResult.value);
+        }
+        // Stamp repoName in chunk metadata
+        for (const chunk of repoChunks) {
+            chunk.metadata.repoName = repoName;
+        }
+        totalFiles += filesToProcess.length;
+        totalChunks += repoChunks.length;
+        totalErrors += parseErrors;
+        const parsedCount = filesToProcess.length - parseErrors - skippedFiles;
+        await logger.info(`[${repoName}] ${filesToProcess.length} files, ${parsedCount} parsed, ${repoChunks.length} chunks` +
+            (skippedFiles > 0 ? ` (${skippedFiles} unsupported skipped)` : ''));
+        if (parseErrors > 0) {
+            for (const detail of parseErrorDetails.slice(0, 3)) {
                 // eslint-disable-next-line no-console
-                console.log(`    ${chalk.gray('→')} ${chalk.red(error)}`);
+                console.log(`    ${chalk.gray('→')} ${detail.file}: ${chalk.yellow(detail.reason)}`);
             }
         }
-        else if (repoResult.filesProcessed === 0) {
-            await logger.succeed(`[${repoResult.repoName}] Up to date`);
+        repoResults.push({
+            repoName, repoPath, repoStoragePath, filesToProcess, chunks: repoChunks,
+            parsedFiles: repoParsedFiles, indexState, indexStatePath, parseErrors, skippedFiles, parseErrorDetails,
+        });
+    }
+    // Show aggregate totals after scan & parse
+    // eslint-disable-next-line no-console
+    console.log('');
+    // eslint-disable-next-line no-console
+    console.log(chalk.bold('Scan complete: ') +
+        `${chalk.cyan(String(totalFiles))} files, ${chalk.cyan(String(totalChunks))} chunks across ${chalk.cyan(String(repos.length))} repos`);
+    // eslint-disable-next-line no-console
+    console.log('');
+    // All chunks across all repos (for unified enrichment)
+    const allChunks = repoResults.flatMap((r) => r.chunks);
+    if (allChunks.length === 0) {
+        // Still update index state for files with no chunks
+        for (const rr of repoResults) {
+            for (const file of rr.filesToProcess) {
+                rr.indexState.setFileState(file.filePath, {
+                    filePath: file.filePath,
+                    contentHash: file.contentHash,
+                    lastIndexedAt: new Date(),
+                    chunkIds: [],
+                });
+            }
+            await writeFile(rr.indexStatePath, JSON.stringify(rr.indexState.toJSON(), null, 2), 'utf-8');
+        }
+        const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+        // eslint-disable-next-line no-console
+        console.log(chalk.yellow('No chunks produced. Nothing to embed.'));
+        // eslint-disable-next-line no-console
+        console.log(`  Time elapsed: ${chalk.cyan(elapsed + 's')}`);
+        return;
+    }
+    // ── Phase 2: Enrich all chunks together (slow) ─────────────────────
+    await logger.setPhase('enrich', { totalChunks: allChunks.length, enrichedChunks: 0 });
+    const ollamaClient = new OllamaClient({ model: config.llm.model });
+    const enricher = new NLEnricher(ollamaClient);
+    // Load checkpoint (shared across repos)
+    const checkpoint = await loadEnrichmentCheckpoint(storagePath);
+    const savedSummaries = checkpoint?.summaries ?? {};
+    await logger.info(`Checkpoint: ${checkpoint ? `loaded (${Object.keys(savedSummaries).length} summaries)` : 'none found'}`);
+    const chunksToEnrich = allChunks.filter((c) => !(c.id in savedSummaries));
+    if (Object.keys(savedSummaries).length > 0) {
+        await logger.info(`Resuming enrichment: ${Object.keys(savedSummaries).length} already done, ${chunksToEnrich.length} remaining`);
+    }
+    else {
+        await logger.info(`Enriching ${allChunks.length} chunks with NL summaries...`);
+    }
+    // Pre-flight: verify Ollama
+    const ollamaAvailable = await ollamaClient.isAvailable();
+    if (!ollamaAvailable) {
+        await logger.fail(`Ollama is not reachable at ${ollamaClient.currentConfig.baseUrl}. Start Ollama first, then re-run.`);
+        throw new Error(`Ollama is not reachable at ${ollamaClient.currentConfig.baseUrl}`);
+    }
+    let enrichErrors = 0;
+    let consecutiveFailures = 0;
+    const MAX_CONSECUTIVE_FAILURES = 3;
+    const totalBatches = Math.ceil(chunksToEnrich.length / ENRICHMENT_BATCH_SIZE);
+    for (let i = 0; i < chunksToEnrich.length; i += ENRICHMENT_BATCH_SIZE) {
+        const batchNum = Math.floor(i / ENRICHMENT_BATCH_SIZE) + 1;
+        const batch = chunksToEnrich.slice(i, i + ENRICHMENT_BATCH_SIZE);
+        await logger.info(`Enrichment batch ${batchNum}/${totalBatches} (${batch.length} chunks, ${Object.keys(savedSummaries).length}/${allChunks.length} total)...`);
+        const enrichResult = await enricher.enrichBatch(batch);
+        if (enrichResult.isOk()) {
+            const { enriched, failedCount } = enrichResult.value;
+            for (const chunk of enriched) {
+                if (chunk.nlSummary) {
+                    savedSummaries[chunk.id] = chunk.nlSummary;
+                }
+            }
+            if (failedCount === 0) {
+                consecutiveFailures = 0;
+            }
+            else if (enriched.length > 0) {
+                consecutiveFailures = 0;
+                enrichErrors++;
+                await logger.warn(`Batch ${batchNum}: ${enriched.length} OK, ${failedCount} failed`);
+            }
+            else {
+                consecutiveFailures++;
+                enrichErrors++;
+                await logger.warn(`Batch ${batchNum}: all ${failedCount} chunks failed`);
+                if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
+                    await logger.fail(`Enrichment aborted: ${MAX_CONSECUTIVE_FAILURES} consecutive batch failures. ` +
+                        `Is Ollama running? Check: curl ${ollamaClient.currentConfig.baseUrl}/api/tags`);
+                    await saveEnrichmentCheckpoint(storagePath, {
+                        summaries: savedSummaries,
+                        totalProcessed: Object.keys(savedSummaries).length,
+                    });
+                    throw new Error(`Enrichment aborted after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`);
+                }
+            }
         }
         else {
-            await logger.succeed(`[${repoResult.repoName}] ${repoResult.filesProcessed} file(s) processed`);
+            enrichErrors++;
+            consecutiveFailures++;
+            await logger.warn(`Batch ${batchNum} enrichment error: ${enrichResult.error.message}`);
         }
+        await saveEnrichmentCheckpoint(storagePath, {
+            summaries: savedSummaries,
+            totalProcessed: Object.keys(savedSummaries).length,
+        });
+        await logger.updateCount('enrichedChunks', Object.keys(savedSummaries).length);
+    }
+    if (enrichErrors > 0) {
+        await logger.warn(`${enrichErrors} enrichment batch(es) failed, some chunks have no NL summary`);
+    }
+    await clearEnrichmentCheckpoint(storagePath);
+    // ── Phase 3: Embed & Store per repo ─────────────────────────────────
+    await logger.setPhase('embed');
+    const resolvedEmbeddingProvider = embeddingProvider ?? createSimpleEmbeddingProvider(config.embedding);
+    for (const rr of repoResults) {
+        if (rr.chunks.length === 0)
+            continue;
+        // Apply saved summaries to this repo's chunks
+        const enrichedChunks = rr.chunks.map((c) => {
+            const summary = savedSummaries[c.id];
+            return summary ? { ...c, nlSummary: summary } : c;
+        });
+        await logger.info(`[${rr.repoName}] Embedding ${enrichedChunks.length} chunks...`);
+        const textsToEmbed = enrichedChunks.map((c) => c.nlSummary ? `${c.nlSummary}\n\n${c.content}` : c.content);
+        const embedResult = await resolvedEmbeddingProvider.embed(textsToEmbed);
+        if (embedResult.isErr()) {
+            totalErrors++;
+            await logger.fail(`[${rr.repoName}] Embedding failed: ${embedResult.error.message}`);
+            continue;
+        }
+        const embeddings = embedResult.value;
+        // Store in LanceDB
+        await logger.info(`[${rr.repoName}] Storing in LanceDB...`);
+        const store = new LanceDBStore(rr.repoStoragePath, config.embedding.dimensions);
+        await store.connect();
+        const ids = enrichedChunks.map((c) => c.id);
+        const metadata = enrichedChunks.map((c) => ({
+            content: c.content,
+            nl_summary: c.nlSummary,
+            chunk_type: c.metadata.chunkType,
+            file_path: c.filePath,
+            language: c.language,
+            start_line: c.startLine,
+            end_line: c.endLine,
+            name: c.metadata.name,
+            ...(c.metadata.repoName ? { repo_name: c.metadata.repoName } : {}),
+        }));
+        const upsertResult = await store.upsert(ids, embeddings, metadata);
+        if (upsertResult.isErr()) {
+            store.close();
+            totalErrors++;
+            await logger.fail(`[${rr.repoName}] Store failed: ${upsertResult.error.message}`);
+            continue;
+        }
+        // BM25 index
+        const bm25Path = join(rr.repoStoragePath, 'bm25-index.json');
+        let bm25;
+        if (options.full) {
+            bm25 = new BM25Index();
+        }
+        else {
+            try {
+                const existingBm25 = await readFile(bm25Path, 'utf-8');
+                bm25 = BM25Index.deserialize(existingBm25);
+                const staleChunkIds = [];
+                for (const file of rr.filesToProcess) {
+                    const fileState = rr.indexState.getFileState(file.filePath);
+                    if (fileState)
+                        staleChunkIds.push(...fileState.chunkIds);
+                }
+                if (staleChunkIds.length > 0) {
+                    try {
+                        bm25.removeChunks(staleChunkIds);
+                    }
+                    catch {
+                        bm25 = await rebuildBm25FromStore(store, logger, `[${rr.repoName}] `);
+                    }
+                }
+            }
+            catch {
+                bm25 = new BM25Index();
+            }
+        }
+        bm25.addChunks(enrichedChunks);
+        await writeFile(bm25Path, bm25.serialize(), 'utf-8');
+        // Dependency graph
+        const graphBuilder = new GraphBuilder(rr.repoPath);
+        const graphResult = graphBuilder.buildFromFiles(rr.parsedFiles);
+        if (graphResult.isOk()) {
+            const graphPath = join(rr.repoStoragePath, 'graph.json');
+            const newGraph = graphResult.value;
+            if (options.full) {
+                await writeFile(graphPath, JSON.stringify(newGraph.toJSON()), 'utf-8');
+            }
+            else {
+                try {
+                    const existingData = await readFile(graphPath, 'utf-8');
+                    const existingGraph = DependencyGraph.fromJSON(JSON.parse(existingData));
+                    const reindexedFiles = new Set(rr.filesToProcess.map((f) => f.filePath));
+                    const existingNodes = existingGraph.getAllNodes();
+                    const existingEdges = existingGraph.getAllEdges();
+                    const keptNodes = existingNodes.filter((n) => !reindexedFiles.has(n.filePath));
+                    const keptNodeIds = new Set(keptNodes.map((n) => n.id));
+                    const keptEdges = existingEdges.filter((e) => keptNodeIds.has(e.source) && keptNodeIds.has(e.target));
+                    const merged = new DependencyGraph();
+                    for (const node of keptNodes)
+                        merged.addNode(node);
+                    for (const edge of keptEdges)
+                        merged.addEdge(edge);
+                    for (const node of newGraph.getAllNodes())
+                        merged.addNode(node);
+                    for (const edge of newGraph.getAllEdges())
+                        merged.addEdge(edge);
+                    await writeFile(graphPath, JSON.stringify(merged.toJSON()), 'utf-8');
+                }
+                catch {
+                    await writeFile(graphPath, JSON.stringify(newGraph.toJSON()), 'utf-8');
+                }
+            }
+        }
+        // Update index state
+        for (const file of rr.filesToProcess) {
+            const fileChunkIds = enrichedChunks
+                .filter((c) => c.filePath === file.filePath)
+                .map((c) => c.id);
+            rr.indexState.setFileState(file.filePath, {
+                filePath: file.filePath,
+                contentHash: file.contentHash,
+                lastIndexedAt: new Date(),
+                chunkIds: fileChunkIds,
+            });
+        }
+        await writeFile(rr.indexStatePath, JSON.stringify(rr.indexState.toJSON(), null, 2), 'utf-8');
+        store.close();
+        await logger.succeed(`[${rr.repoName}] ${enrichedChunks.length} chunks indexed`);
     }
-    // Total summary
+    // ── Final summary ───────────────────────────────────────────────────
     const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
     // eslint-disable-next-line no-console
     console.log('');

package/dist/index.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 import { Command } from 'commander';
+import { createRequire } from 'node:module';
 import { registerInitCommand } from './commands/init.js';
 import { registerIndexCommand } from './commands/index-cmd.js';
 import { registerSearchCommand } from './commands/search.js';
@@ -8,11 +9,13 @@ import { registerStatusCommand } from './commands/status.js';
 import { registerViewerCommand } from './commands/viewer.js';
 import { registerWatchCommand } from './commands/watch-cmd.js';
 import { registerHooksCommand } from './commands/hooks-cmd.js';
+const require = createRequire(import.meta.url);
+const pkg = require('../package.json');
 const program = new Command();
 program
     .name('coderag')
     .description('CodeRAG — intelligent codebase context engine for AI coding agents')
-    .version('0.1.0');
+    .version(pkg.version);
 registerInitCommand(program);
 registerIndexCommand(program);
 registerSearchCommand(program);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@code-rag/cli",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "CLI tool for CodeRAG — init, index, search, serve, and status commands for codebase context engine",
   "type": "module",
   "license": "MIT",
@@ -48,9 +48,9 @@
     "commander": "^13.1.0",
     "ora": "^8.2.0",
     "yaml": "^2.7.0",
-    "@code-rag/api-server": "0.1.2",
-    "@code-rag/mcp-server": "0.1.2",
-    "@code-rag/core": "0.1.2"
+    "@code-rag/api-server": "0.1.4",
+    "@code-rag/core": "0.1.4",
+    "@code-rag/mcp-server": "0.1.4"
   },
   "devDependencies": {
     "@types/node": "^22.13.4",