@optiqcode/cli 2.9.0 → 3.0.1

package/dist/commands/index.d.ts

@@ -1,5 +1,7 @@
 interface IndexOptions {
     path?: string;
+    verbose?: boolean;
+    debug?: boolean;
 }
 export declare function index(options: IndexOptions): Promise<void>;
 export {};

package/dist/commands/index.js

@@ -49,18 +49,29 @@ export async function index(options) {
         console.log(chalk.yellow('\n⚠️  Indexing cancelled'));
         return;
     }
+    // Debug mode: show ignore patterns
+    if (options.debug) {
+        const ignorePatterns = await getGitIgnorePatterns(targetPath);
+        console.log(chalk.blue('\n📋 Ignore patterns being used:'));
+        console.log(chalk.dim(ignorePatterns.slice(0, 30).join('\n')));
+        if (ignorePatterns.length > 30) {
+            console.log(chalk.dim(`... and ${ignorePatterns.length - 30} more`));
+        }
+        console.log('');
+    }
     const spinner = ora('Collecting files...').start();
     try {
         // Collect all files
-        const files = await collectFiles(targetPath);
+        const files = await collectFiles(targetPath, options.verbose || options.debug);
         spinner.text = `Reading ${files.length} files...`;
         // Read file contents
         const fileContents = [];
+        let skippedFiles = 0;
         for (const file of files) {
             try {
                 const content = await fs.readFile(file, 'utf-8');
                 const relativePath = path.relative(targetPath, file);
-                // CRITICAL FIX: Normalize path separators to forward slashes for cross-platform consistency
+                // Normalize path separators to forward slashes for cross-platform consistency
                 const normalizedPath = relativePath.replace(/\\/g, '/');
                 fileContents.push({
                     path: normalizedPath,
@@ -68,63 +79,123 @@ export async function index(options) {
                 });
             }
             catch (error) {
-                // Skip files that can't be read
+                skippedFiles++;
+                // Skip files that can't be read (binary, permissions, etc.)
             }
         }
+        if (skippedFiles > 0) {
+            console.log(chalk.dim(`\n   Skipped ${skippedFiles} unreadable files`));
+        }
         spinner.text = 'Indexing with Optiq Engine...';
         // Generate repository ID from path
         const repoId = generateRepoId(targetPath);
-        // Batch upload files (20 at a time to avoid gateway timeouts)
-        const BATCH_SIZE = 20;
+        console.log(chalk.blue('\n📊 Repository ID:'), chalk.bold(repoId));
+        console.log(chalk.dim(`   Indexing ${fileContents.length} files...\n`));
+        // PARALLEL STREAMING: Send multiple batches concurrently for maximum speed
+        const BATCH_SIZE = 40; // Files per batch
+        const CONCURRENT_BATCHES = 4; // Number of batches to send in parallel
+        const MAX_RETRIES = 3;
+        const RETRY_DELAY_MS = 2000;
+        const TIMEOUT_MS = 180000; // 3 minutes timeout
         let uploadedCount = 0;
         let totalChunksCreated = 0;
+        let failedBatches = [];
+        const totalBatches = Math.ceil(fileContents.length / BATCH_SIZE);
+        // Create all batches upfront
+        const batches = [];
         for (let i = 0; i < fileContents.length; i += BATCH_SIZE) {
-            const batch = fileContents.slice(i, i + BATCH_SIZE);
-            const batchNum = Math.floor(i / BATCH_SIZE) + 1;
-            const totalBatches = Math.ceil(fileContents.length / BATCH_SIZE);
-            spinner.text = `Indexing ${batchNum}/${totalBatches}...`;
-            // Retry logic for transient failures
-            let retries = 3;
-            let lastError = null;
-            while (retries > 0) {
+            batches.push({
+                batchNum: Math.floor(i / BATCH_SIZE) + 1,
+                files: fileContents.slice(i, i + BATCH_SIZE),
+            });
+        }
+        // Process a single batch with retries
+        async function processBatch(batch) {
+            for (let retry = 0; retry < MAX_RETRIES; retry++) {
                 try {
-                    const response = await axios.post(`${ENGINE_URL}/api/v1/index`, {
+                    const response = await axios.post(`${ENGINE_URL}/api/v1/index_simple`, {
                         repository_id: repoId,
                         branch: 'main',
-                        files: batch,
+                        files: batch.files,
                     }, {
-                        headers: {
-                            'Content-Type': 'application/json',
-                        },
-                        timeout: 120000,
+                        headers: { 'Content-Type': 'application/json' },
+                        timeout: TIMEOUT_MS,
+                        maxContentLength: Infinity,
+                        maxBodyLength: Infinity,
                     });
-                    if (!response.data.success) {
-                        spinner.fail(chalk.red('✗ Indexing failed'));
-                        console.log(chalk.red(response.data.result?.errors?.join('\n') || 'Unknown error'));
-                        return;
+                    if (response.data.success) {
+                        return {
+                            success: true,
+                            batchNum: batch.batchNum,
+                            chunksCreated: response.data.result?.chunks_created || 0,
+                            filesCount: batch.files.length,
+                        };
+                    }
+                    else {
+                        const errorMsg = response.data.result?.errors?.join(', ') || 'Unknown API error';
+                        if (retry < MAX_RETRIES - 1) {
+                            await sleep(RETRY_DELAY_MS * (retry + 1));
+                        }
+                        else {
+                            return { success: false, batchNum: batch.batchNum, chunksCreated: 0, filesCount: 0, error: errorMsg };
+                        }
                     }
-                    totalChunksCreated += response.data.result?.chunks_created || 0;
-                    uploadedCount += batch.length;
-                    break;
                 }
                 catch (err) {
-                    lastError = err;
-                    retries--;
-                    if (retries > 0) {
-                        spinner.text = `Retrying batch ${batchNum}... (${retries} left)`;
-                        await new Promise(r => setTimeout(r, 2000));
+                    if (retry < MAX_RETRIES - 1) {
+                        await sleep(RETRY_DELAY_MS * (retry + 1));
+                    }
+                    else {
+                        const errorMsg = err.response?.data?.error || err.message || 'Unknown error';
+                        return { success: false, batchNum: batch.batchNum, chunksCreated: 0, filesCount: 0, error: errorMsg };
                     }
                 }
             }
-            if (retries === 0 && lastError) {
-                throw lastError;
+            return { success: false, batchNum: batch.batchNum, chunksCreated: 0, filesCount: 0, error: 'Max retries exceeded' };
+        }
+        // Process batches in parallel waves
+        let completedBatches = 0;
+        for (let i = 0; i < batches.length; i += CONCURRENT_BATCHES) {
+            const wave = batches.slice(i, i + CONCURRENT_BATCHES);
+            spinner.text = `Indexing batches ${i + 1}-${Math.min(i + CONCURRENT_BATCHES, batches.length)}/${totalBatches} (${CONCURRENT_BATCHES} concurrent)...`;
+            // Fire all batches in this wave concurrently
+            const results = await Promise.all(wave.map(processBatch));
+            // Process results
+            for (const result of results) {
+                completedBatches++;
+                if (result.success) {
+                    uploadedCount += result.filesCount;
+                    totalChunksCreated += result.chunksCreated;
+                }
+                else {
+                    failedBatches.push(result.batchNum);
+                    console.log(chalk.yellow(`\n⚠️  Batch ${result.batchNum} failed: ${result.error}`));
+                }
             }
+            spinner.text = `Indexed ${completedBatches}/${totalBatches} batches (${uploadedCount} files, ${totalChunksCreated} chunks)...`;
+        }
+        // Summary
+        spinner.stop();
+        if (failedBatches.length === 0) {
+            console.log(chalk.green('\n✓ Indexing complete!'));
+        }
+        else if (failedBatches.length < totalBatches) {
+            console.log(chalk.yellow(`\n⚠️  Indexing partially complete (${failedBatches.length}/${totalBatches} batches failed)`));
+        }
+        else {
+            console.log(chalk.red('\n✗ Indexing failed - all batches failed'));
+            process.exit(1);
         }
-        spinner.succeed(chalk.green('✓ Indexing complete'));
         console.log(chalk.blue('📊 Repository ID:'), chalk.bold(repoId));
-        console.log(chalk.blue('📁 Files indexed:'), chalk.bold(uploadedCount));
+        console.log(chalk.blue('📁 Files indexed:'), chalk.bold(uploadedCount), chalk.dim(`/ ${fileContents.length}`));
         console.log(chalk.blue('📝 Chunks created:'), chalk.bold(totalChunksCreated));
+        if (failedBatches.length > 0) {
+            console.log(chalk.yellow('⚠️  Failed batches:'), chalk.bold(failedBatches.join(', ')));
+            console.log(chalk.dim('\nTip: Run the index command again to retry failed files.'));
+        }
         console.log(chalk.dim('\nUse this repo_id for searches'));
+        // Exit successfully - prevents hanging due to open handles from prompts/axios
+        process.exit(0);
     }
     catch (error) {
         spinner.fail(chalk.red('✗ Indexing failed'));
@@ -134,27 +205,44 @@ export async function index(options) {
         else {
             console.log(chalk.red(error.message));
         }
+        // Exit with error code
+        process.exit(1);
     }
 }
-async function collectFiles(dir) {
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+async function collectFiles(dir, verbose = false) {
     const files = [];
     const ignorePatterns = await getGitIgnorePatterns(dir);
+    const ignoredDirs = [];
     async function walk(currentPath) {
-        const entries = await fs.readdir(currentPath, { withFileTypes: true });
-        for (const entry of entries) {
-            const fullPath = path.join(currentPath, entry.name);
-            const relativePath = path.relative(dir, fullPath);
-            if (shouldIgnoreFile(relativePath, ignorePatterns)) {
-                continue;
-            }
-            if (entry.isDirectory()) {
-                await walk(fullPath);
-            }
-            else if (entry.isFile()) {
-                files.push(fullPath);
+        try {
+            const entries = await fs.readdir(currentPath, { withFileTypes: true });
+            for (const entry of entries) {
+                const fullPath = path.join(currentPath, entry.name);
+                const relativePath = path.relative(dir, fullPath);
+                if (shouldIgnoreFile(relativePath, ignorePatterns)) {
+                    if (verbose && entry.isDirectory()) {
+                        ignoredDirs.push(relativePath);
+                    }
+                    continue;
+                }
+                if (entry.isDirectory()) {
+                    await walk(fullPath);
+                }
+                else if (entry.isFile()) {
+                    files.push(fullPath);
+                }
             }
         }
+        catch (error) {
+            // Skip directories we can't read (permissions, etc.)
+        }
     }
     await walk(dir);
+    if (verbose && ignoredDirs.length > 0) {
+        console.log(chalk.dim(`\n   Ignored directories: ${ignoredDirs.slice(0, 20).join(', ')}${ignoredDirs.length > 20 ? '...' : ''}`));
+    }
     return files;
 }

package/dist/index.js

@@ -10,8 +10,8 @@ import { getConfig, saveConfig } from './utils/config.js';
 import { isValidDirectory, getGitIgnorePatterns, shouldIgnoreFile } from './utils/files.js';
 const BACKEND_URL = process.env.OPTIQ_BACKEND_URL || 'https://api.optiqcode.com';
 const ENGINE_URL = process.env.OPTIQ_ENGINE_URL || 'http://25.36.113.3:3002';
-// Debug mode - set OPTIQ_DEBUG=1 to see request details
-const DEBUG = process.env.OPTIQ_DEBUG === '1';
+// Debug mode - set OPTIQ_DEBUG=1 or OPTIQ_DEBUG=true to see request details
+const DEBUG = process.env.OPTIQ_DEBUG === '1' || process.env.OPTIQ_DEBUG === 'true' || process.env.OPTIQ_DEBUG?.toLowerCase() === 'true';
 // Helper to generate repository ID from path
 function generateRepoId(targetPath) {
     // Use path basename + hash of full path for uniqueness
@@ -267,10 +267,13 @@ async function main() {
     }
 }
 async function indexOnce(targetPath, config) {
+    console.log(chalk.cyan('\n🚀 Starting index...\n'));
     const spinner = ora({ text: 'Collecting files...', color: 'cyan' }).start();
     try {
+        spinner.stop();
         const files = await collectFiles(targetPath);
-        spinner.text = `Reading ${files.length} files...`;
+        console.log(chalk.cyan(`\n📁 Collected ${files.length} files to index\n`));
+        spinner.start(`Reading ${files.length} files...`);
         // Read files in parallel (100 at a time)
         const PARALLEL_READS = 100;
         const MAX_FILE_SIZE = 100_000; // 100KB max per file
@@ -302,7 +305,7 @@ async function indexOnce(targetPath, config) {
         const BATCH_SIZE = 50;
         let totalChunks = 0;
         if (DEBUG) {
-            console.log(chalk.gray(`\n[DEBUG] Sending to: ${ENGINE_URL}/api/v1/index`));
+            console.log(chalk.gray(`\n[DEBUG] Sending to: ${ENGINE_URL}/api/v1/index_simple`));
             console.log(chalk.gray(`[DEBUG] Repo ID: ${repoId}`));
             console.log(chalk.gray(`[DEBUG] Total files: ${filesArray.length}`));
         }
@@ -311,75 +314,102 @@ async function indexOnce(targetPath, config) {
         if (DEBUG) {
             console.log(chalk.gray(`\n[DEBUG] Total files: ${filesArray.length}, Batch size: ${BATCH_SIZE}, Total batches: ${totalBatches}`));
         }
+        // PARALLEL BATCH PROCESSING - send multiple batches concurrently
+        const CONCURRENT_BATCHES = 4;
+        const MAX_RETRIES = 3;
+        const RETRY_DELAY_MS = 2000;
+        // Create all batches upfront
+        const batches = [];
         for (let i = 0; i < filesArray.length; i += BATCH_SIZE) {
-            const batch = filesArray.slice(i, i + BATCH_SIZE);
-            const batchNum = Math.floor(i / BATCH_SIZE) + 1;
-            const filesProcessed = Math.min(i + BATCH_SIZE, filesArray.length);
-            spinner.text = `Indexing... ${filesProcessed}/${filesArray.length} files (batch ${batchNum}/${totalBatches})`;
-            if (DEBUG) {
-                console.log(chalk.gray(`\n[DEBUG] Starting batch ${batchNum}/${totalBatches} (files ${i + 1}-${Math.min(i + BATCH_SIZE, filesArray.length)})`));
-                console.log(chalk.gray(`[DEBUG] Batch files: ${batch.map(f => f.path).join(', ')}`));
-            }
-            // Retry logic for transient failures
-            let retries = 3;
-            let lastError = null;
-            while (retries > 0) {
+            batches.push({
+                batchNum: Math.floor(i / BATCH_SIZE) + 1,
+                files: filesArray.slice(i, i + BATCH_SIZE),
+            });
+        }
+        if (DEBUG) {
+            console.log(chalk.gray(`\n[DEBUG] Created ${batches.length} batches, processing ${CONCURRENT_BATCHES} concurrently`));
+        }
+        // Process a single batch with retries
+        async function processBatch(batch) {
+            for (let retry = 0; retry < MAX_RETRIES; retry++) {
                 try {
-                    const response = await axios.post(`${ENGINE_URL}/api/v1/index`, {
+                    const response = await axios.post(`${ENGINE_URL}/api/v1/index_simple`, {
                         repository_id: repoId,
                         branch: 'main',
-                        files: batch,
+                        files: batch.files,
                     }, {
-                        headers: {
-                            'Content-Type': 'application/json',
-                        },
-                        timeout: 300000, // 5 minutes per batch
+                        headers: { 'Content-Type': 'application/json' },
+                        timeout: 300000,
                     });
-                    if (DEBUG) {
-                        console.log(chalk.gray(`[DEBUG] Batch ${batchNum} response: success=${response.data.success}, chunks=${response.data.result?.chunks_created}, errors=${JSON.stringify(response.data.result?.errors || [])}`));
-                    }
-                    if (!response.data.success) {
-                        spinner.fail(chalk.red('Indexing failed'));
-                        console.log(chalk.gray(response.data.result?.errors?.join('\n') || 'Unknown error'));
-                        return;
+                    if (response.data.success) {
+                        return {
+                            success: true,
+                            batchNum: batch.batchNum,
+                            chunksCreated: response.data.result?.chunks_created || 0,
+                        };
                     }
-                    totalChunks += response.data.result?.chunks_created || 0;
-                    if (DEBUG) {
-                        console.log(chalk.gray(`[DEBUG] Batch ${batchNum} complete. Total chunks so far: ${totalChunks}`));
+                    else {
+                        const errorMsg = response.data.result?.errors?.join(', ') || 'Unknown API error';
+                        if (retry < MAX_RETRIES - 1) {
+                            await new Promise(r => setTimeout(r, RETRY_DELAY_MS * (retry + 1)));
+                        }
+                        else {
+                            return { success: false, batchNum: batch.batchNum, chunksCreated: 0, error: errorMsg };
+                        }
                     }
-                    break;
                 }
                 catch (err) {
-                    lastError = err;
-                    retries--;
-                    // Stop spinner to show error clearly
-                    spinner.stop();
-                    console.log(chalk.red(`\n✗ Batch ${batchNum} failed: ${err.code || err.message}`));
-                    console.log(chalk.yellow(`  Files: ${batch.map(f => f.path).join(', ')}`));
-                    console.log(chalk.gray(`  Sizes: ${batch.map(f => `${f.path}(${Math.round(f.content.length / 1024)}KB)`).join(', ')}`));
-                    if (DEBUG && err.response) {
-                        console.log(chalk.gray(`[DEBUG] Response status: ${err.response.status}`));
-                        console.log(chalk.gray(`[DEBUG] Response data: ${JSON.stringify(err.response.data)}`));
+                    if (retry < MAX_RETRIES - 1) {
+                        await new Promise(r => setTimeout(r, RETRY_DELAY_MS * (retry + 1)));
                     }
-                    if (retries > 0) {
-                        console.log(chalk.gray(`  Retrying... (${retries} left)\n`));
-                        await new Promise(r => setTimeout(r, 2000));
-                        spinner.start(`Retrying batch ${batchNum}...`);
+                    else {
+                        const errorMsg = err.response?.data?.error || err.message || 'Unknown error';
+                        return { success: false, batchNum: batch.batchNum, chunksCreated: 0, error: errorMsg };
                     }
                 }
             }
-            if (retries === 0 && lastError) {
-                throw lastError;
+            return { success: false, batchNum: batch.batchNum, chunksCreated: 0, error: 'Max retries exceeded' };
+        }
+        // Process batches in parallel waves
+        let completedBatches = 0;
+        let failedBatches = [];
+        for (let i = 0; i < batches.length; i += CONCURRENT_BATCHES) {
+            const wave = batches.slice(i, i + CONCURRENT_BATCHES);
+            const waveEnd = Math.min(i + CONCURRENT_BATCHES, batches.length);
+            spinner.text = `Indexing batches ${i + 1}-${waveEnd}/${totalBatches} (${CONCURRENT_BATCHES} concurrent)...`;
+            if (DEBUG) {
+                console.log(chalk.gray(`\n[DEBUG] Starting wave: batches ${i + 1}-${waveEnd}`));
+            }
+            // Fire all batches in this wave concurrently
+            const results = await Promise.all(wave.map(processBatch));
+            // Process results
+            spinner.stop();
+            for (const result of results) {
+                completedBatches++;
+                if (result.success) {
+                    totalChunks += result.chunksCreated;
+                    console.log(chalk.green(`✓ Batch ${result.batchNum}/${totalBatches}: ${result.chunksCreated} chunks`));
+                }
+                else {
+                    failedBatches.push(result.batchNum);
+                    console.log(chalk.red(`✗ Batch ${result.batchNum}/${totalBatches} failed: ${result.error}`));
+                }
             }
+            spinner.start(`Indexed ${completedBatches}/${totalBatches} batches (${totalChunks} chunks)...`);
         }
         if (DEBUG) {
             console.log(chalk.gray(`\n[DEBUG] All ${totalBatches} batches complete. Total chunks: ${totalChunks}`));
         }
+        if (failedBatches.length > 0) {
+            console.log(chalk.yellow(`\n⚠️  ${failedBatches.length} batches failed: ${failedBatches.join(', ')}`));
+        }
         spinner.succeed(chalk.cyan('Indexed'));
         console.log(chalk.gray(`   ${filesArray.length} files • ${totalChunks} chunks`));
         console.log(chalk.cyan('\n📊 Repository ID:'));
         console.log(chalk.white(`   ${repoId}`));
         console.log(chalk.gray('\n   Use this ID for searches\n'));
+        // Exit successfully - prevents hanging due to open handles from prompts/axios
+        process.exit(0);
     }
     catch (error) {
         spinner.fail(chalk.red('Failed'));
@@ -389,6 +419,8 @@ async function indexOnce(targetPath, config) {
         else {
             console.log(chalk.gray('   ' + error.message));
         }
+        // Exit with error code
+        process.exit(1);
     }
 }
 async function watchDirectory(targetPath, config) {
@@ -425,52 +457,51 @@ async function watchDirectory(targetPath, config) {
         if (DEBUG) {
             console.log(chalk.gray(`\n[DEBUG] Watch initial index: ${filesArray.length} files, ${totalBatches} batches`));
         }
+        // PARALLEL BATCH PROCESSING for initial watch index
+        const CONCURRENT_BATCHES = 4;
+        const MAX_RETRIES = 3;
+        const RETRY_DELAY_MS = 2000;
+        // Create all batches
+        const batches = [];
         for (let i = 0; i < filesArray.length; i += BATCH_SIZE) {
-            const batch = filesArray.slice(i, i + BATCH_SIZE);
-            const batchNum = Math.floor(i / BATCH_SIZE) + 1;
-            spinner.text = `Indexing... ${batchNum}/${totalBatches}`;
-            if (DEBUG) {
-                console.log(chalk.gray(`[DEBUG] Watch batch ${batchNum}/${totalBatches} starting...`));
-            }
-            // Retry logic for transient failures
-            let retries = 3;
-            let lastError = null;
-            while (retries > 0) {
+            batches.push({
+                batchNum: Math.floor(i / BATCH_SIZE) + 1,
+                files: filesArray.slice(i, i + BATCH_SIZE),
+            });
+        }
+        if (DEBUG) {
+            console.log(chalk.gray(`\n[DEBUG] Watch initial: ${batches.length} batches, ${CONCURRENT_BATCHES} concurrent`));
+        }
+        // Process a single batch with retries
+        async function processWatchBatch(batch) {
+            for (let retry = 0; retry < MAX_RETRIES; retry++) {
                 try {
-                    const response = await axios.post(`${ENGINE_URL}/api/v1/index`, {
-                        repository_id: repoId,
-                        branch: 'main',
-                        files: batch,
-                    }, {
-                        headers: {
-                            'Content-Type': 'application/json',
-                        },
-                        timeout: 300000, // 5 minutes per batch
-                    });
-                    if (DEBUG) {
-                        console.log(chalk.gray(`[DEBUG] Watch batch ${batchNum} response: success=${response.data.success}, chunks=${response.data.result?.chunks_created}`));
-                    }
-                    if (!response.data.success) {
-                        spinner.fail(chalk.red('Failed'));
-                        console.log(chalk.gray(response.data.result?.errors?.join('\n') || 'Unknown error'));
-                        return;
-                    }
-                    break;
+                    const response = await axios.post(`${ENGINE_URL}/api/v1/index_simple`, { repository_id: repoId, branch: 'main', files: batch.files }, { headers: { 'Content-Type': 'application/json' }, timeout: 300000 });
+                    if (response.data.success)
+                        return true;
+                    if (retry < MAX_RETRIES - 1)
+                        await new Promise(r => setTimeout(r, RETRY_DELAY_MS * (retry + 1)));
                 }
                 catch (err) {
-                    lastError = err;
-                    retries--;
-                    if (DEBUG) {
-                        console.log(chalk.gray(`[DEBUG] Watch batch ${batchNum} error: ${err.message}`));
-                    }
-                    if (retries > 0) {
-                        spinner.text = `Retrying batch ${batchNum}... (${retries} left)`;
-                        await new Promise(r => setTimeout(r, 2000));
-                    }
+                    if (DEBUG)
+                        console.log(chalk.gray(`[DEBUG] Watch batch ${batch.batchNum} error: ${err.message}`));
+                    if (retry < MAX_RETRIES - 1)
+                        await new Promise(r => setTimeout(r, RETRY_DELAY_MS * (retry + 1)));
                 }
             }
-            if (retries === 0 && lastError) {
-                throw lastError;
+            return false;
+        }
+        // Process in parallel waves
+        let completedBatches = 0;
+        for (let i = 0; i < batches.length; i += CONCURRENT_BATCHES) {
+            const wave = batches.slice(i, i + CONCURRENT_BATCHES);
+            spinner.text = `Indexing... ${Math.min(i + CONCURRENT_BATCHES, batches.length)}/${totalBatches} batches`;
+            const results = await Promise.all(wave.map(processWatchBatch));
+            completedBatches += results.length;
+            const failures = results.filter(r => !r).length;
+            if (failures > 0) {
+                spinner.fail(chalk.red(`Failed (${failures} batches failed)`));
+                return;
             }
         }
         if (DEBUG) {
@@ -589,10 +620,10 @@ async function watchDirectory(targetPath, config) {
             }
             // Filter out deleted files (content: null) - send only files with content
             const filesToIndex = filesArray.filter(f => f.content !== null);
-            // For incremental updates, use the same /api/v1/index endpoint
+            // For incremental updates, use the simplified index endpoint
             // The Rust engine handles upserts automatically
             if (filesToIndex.length > 0) {
-                const response = await axios.post(`${ENGINE_URL}/api/v1/index`, {
+                const response = await axios.post(`${ENGINE_URL}/api/v1/index_simple`, {
                     repository_id: repoId,
                     branch: 'main',
                     files: filesToIndex,
@@ -689,23 +720,125 @@ async function watchDirectory(targetPath, config) {
 async function collectFiles(dir) {
     const files = [];
     const ignorePatterns = await getGitIgnorePatterns(dir);
+    const ignoredItems = [];
+    // Always log ignore patterns for debugging
+    console.log(chalk.gray('\n📋 Checking ignore patterns...'));
     async function walk(currentPath) {
-        const entries = await fs.readdir(currentPath, { withFileTypes: true });
-        for (const entry of entries) {
-            const fullPath = path.join(currentPath, entry.name);
-            const relativePath = path.relative(dir, fullPath);
-            if (shouldIgnoreFile(relativePath, ignorePatterns)) {
-                continue;
+        try {
+            const entries = await fs.readdir(currentPath, { withFileTypes: true });
+            for (const entry of entries) {
+                const fullPath = path.join(currentPath, entry.name);
+                const relativePath = path.relative(dir, fullPath);
+                if (shouldIgnoreFile(relativePath, ignorePatterns)) {
+                    // Log ignored directories (especially admin!)
+                    if (entry.isDirectory()) {
+                        ignoredItems.push(`DIR: ${relativePath}`);
+                        // Special check for admin
+                        if (relativePath.includes('admin')) {
+                            console.log(chalk.yellow(`⚠️  IGNORING admin directory: ${relativePath}`));
+                            // Find which pattern matched
+                            for (const pattern of ignorePatterns) {
+                                if (relativePath.includes(pattern) || pattern.includes('admin')) {
+                                    console.log(chalk.yellow(`   Matched pattern: "${pattern}"`));
+                                }
+                            }
+                        }
+                    }
+                    continue;
+                }
+                if (entry.isDirectory()) {
+                    await walk(fullPath);
+                }
+                else if (entry.isFile()) {
+                    files.push(fullPath);
+                    // Log admin files being collected
+                    if (relativePath.includes('admin')) {
+                        console.log(chalk.green(`✓ Found admin file: ${relativePath}`));
+                    }
+                }
             }
-            if (entry.isDirectory()) {
-                await walk(fullPath);
+        }
+        catch (error) {
+            console.log(chalk.yellow(`⚠️  Cannot read directory: ${currentPath}`));
+        }
+    }
+    await walk(dir);
+    // Show summary of ignored directories
+    if (ignoredItems.length > 0) {
+        console.log(chalk.gray(`\n📁 Ignored ${ignoredItems.length} items`));
+        // Show first 20
+        for (const item of ignoredItems.slice(0, 20)) {
+            console.log(chalk.gray(`   ${item}`));
+        }
+        if (ignoredItems.length > 20) {
+            console.log(chalk.gray(`   ... and ${ignoredItems.length - 20} more`));
+        }
+    }
+    // Check if any admin files were found
+    const adminFiles = files.filter(f => f.includes('admin'));
+    if (adminFiles.length > 0) {
+        console.log(chalk.green(`\n✓ Found ${adminFiles.length} files in admin directories`));
+    }
+    else {
+        console.log(chalk.yellow(`\n⚠️  No admin files found! Check if admin is being ignored.`));
+    }
+    console.log('');
+    // Log first 10 ignore patterns from gitignore
+    const gitignorePatterns = ignorePatterns.slice(180); // Skip default patterns
+    if (gitignorePatterns.length > 0) {
+        console.log(chalk.gray('📋 .gitignore patterns:'));
+        for (const p of gitignorePatterns.slice(0, 15)) {
+            console.log(chalk.gray(`   ${p}`));
+        }
+        if (gitignorePatterns.length > 15) {
+            console.log(chalk.gray(`   ... and ${gitignorePatterns.length - 15} more`));
+        }
+        console.log('');
+    }
+    return files;
+}
+// Keep old version for reference - DELETE THIS
+async function collectFilesOld(dir) {
+    const files = [];
+    const ignorePatterns = await getGitIgnorePatterns(dir);
+    const ignoredDirs = [];
+    async function walk(currentPath) {
+        try {
+            const entries = await fs.readdir(currentPath, { withFileTypes: true });
+            for (const entry of entries) {
+                const fullPath = path.join(currentPath, entry.name);
+                const relativePath = path.relative(dir, fullPath);
+                if (shouldIgnoreFile(relativePath, ignorePatterns)) {
+                    if (DEBUG && entry.isDirectory()) {
+                        ignoredDirs.push(relativePath);
+                    }
+                    continue;
+                }
+                if (entry.isDirectory()) {
+                    await walk(fullPath);
+                }
+                else if (entry.isFile()) {
+                    files.push(fullPath);
+                }
             }
-            else if (entry.isFile()) {
-                files.push(fullPath);
+        }
+        catch (error) {
+            // Skip directories we can't read
+            if (DEBUG) {
+                console.log(chalk.yellow(`[DEBUG] Cannot read directory: ${currentPath}`));
             }
         }
     }
     await walk(dir);
+    if (DEBUG && ignoredDirs.length > 0) {
+        console.log(chalk.gray(`\n[DEBUG] Ignored directories (${ignoredDirs.length}):`));
+        for (const d of ignoredDirs.slice(0, 30)) {
+            console.log(chalk.gray(`  - ${d}`));
+        }
+        if (ignoredDirs.length > 30) {
+            console.log(chalk.gray(`  ... and ${ignoredDirs.length - 30} more`));
+        }
+    }
     return files;
 }
 main().catch((error) => {

package/dist/utils/files.js

@@ -51,7 +51,8 @@ const DEFAULT_IGNORE_PATTERNS = [
     'build',
     'eggs',
     '.eggs',
-    'lib',
+    // NOTE: 'lib' removed - it's a common source directory in JS/TS projects
+    // Python's lib/ is usually inside venv/ which is already ignored
     'lib64',
     'parts',
     'sdist',
@@ -303,16 +304,33 @@ export async function getGitIgnorePatterns(dir) {
     }
     return patterns;
 }
+// Patterns that should only match at root level (not nested directories)
+const ROOT_ONLY_PATTERNS = new Set([
+    // NOTE: 'lib' removed - it's a common source directory in JS/TS projects
+    'lib64', 'bin', 'out', 'build', 'dist', 'vendor', 'tmp', 'temp',
+    'env', 'venv', 'ENV', '.venv', 'eggs', '.eggs', 'parts', 'sdist', 'var', 'wheels',
+    'deps', 'packages', 'site', 'coverage', 'htmlcov',
+]);
 export function shouldIgnoreFile(relativePath, patterns) {
     // Normalize path separators to forward slashes for consistent matching
     const normalizedPath = relativePath.replace(/\\/g, '/');
     const parts = normalizedPath.split('/');
     const fileName = parts[parts.length - 1];
+    const firstPart = parts[0]; // Root-level directory
     for (const pattern of patterns) {
         // Skip empty patterns
         if (!pattern)
             continue;
+        // For root-only patterns, only match if it's the first directory component
+        if (ROOT_ONLY_PATTERNS.has(pattern)) {
+            if (firstPart === pattern) {
+                return true;
+            }
+            // Don't match nested directories like frontend/src/lib
+            continue;
+        }
         // Exact directory/file name match (e.g., 'node_modules', '.git')
+        // These match anywhere in the path
         if (parts.includes(pattern)) {
             return true;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@optiqcode/cli",
-  "version": "2.9.0",
+  "version": "3.0.1",
   "description": "CLI tool for Optiq - automatic code indexing and context engine",
   "type": "module",
   "main": "dist/index.js",