@optiqcode/cli 2.0.0 → 2.1.0

package/dist/commands/index.js

@@ -6,7 +6,15 @@ import path from 'path';
 import fs from 'fs/promises';
 import { getConfig } from '../utils/config.js';
 import { isValidDirectory, getGitIgnorePatterns, shouldIgnoreFile } from '../utils/files.js';
-const API_URL = process.env.OPTIQ_BACKEND_URL ? `${process.env.OPTIQ_BACKEND_URL}/api` : 'https://api.optiqcode.com/api';
+const ENGINE_URL = process.env.OPTIQ_ENGINE_URL || 'https://api.optiqcode.com';
+// Helper to generate repository ID from path
+function generateRepoId(targetPath) {
+    const basename = path.basename(targetPath);
+    const hash = targetPath.split('').reduce((acc, char) => {
+        return ((acc << 5) - acc) + char.charCodeAt(0) | 0;
+    }, 0);
+    return `${basename}-${Math.abs(hash).toString(16)}`;
+}
 export async function index(options) {
     const config = await getConfig();
     if (!config) {
@@ -63,43 +71,41 @@ export async function index(options) {
                 // Skip files that can't be read
             }
         }
-        spinner.text = 'Uploading to Optiq...';
-        // Batch upload files (50 at a time to avoid 413 errors)
-        const BATCH_SIZE = 50;
+        spinner.text = 'Indexing with Optiq Engine...';
+        // Generate repository ID from path
+        const repoId = generateRepoId(targetPath);
+        // Batch upload files (150 at a time for parallel processing)
+        const BATCH_SIZE = 150;
         let uploadedCount = 0;
-        let repoId = '';
-        let totalFilesProcessed = 0;
-        let totalEntitiesIndexed = 0;
+        let totalChunksCreated = 0;
         for (let i = 0; i < fileContents.length; i += BATCH_SIZE) {
             const batch = fileContents.slice(i, i + BATCH_SIZE);
             const batchNum = Math.floor(i / BATCH_SIZE) + 1;
             const totalBatches = Math.ceil(fileContents.length / BATCH_SIZE);
-            spinner.text = `Uploading batch ${batchNum}/${totalBatches} (${uploadedCount}/${fileContents.length} files)...`;
-            const response = await axios.post(`${API_URL}/nexus/index/content`, {
-                repository_path: targetPath,
+            spinner.text = `Indexing batch ${batchNum}/${totalBatches} (${uploadedCount}/${fileContents.length} files)...`;
+            const response = await axios.post(`${ENGINE_URL}/api/v1/index`, {
+                repository_id: repoId,
+                branch: 'main',
                 files: batch,
             }, {
                 headers: {
-                    Authorization: `Bearer ${config.apiKey}`,
                     'Content-Type': 'application/json',
                 },
                 timeout: 0, // No timeout for large codebases
             });
             if (!response.data.success) {
                 spinner.fail(chalk.red('✗ Indexing failed'));
-                console.log(chalk.red(response.data.error || 'Unknown error'));
+                console.log(chalk.red(response.data.result?.errors?.join('\n') || 'Unknown error'));
                 return;
             }
-            repoId = response.data.repo_id;
-            totalFilesProcessed += response.data.files_processed || 0;
-            totalEntitiesIndexed += response.data.entities_indexed || 0;
+            totalChunksCreated += response.data.result?.chunks_created || 0;
             uploadedCount += batch.length;
         }
         spinner.succeed(chalk.green('✓ Indexing complete'));
         console.log(chalk.blue('📊 Repository ID:'), chalk.bold(repoId));
-        console.log(chalk.blue('📁 Files indexed:'), chalk.bold(totalFilesProcessed));
-        console.log(chalk.blue('📝 Entities indexed:'), chalk.bold(totalEntitiesIndexed));
-        console.log(chalk.dim('\nUse this repo_id with the MCP server or API'));
+        console.log(chalk.blue('📁 Files indexed:'), chalk.bold(uploadedCount));
+        console.log(chalk.blue('📝 Chunks created:'), chalk.bold(totalChunksCreated));
+        console.log(chalk.dim('\nUse this repo_id for searches'));
     }
     catch (error) {
         spinner.fail(chalk.red('✗ Indexing failed'));

package/dist/index.js

@@ -9,6 +9,16 @@ import fs from 'fs/promises';
 import { getConfig, saveConfig } from './utils/config.js';
 import { isValidDirectory, getGitIgnorePatterns, shouldIgnoreFile } from './utils/files.js';
 const BACKEND_URL = process.env.OPTIQ_BACKEND_URL || 'https://api.optiqcode.com';
+const ENGINE_URL = process.env.OPTIQ_ENGINE_URL || 'https://api.optiqcode.com';
+// Helper to generate repository ID from path
+function generateRepoId(targetPath) {
+    // Use path basename + hash of full path for uniqueness
+    const basename = path.basename(targetPath);
+    const hash = targetPath.split('').reduce((acc, char) => {
+        return ((acc << 5) - acc) + char.charCodeAt(0) | 0;
+    }, 0);
+    return `${basename}-${Math.abs(hash).toString(16)}`;
+}
 async function showBanner() {
     console.clear();
     console.log(chalk.white.bold(`
@@ -277,41 +287,40 @@ async function indexOnce(targetPath, config) {
             }
             spinner.text = `Reading... ${Math.min(i + PARALLEL_READS, files.length)}/${files.length}`;
         }
+        // Generate repository ID from path
+        const repoId = generateRepoId(targetPath);
         // Upload in larger batches (150 files per batch)
         const BATCH_SIZE = 150;
-        let totalFiles = 0;
-        let totalEntities = 0;
-        let repoId = '';
-        spinner.text = 'Uploading...';
+        let totalChunks = 0;
+        spinner.text = 'Indexing...';
         for (let i = 0; i < filesArray.length; i += BATCH_SIZE) {
             const batch = filesArray.slice(i, i + BATCH_SIZE);
             const batchNum = Math.floor(i / BATCH_SIZE) + 1;
             const totalBatches = Math.ceil(filesArray.length / BATCH_SIZE);
-            spinner.text = `Uploading... ${batchNum}/${totalBatches}`;
-            const response = await axios.post(`${BACKEND_URL}/api/nexus/index/content`, {
-                repository_path: targetPath,
+            spinner.text = `Indexing... ${batchNum}/${totalBatches} (parallel processing)`;
+            // Call the Rust engine's /api/v1/index endpoint
+            const response = await axios.post(`${ENGINE_URL}/api/v1/index`, {
+                repository_id: repoId,
+                branch: 'main',
                 files: batch,
             }, {
                 headers: {
-                    'X-API-Key': config.apiKey,
                     'Content-Type': 'application/json',
                 },
-                timeout: 0,
+                timeout: 0, // No timeout - parallel processing handles large batches
             });
             if (!response.data.success) {
                 spinner.fail(chalk.red('Indexing failed'));
-                console.log(chalk.gray(response.data.error || 'Unknown error'));
+                console.log(chalk.gray(response.data.result?.errors?.join('\n') || 'Unknown error'));
                 return;
             }
-            repoId = response.data.repo_id;
-            totalFiles += response.data.stats?.files_indexed || 0;
-            totalEntities += response.data.stats?.entities_indexed || 0;
+            totalChunks += response.data.result?.chunks_created || 0;
         }
         spinner.succeed(chalk.cyan('Indexed'));
-        console.log(chalk.gray(`   ${totalFiles} files • ${totalEntities} entities`));
+        console.log(chalk.gray(`   ${filesArray.length} files • ${totalChunks} chunks`));
         console.log(chalk.cyan('\n📊 Repository ID:'));
         console.log(chalk.white(`   ${repoId}`));
-        console.log(chalk.gray('\n   Use this ID with the MCP server\n'));
+        console.log(chalk.gray('\n   Use this ID for searches\n'));
     }
     catch (error) {
         spinner.fail(chalk.red('Failed'));
@@ -324,89 +333,65 @@ async function indexOnce(targetPath, config) {
     }
 }
 async function watchDirectory(targetPath, config) {
-    // Check if repo is already indexed
-    let repoId = null;
+    // Generate repository ID from path
+    let repoId = generateRepoId(targetPath);
+    // Do initial full index
+    const spinner = ora({ text: 'Collecting files...', color: 'cyan' }).start();
     try {
-        const repoListResponse = await axios.get(`${BACKEND_URL}/api/repositories`, {
-            headers: {
-                'X-API-Key': config.apiKey,
-            },
-            timeout: 10000,
-        });
-        if (repoListResponse.data.success && repoListResponse.data.repositories) {
-            const existingRepo = repoListResponse.data.repositories.find((r) => r.path === targetPath);
-            if (existingRepo) {
-                repoId = existingRepo.id;
-                console.log(chalk.cyan('✓ Already indexed'));
-                console.log(chalk.gray(`   Repo ID: ${repoId}\n`));
+        const files = await collectFiles(targetPath);
+        spinner.text = `Reading ${files.length} files...`;
+        // Read files in parallel (100 at a time)
+        const PARALLEL_READS = 100;
+        const filesArray = [];
+        for (let i = 0; i < files.length; i += PARALLEL_READS) {
+            const chunk = files.slice(i, i + PARALLEL_READS);
+            const results = await Promise.allSettled(chunk.map(async (file) => {
+                const content = await fs.readFile(file, 'utf-8');
+                const relativePath = path.relative(targetPath, file).replace(/\\/g, '/');
+                return { path: relativePath, content };
+            }));
+            for (const result of results) {
+                if (result.status === 'fulfilled') {
+                    filesArray.push(result.value);
+                }
             }
+            spinner.text = `Reading... ${Math.min(i + PARALLEL_READS, files.length)}/${files.length}`;
         }
+        // Upload in larger batches using Rust engine
+        const BATCH_SIZE = 150;
+        for (let i = 0; i < filesArray.length; i += BATCH_SIZE) {
+            const batch = filesArray.slice(i, i + BATCH_SIZE);
+            const batchNum = Math.floor(i / BATCH_SIZE) + 1;
+            const totalBatches = Math.ceil(filesArray.length / BATCH_SIZE);
+            spinner.text = `Indexing... ${batchNum}/${totalBatches} (parallel processing)`;
+            const response = await axios.post(`${ENGINE_URL}/api/v1/index`, {
+                repository_id: repoId,
+                branch: 'main',
+                files: batch,
+            }, {
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                timeout: 0,
+            });
+            if (!response.data.success) {
+                spinner.fail(chalk.red('Failed'));
+                console.log(chalk.gray(response.data.result?.errors?.join('\n') || 'Unknown error'));
+                return;
+            }
+        }
+        spinner.succeed(chalk.cyan(`Indexed ${files.length} files`));
+        console.log(chalk.gray(`   Repo ID: ${repoId}\n`));
     }
     catch (error) {
-        // Ignore errors, will do full index
-    }
-    // Only do full index if repo doesn't exist
-    if (!repoId) {
-        const spinner = ora({ text: 'Collecting files...', color: 'cyan' }).start();
-        try {
-            const files = await collectFiles(targetPath);
-            spinner.text = `Reading ${files.length} files...`;
-            // Read files in parallel (100 at a time)
-            const PARALLEL_READS = 100;
-            const filesArray = [];
-            for (let i = 0; i < files.length; i += PARALLEL_READS) {
-                const chunk = files.slice(i, i + PARALLEL_READS);
-                const results = await Promise.allSettled(chunk.map(async (file) => {
-                    const content = await fs.readFile(file, 'utf-8');
-                    const relativePath = path.relative(targetPath, file).replace(/\\/g, '/');
-                    return { path: relativePath, content };
-                }));
-                for (const result of results) {
-                    if (result.status === 'fulfilled') {
-                        filesArray.push(result.value);
-                    }
-                }
-                spinner.text = `Reading... ${Math.min(i + PARALLEL_READS, files.length)}/${files.length}`;
-            }
-            // Upload in larger batches
-            const BATCH_SIZE = 150;
-            let uploadedCount = 0;
-            for (let i = 0; i < filesArray.length; i += BATCH_SIZE) {
-                const batch = filesArray.slice(i, i + BATCH_SIZE);
-                const batchNum = Math.floor(i / BATCH_SIZE) + 1;
-                const totalBatches = Math.ceil(filesArray.length / BATCH_SIZE);
-                spinner.text = `Uploading... ${batchNum}/${totalBatches}`;
-                const response = await axios.post(`${BACKEND_URL}/api/nexus/index/content`, {
-                    repository_path: targetPath,
-                    files: batch,
-                }, {
-                    headers: {
-                        'X-API-Key': config.apiKey,
-                        'Content-Type': 'application/json',
-                    },
-                    timeout: 0,
-                });
-                if (!response.data.success) {
-                    spinner.fail(chalk.red('Failed'));
-                    console.log(chalk.gray(response.data.error || 'Unknown error'));
-                    return;
-                }
-                repoId = response.data.repo_id;
-                uploadedCount += batch.length;
-            }
-            spinner.succeed(chalk.cyan(`Indexed ${files.length} files`));
-            console.log(chalk.gray(`   Repo ID: ${repoId}\n`));
+        spinner.fail(chalk.red('Failed'));
+        if (error.response?.data?.error) {
+            console.log(chalk.gray('   ' + error.response.data.error));
         }
-        catch (error) {
-            spinner.fail(chalk.red('Failed'));
-            if (error.response?.data?.error) {
-                console.log(chalk.gray('   ' + error.response.data.error));
-            }
-            else {
-                console.log(chalk.gray('   ' + error.message));
-            }
-            return;
+        else {
+            console.log(chalk.gray('   ' + error.message));
         }
+        return;
     }
     const ignorePatterns = await getGitIgnorePatterns(targetPath);
     const watcher = chokidar.watch(targetPath, {
@@ -506,42 +491,52 @@ async function watchDirectory(targetPath, config) {
                 isProcessing = false;
                 return;
             }
-            const response = await axios.post(`${BACKEND_URL}/api/nexus/index/incremental`, {
-                repository_id: repoId,
-                repository_path: targetPath,
-                files: filesArray,
-            }, {
-                headers: {
-                    'X-API-Key': config.apiKey,
-                    'Content-Type': 'application/json',
-                },
-                timeout: 60000,
-            });
-            if (response.data.success) {
-                // Add successfully indexed files to the set
-                for (const file of filesArray) {
-                    if (file.content !== null) {
+            // Filter out deleted files (content: null) - send only files with content
+            const filesToIndex = filesArray.filter(f => f.content !== null);
+            // For incremental updates, use the same /api/v1/index endpoint
+            // The Rust engine handles upserts automatically
+            if (filesToIndex.length > 0) {
+                const response = await axios.post(`${ENGINE_URL}/api/v1/index`, {
+                    repository_id: repoId,
+                    branch: 'main',
+                    files: filesToIndex,
+                }, {
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                    timeout: 60000,
+                });
+                if (response.data.success) {
+                    // Add successfully indexed files to the set
+                    for (const file of filesToIndex) {
                         allIndexedFiles.add(file.path);
                     }
-                }
-                totalIndexed += filesArray.length;
-                lastIndexedTime = Date.now();
-                // Update last indexed file
-                if (filesArray.length === 1) {
-                    lastIndexedFile = filesArray[0].path;
+                    totalIndexed += filesToIndex.length;
+                    lastIndexedTime = Date.now();
+                    // Update last indexed file
+                    if (filesToIndex.length === 1) {
+                        lastIndexedFile = filesToIndex[0].path;
+                    }
+                    else {
+                        lastIndexedFile = `${filesToIndex.length} files`;
+                    }
+                    // Update dashboard
+                    showDashboard();
                 }
                 else {
-                    lastIndexedFile = `${filesArray.length} files`;
+                    // Clear dashboard, show error, redraw dashboard
+                    process.stdout.write('\x1b[5A');
+                    process.stdout.write('\x1b[0J');
+                    dashboardShown = false;
+                    console.log(chalk.red(`✗ Failed to index: ${response.data.result?.errors?.join(', ') || 'Unknown error'}\n`));
+                    showDashboard();
                 }
-                // Update dashboard
-                showDashboard();
             }
             else {
-                // Clear dashboard, show error, redraw dashboard
-                process.stdout.write('\x1b[5A');
-                process.stdout.write('\x1b[0J');
-                dashboardShown = false;
-                console.log(chalk.red(`✗ Failed to index: ${response.data.error}\n`));
+                // Only deletions - just update the dashboard
+                totalIndexed += filesArray.length;
+                lastIndexedTime = Date.now();
+                lastIndexedFile = `${filesArray.length} deleted`;
                 showDashboard();
             }
         }

package/dist/utils/files.js

@@ -304,22 +304,50 @@ export async function getGitIgnorePatterns(dir) {
     return patterns;
 }
 export function shouldIgnoreFile(relativePath, patterns) {
-    const parts = relativePath.split(path.sep);
+    // Normalize path separators to forward slashes for consistent matching
+    const normalizedPath = relativePath.replace(/\\/g, '/');
+    const parts = normalizedPath.split('/');
+    const fileName = parts[parts.length - 1];
     for (const pattern of patterns) {
-        // Exact match
+        // Skip empty patterns
+        if (!pattern)
+            continue;
+        // Exact directory/file name match (e.g., 'node_modules', '.git')
         if (parts.includes(pattern)) {
             return true;
         }
-        // Wildcard match
-        if (pattern.includes('*')) {
-            const regex = new RegExp('^' + pattern.replace(/\*/g, '.*') + '$');
-            if (regex.test(relativePath)) {
+        // Extension match (e.g., '*.log', '*.pyc')
+        if (pattern.startsWith('*.')) {
+            const ext = pattern.slice(1); // Get '.log' from '*.log'
+            if (fileName.endsWith(ext)) {
                 return true;
             }
         }
-        // Extension match
-        if (pattern.startsWith('*.') && relativePath.endsWith(pattern.slice(1))) {
-            return true;
+        // Glob pattern match (e.g., '**/*.rs.bk', 'cmake-build-*')
+        if (pattern.includes('*') && !pattern.startsWith('*.')) {
+            try {
+                // Convert glob to regex
+                let regexPattern = pattern
+                    .replace(/[.+^${}()|[\]\\]/g, '\\$&') // Escape special regex chars first
+                    .replace(/\*\*/g, '{{GLOBSTAR}}') // Temporarily replace **
+                    .replace(/\*/g, '[^/]*') // * matches anything except /
+                    .replace(/{{GLOBSTAR}}/g, '.*') // ** matches anything including /
+                    .replace(/\?/g, '[^/]'); // ? matches single char except /
+                const regex = new RegExp(`(^|/)${regexPattern}($|/|$)`);
+                if (regex.test(normalizedPath)) {
+                    return true;
+                }
+            }
+            catch {
+                // Invalid regex pattern, skip
+            }
+        }
+        // Path prefix match (e.g., 'docs/_build' should match 'docs/_build/something')
+        if (pattern.includes('/') && !pattern.includes('*')) {
+            const normalizedPattern = pattern.replace(/\\/g, '/');
+            if (normalizedPath === normalizedPattern || normalizedPath.startsWith(normalizedPattern + '/')) {
+                return true;
+            }
         }
     }
     return false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@optiqcode/cli",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "CLI tool for Optiq - automatic code indexing and context engine",
   "type": "module",
   "main": "dist/index.js",