codehere 0.2.2 → 0.3.0-rc.10

package/dist/parallel-processor.js

@@ -1,534 +0,0 @@
-/**
- * Parallel Processing & Caching System
- * Now supports sequential processing with rate limiting for API limits
- */
-import { readFileSync, writeFileSync, existsSync, statSync, readdirSync, mkdirSync, rmSync, unlinkSync } from 'fs';
-import { join } from 'path';
-import { config } from 'dotenv';
-import { encoding_for_model } from 'tiktoken';
-// Load .env
-config();
-const enc = encoding_for_model('gpt-3.5-turbo');
-const CHUNK_SIZE = 2000; // ~500 tokens. OpenAI handles 8191 tokens per embedding.
-const OVERLAP = 50;
-// Rate limiting state
-let lastApiCallTime = 0;
-let apiCallCount = 0;
-let monthlyApiCallCount = 0;
-const MIN_DELAY_MS = 200; // 200ms between calls (OpenAI allows 3000 RPM for embeddings)
-const MONTHLY_LIMIT = 50000; // OpenAI production limit is much higher
-/**
- * Rate-limited API call wrapper
- */
-async function rateLimitedCall(fn) {
-    // Check monthly limit
-    if (monthlyApiCallCount >= MONTHLY_LIMIT) {
-        throw new Error(`Monthly API limit reached (${MONTHLY_LIMIT} calls). Please upgrade to production key or wait for limit reset.`);
-    }
-    // Calculate delay needed
-    const now = Date.now();
-    const timeSinceLastCall = now - lastApiCallTime;
-    if (timeSinceLastCall < MIN_DELAY_MS) {
-        const delay = MIN_DELAY_MS - timeSinceLastCall;
-        await new Promise(resolve => setTimeout(resolve, delay));
-    }
-    // Make API call
-    lastApiCallTime = Date.now();
-    apiCallCount++;
-    monthlyApiCallCount++;
-    try {
-        return await fn();
-    }
-    catch (error) {
-        // If rate limit error, wait longer and retry once
-        if (error instanceof Error && error.message.includes('429')) {
-            console.log('Rate limit hit, waiting 5 seconds...');
-            await new Promise(resolve => setTimeout(resolve, 5000));
-            lastApiCallTime = Date.now();
-            monthlyApiCallCount++;
-            return await fn();
-        }
-        throw error;
-    }
-}
-/**
- * Sequential indexing with caching and rate limiting
- */
-export async function indexParallel(files, options = {}) {
-    // Auto-detect: use parallel for large repos (>100 files), sequential for small ones
-    const shouldUseParallel = options.sequential === false || (options.sequential === undefined && files.length > 100);
-    const maxConcurrency = options.maxConcurrency || (shouldUseParallel ? 10 : 1);
-    const useCache = options.useCache !== false;
-    const cacheDir = options.cacheDir || '.codehere-cache';
-    const startTime = Date.now();
-    let indexed = 0;
-    let cached = 0;
-    let totalChunks = 0;
-    let processed = 0;
-    let lastProgressUpdate = Date.now();
-    // Create cache directory
-    if (useCache && !existsSync(cacheDir)) {
-        mkdirSync(cacheDir, { recursive: true });
-    }
-    // Reset rate limiting state
-    lastApiCallTime = 0;
-    apiCallCount = 0;
-    // Progress update helper (throttled to every 2 seconds)
-    const updateProgress = (force = false) => {
-        const now = Date.now();
-        if (force || now - lastProgressUpdate > 2000) {
-            const percent = ((processed / files.length) * 100).toFixed(1);
-            const elapsed = ((now - startTime) / 1000).toFixed(0);
-            process.stdout.write(`\r[${processed}/${files.length}] ${percent}% | Cached: ${cached} | Indexed: ${indexed} | Time: ${elapsed}s`);
-            lastProgressUpdate = now;
-        }
-    };
-    if (shouldUseParallel && maxConcurrency > 1) {
-        // Parallel processing with smart batching
-        console.log(`Processing ${files.length} files in parallel (concurrency: ${maxConcurrency})...`);
-        // First pass: Check cache status for all files (fast, no API calls)
-        const fileStatuses = [];
-        if (useCache) {
-            // Check cache in parallel (no rate limiting needed)
-            const cacheChecks = await Promise.allSettled(files.map(async (file) => {
-                const cacheKey = getCacheKey(file);
-                const cachePath = join(cacheDir, cacheKey);
-                if (existsSync(cachePath)) {
-                    try {
-                        const cacheEntry = JSON.parse(readFileSync(cachePath, 'utf-8'));
-                        const fileHash = getFileHash(file);
-                        return { file, needsProcessing: cacheEntry.hash !== fileHash };
-                    }
-                    catch {
-                        return { file, needsProcessing: true };
-                    }
-                }
-                return { file, needsProcessing: true };
-            }));
-            for (const check of cacheChecks) {
-                if (check.status === 'fulfilled') {
-                    fileStatuses.push(check.value);
-                }
-                else {
-                    fileStatuses.push({ file: '', needsProcessing: true });
-                }
-            }
-        }
-        else {
-            fileStatuses.push(...files.map(file => ({ file, needsProcessing: true })));
-        }
-        // Separate cached vs needs-processing files
-        const cachedFiles = fileStatuses.filter(s => !s.needsProcessing).map(s => s.file);
-        const needsProcessingFiles = fileStatuses.filter(s => s.needsProcessing).map(s => s.file);
-        // Process cached files in parallel (no rate limiting)
-        if (cachedFiles.length > 0) {
-            const cachedResults = await Promise.allSettled(cachedFiles.map(file => processFileWithCache(file, useCache, cacheDir)));
-            for (const result of cachedResults) {
-                processed++;
-                if (result.status === 'fulfilled') {
-                    cached++;
-                    totalChunks += result.value.chunks;
-                }
-            }
-            updateProgress(true);
-        }
-        // Process files that need API calls with rate limiting and batching
-        if (needsProcessingFiles.length > 0) {
-            const batches = [];
-            for (let i = 0; i < needsProcessingFiles.length; i += maxConcurrency) {
-                batches.push(needsProcessingFiles.slice(i, i + maxConcurrency));
-            }
-            for (let batchIdx = 0; batchIdx < batches.length; batchIdx++) {
-                const batch = batches[batchIdx];
-                // Process batch in parallel (rate limiting handled inside processFileWithCache)
-                const results = await Promise.allSettled(batch.map(file => processFileWithCache(file, useCache, cacheDir)));
-                // Process results
-                for (let i = 0; i < results.length; i++) {
-                    const result = results[i];
-                    const file = batch[i];
-                    processed++;
-                    if (result.status === 'fulfilled') {
-                        if (result.value.cached) {
-                            cached++;
-                        }
-                        else {
-                            indexed++;
-                        }
-                        totalChunks += result.value.chunks;
-                    }
-                    else {
-                        const error = result.reason instanceof Error ? result.reason.message : String(result.reason);
-                        console.error(`\n✗ Error: ${file}: ${error}`);
-                        // If monthly limit hit, stop processing
-                        if (error.includes('Monthly API limit')) {
-                            console.error('\nStopping indexing due to monthly API limit.');
-                            break;
-                        }
-                    }
-                }
-                // Update progress
-                updateProgress();
-                // Check if we should stop
-                if (processed >= files.length)
-                    break;
-            }
-        }
-    }
-    else {
-        // Sequential processing (for small repos or when explicitly requested)
-        console.log(`Processing ${files.length} files sequentially...`);
-        for (let i = 0; i < files.length; i++) {
-            const file = files[i];
-            processed++;
-            try {
-                const result = await processFileWithCache(file, useCache, cacheDir);
-                if (result.cached) {
-                    cached++;
-                }
-                else {
-                    indexed++;
-                }
-                totalChunks += result.chunks;
-            }
-            catch (error) {
-                const errorMsg = error instanceof Error ? error.message : String(error);
-                console.error(`\n[${i + 1}/${files.length}] ✗ Error: ${file}: ${errorMsg}`);
-                // If monthly limit hit, stop processing
-                if (errorMsg.includes('Monthly API limit')) {
-                    console.error('Stopping indexing due to monthly API limit.');
-                    break;
-                }
-            }
-            // Update progress every 10 files or every 2 seconds
-            if (i % 10 === 0 || i === files.length - 1) {
-                updateProgress(true);
-            }
-        }
-    }
-    // Final progress update
-    updateProgress(true);
-    console.log(''); // New line after progress
-    const timeMs = Date.now() - startTime;
-    console.log(`\nAPI Call Stats: ${apiCallCount} calls made, ${monthlyApiCallCount} total this month`);
-    return {
-        indexed,
-        cached,
-        totalChunks,
-        timeMs,
-    };
-}
-/**
- * Process file with caching
- */
-async function processFileWithCache(filepath, useCache, cacheDir) {
-    if (!useCache) {
-        // Process without cache
-        const content = readFileSync(filepath, 'utf-8');
-        const chunks = chunkTextLocal(content, CHUNK_SIZE, OVERLAP);
-        // Embed chunks (this is the slow part)
-        for (const chunk of chunks) {
-            await embedTextLocal(chunk);
-        }
-        return { cached: false, chunks: chunks.length };
-    }
-    // Check cache
-    const cacheKey = getCacheKey(filepath);
-    const cachePath = join(cacheDir, cacheKey);
-    if (existsSync(cachePath)) {
-        const cacheEntry = JSON.parse(readFileSync(cachePath, 'utf-8'));
-        // Check if file hasn't changed
-        const fileHash = getFileHash(filepath);
-        if (cacheEntry.hash === fileHash) {
-            // Cache hit - return cached chunks
-            return { cached: true, chunks: cacheEntry.chunks.length };
-        }
-    }
-    // Cache miss or stale - process file
-    try {
-        const content = readFileSync(filepath, 'utf-8');
-        if (!content || content.trim().length === 0) {
-            return { cached: false, chunks: 0 };
-        }
-        const chunks = chunkTextLocal(content, CHUNK_SIZE, OVERLAP);
-        if (chunks.length === 0) {
-            return { cached: false, chunks: 0 };
-        }
-        const fileHash = getFileHash(filepath);
-        // Embed chunks (rate limiting handled by embedTextLocal)
-        const embeddings = [];
-        for (let i = 0; i < chunks.length; i++) {
-            const chunk = chunks[i];
-            try {
-                const embedding = await embedTextLocal(chunk);
-                embeddings[i] = embedding;
-            }
-            catch (error) {
-                // If embedding fails, log and skip this chunk
-                const errorMsg = error instanceof Error ? error.message : String(error);
-                console.error(`Failed to embed chunk ${i + 1}/${chunks.length} of ${filepath}: ${errorMsg}`);
-                // If monthly limit hit, stop processing this file
-                if (errorMsg.includes('Monthly API limit')) {
-                    throw error; // Re-throw to stop processing
-                }
-                // Continue with other chunks for other errors
-                continue;
-            }
-        }
-        // Only save if we got at least one embedding
-        if (embeddings.length === 0) {
-            return { cached: false, chunks: 0 };
-        }
-        // Save to cache
-        const cacheEntry = {
-            filepath,
-            hash: fileHash,
-            chunks: chunks.slice(0, embeddings.length).map((content, i) => ({
-                content,
-                embedding: embeddings[i],
-            })),
-            timestamp: Date.now(),
-        };
-        writeFileSync(cachePath, JSON.stringify(cacheEntry, null, 2), 'utf-8');
-        return { cached: false, chunks: embeddings.length };
-    }
-    catch (error) {
-        // Re-throw monthly limit errors to stop processing
-        if (error instanceof Error && error.message.includes('Monthly API limit')) {
-            throw error;
-        }
-        // Log other errors but don't throw - allow other files to process
-        console.error(`Error processing ${filepath}: ${error instanceof Error ? error.message : String(error)}`);
-        return { cached: false, chunks: 0 };
-    }
-}
-/**
- * Get cache key from filepath
- */
-function getCacheKey(filepath) {
-    // Create a safe filename from filepath
-    return filepath.replace(/[^a-zA-Z0-9]/g, '_') + '.json';
-}
-/**
- * Get file hash (simple: use mtime + size)
- */
-function getFileHash(filepath) {
-    try {
-        const stat = statSync(filepath);
-        return `${stat.mtimeMs}-${stat.size}`;
-    }
-    catch {
-        return 'unknown';
-    }
-}
-/**
- * Incremental update: only process changed files
- */
-export async function updateIndexIncremental(baseDir, options = {}) {
-    const { cacheDir = '.codehere-cache', watchMode = false } = options;
-    // Find all code files
-    const files = findCodeFiles(baseDir);
-    let updated = 0;
-    let unchanged = 0;
-    let totalChunks = 0;
-    for (const filepath of files) {
-        const cacheKey = getCacheKey(filepath);
-        const cachePath = join(cacheDir, cacheKey);
-        if (existsSync(cachePath)) {
-            const cacheEntry = JSON.parse(readFileSync(cachePath, 'utf-8'));
-            const fileHash = getFileHash(filepath);
-            if (cacheEntry.hash === fileHash) {
-                // File unchanged
-                unchanged++;
-                totalChunks += cacheEntry.chunks.length;
-                continue;
-            }
-        }
-        // File changed or new - process it
-        const result = await processFileWithCache(filepath, true, cacheDir);
-        updated++;
-        totalChunks += result.chunks;
-    }
-    return {
-        updated,
-        unchanged,
-        totalChunks,
-    };
-}
-/**
- * Find code files recursively
- */
-function findCodeFiles(dir) {
-    const files = [];
-    try {
-        const entries = readdirSync(dir, { withFileTypes: true });
-        for (const entry of entries) {
-            const fullPath = join(dir, entry.name);
-            // Skip build artifacts, caches, and hidden dirs
-            if (entry.name.startsWith('.') || entry.name === 'node_modules' || entry.name === 'dist' || entry.name === 'build' || entry.name === '.codehere-cache' || entry.name === '.codehere') {
-                continue;
-            }
-            if (entry.isDirectory()) {
-                files.push(...findCodeFiles(fullPath));
-            }
-            else if (entry.isFile()) {
-                const ext = entry.name.split('.').pop()?.toLowerCase();
-                if (ext && ['ts', 'tsx', 'js', 'jsx'].includes(ext)) {
-                    files.push(fullPath);
-                }
-            }
-        }
-    }
-    catch {
-        // Skip directories we can't read
-    }
-    return files;
-}
-/**
- * Clear cache
- */
-export function clearCache(cacheDir = '.codehere-cache') {
-    if (!existsSync(cacheDir)) {
-        return;
-    }
-    try {
-        rmSync(cacheDir, { recursive: true, force: true });
-    }
-    catch {
-        // Fallback: delete files individually
-        const files = readdirSync(cacheDir);
-        for (const file of files) {
-            const filePath = join(cacheDir, file);
-            try {
-                unlinkSync(filePath);
-            }
-            catch {
-                // Skip files we can't delete
-            }
-        }
-    }
-}
-/**
- * Chunk text locally (duplicate from embed.ts for independence)
- */
-function chunkTextLocal(text, targetTokens, overlap) {
-    const tokens = enc.encode(text);
-    const chunks = [];
-    if (tokens.length <= targetTokens) {
-        return [text];
-    }
-    let start = 0;
-    while (start < tokens.length) {
-        const end = Math.min(start + targetTokens, tokens.length);
-        const chunkTokens = tokens.slice(start, end);
-        const decoded = enc.decode(chunkTokens);
-        const chunkText = typeof decoded === 'string' ? decoded : new TextDecoder().decode(decoded);
-        chunks.push(chunkText);
-        if (end >= tokens.length)
-            break;
-        start = end - overlap;
-    }
-    return chunks;
-}
-/**
- * Embed text locally (duplicate from embed.ts for independence)
- * Now uses rate limiting
- */
-async function embedTextLocal(text, retries = 3) {
-    // Check for FAKE_AI mode - return fake embeddings
-    if (process.env.CODEHERE_FAKE_AI === '1') {
-        // Return fake embeddings (1024 dimensions, small random values)
-        return Array(1024).fill(0).map(() => Math.random() * 0.001 - 0.0005);
-    }
-    // Use rate-limited wrapper with auto-detected provider
-    return await rateLimitedCall(async () => {
-        for (let attempt = 0; attempt < retries; attempt++) {
-            try {
-                if (process.env.OPENAI_API_KEY) {
-                    const controller = new AbortController();
-                    const timeout = setTimeout(() => controller.abort(), 10000);
-                    const response = await fetch('https://api.openai.com/v1/embeddings', {
-                        method: 'POST',
-                        headers: {
-                            'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
-                            'Content-Type': 'application/json',
-                        },
-                        body: JSON.stringify({ input: text, model: 'text-embedding-3-small' }),
-                        signal: controller.signal,
-                    });
-                    clearTimeout(timeout);
-                    if (!response.ok)
-                        throw new Error(`OpenAI embedding error: ${response.status}`);
-                    const data = await response.json();
-                    return data.data?.[0]?.embedding || [];
-                }
-                else if (process.env.COHERE_API_KEY) {
-                    const { CohereClient } = require('cohere-ai');
-                    const cohere = new CohereClient({ token: process.env.COHERE_API_KEY });
-                    const response = await cohere.embed({
-                        texts: [text],
-                        model: 'embed-english-v3.0',
-                        inputType: 'search_document',
-                    });
-                    const embeddings = Array.isArray(response.embeddings)
-                        ? response.embeddings
-                        : response.embeddings.embeddings || [];
-                    return embeddings[0] || [];
-                }
-                else {
-                    throw new Error('No embedding API key found. Set OPENAI_API_KEY or COHERE_API_KEY.');
-                }
-            }
-            catch (error) {
-                if (attempt === retries - 1)
-                    throw error;
-                await new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 1000));
-            }
-        }
-        throw new Error('Failed to embed text after retries');
-    });
-}
-/**
- * Get cache statistics
- */
-export function getCacheStats(cacheDir = '.codehere-cache') {
-    if (!existsSync(cacheDir)) {
-        return {
-            totalFiles: 0,
-            totalChunks: 0,
-            totalSize: 0,
-            oldestEntry: 0,
-            newestEntry: 0,
-        };
-    }
-    const files = readdirSync(cacheDir).filter(f => f.endsWith('.json'));
-    let totalChunks = 0;
-    let totalSize = 0;
-    let oldestEntry = Date.now();
-    let newestEntry = 0;
-    for (const file of files) {
-        try {
-            const cachePath = join(cacheDir, file);
-            const stat = statSync(cachePath);
-            totalSize += stat.size;
-            const cacheEntry = JSON.parse(readFileSync(cachePath, 'utf-8'));
-            totalChunks += cacheEntry.chunks.length;
-            if (cacheEntry.timestamp < oldestEntry) {
-                oldestEntry = cacheEntry.timestamp;
-            }
-            if (cacheEntry.timestamp > newestEntry) {
-                newestEntry = cacheEntry.timestamp;
-            }
-        }
-        catch {
-            // Skip invalid cache entries
-        }
-    }
-    return {
-        totalFiles: files.length,
-        totalChunks,
-        totalSize,
-        oldestEntry,
-        newestEntry,
-    };
-}
-//# sourceMappingURL=parallel-processor.js.map