codehere 0.2.1 → 0.3.0-rc.1

package/dist/embed.js

@@ -1,459 +0,0 @@
-import { readFileSync, readdirSync, statSync, existsSync, writeFileSync, mkdirSync } from 'fs';
-import { join, relative, dirname } from 'path';
-import { fileURLToPath } from 'url';
-import { config } from 'dotenv';
-// CohereClient is lazy-loaded in getCohereClient() to avoid hard dependency
-import initSqlJs from 'sql.js';
-import { encoding_for_model } from 'tiktoken';
-import { indexParallel } from './parallel-processor.js';
-import { colors, createProgressBar, createTable, success, warning, info, formatPath, formatNumber, sectionHeader, newline, statusIndicator, } from './ui.js';
-// Load .env from agent directory
-try {
-    const __filename = fileURLToPath(import.meta.url);
-    const __dirname = dirname(__filename);
-    const agentDir = join(__dirname, '..');
-    config({ path: join(agentDir, '.env') });
-}
-catch {
-    // Fallback
-}
-config(); // Also try current directory
-function detectEmbeddingProvider() {
-    if (process.env.OPENAI_API_KEY)
-        return 'openai';
-    if (process.env.COHERE_API_KEY)
-        return 'cohere';
-    return 'none';
-}
-// Lazy-init Cohere client only when needed
-let _cohereClient = null;
-function getCohereClient() {
-    if (!_cohereClient) {
-        const { CohereClient } = require('cohere-ai');
-        _cohereClient = new CohereClient({ token: process.env.COHERE_API_KEY || '' });
-    }
-    return _cohereClient;
-}
-import { getEmbeddingsDbPath } from './infrastructure/storage/embeddings-path.js';
-/**
- * @deprecated Legacy global DB path - use getEmbeddingsDbPath() instead
- * This function and DB_PATH are only used by the legacy indexRepository() function.
- * The active index command uses indexRepositoryParallel() which uses per-repo paths.
- */
-function getLegacyDbPath() {
-    const __filename = fileURLToPath(import.meta.url);
-    const __dirname = dirname(__filename);
-    const agentDir = join(__dirname, '..');
-    // Check if we're in the agent directory structure
-    if (existsSync(join(agentDir, 'package.json'))) {
-        return join(agentDir, 'data', 'embeddings.sqlite');
-    }
-    // Otherwise use current directory
-    return join(process.cwd(), 'agent', 'data', 'embeddings.sqlite');
-}
-// Legacy DB_PATH - only used by deprecated indexRepository() function
-// Active CLI uses indexRepositoryParallel() with per-repo paths
-const DB_PATH = getLegacyDbPath();
-const CHUNK_SIZE = 300; // target tokens per chunk
-const OVERLAP = 50; // overlap tokens between chunks
-const enc = encoding_for_model('gpt-3.5-turbo');
-function shouldIgnorePath(filepath) {
-    const ignorePatterns = [
-        'node_modules',
-        '.git',
-        'dist',
-        'build',
-        '.next',
-        'coverage',
-        '.env',
-        'embeddings.sqlite',
-        'logs.jsonl',
-        'agent/data',
-        'agent/dist',
-        'agent/node_modules',
-    ];
-    return ignorePatterns.some(pattern => filepath.includes(pattern));
-}
-function getAllFiles(dir, baseDir = dir) {
-    const files = [];
-    try {
-        const entries = readdirSync(dir);
-        for (const entry of entries) {
-            const fullPath = join(dir, entry);
-            const relPath = relative(baseDir, fullPath);
-            if (shouldIgnorePath(relPath)) {
-                continue;
-            }
-            const stat = statSync(fullPath);
-            if (stat.isDirectory()) {
-                files.push(...getAllFiles(fullPath, baseDir));
-            }
-            else if (stat.isFile()) {
-                // Only process text files
-                const ext = entry.split('.').pop()?.toLowerCase();
-                const textExtensions = [
-                    'ts', 'tsx', 'js', 'jsx', 'json', 'md', 'txt', 'py', 'java', 'cpp', 'c', 'h',
-                    'go', 'rs', 'rb', 'php', 'html', 'css', 'scss', 'sql', 'sh', 'yaml', 'yml',
-                    'xml', 'toml', 'ini', 'conf', 'config'
-                ];
-                if (ext && textExtensions.includes(ext)) {
-                    files.push(fullPath);
-                }
-            }
-        }
-    }
-    catch (error) {
-        // Skip directories we can't read
-    }
-    return files;
-}
-function chunkText(text, targetTokens, overlap) {
-    const tokens = enc.encode(text);
-    const chunks = [];
-    if (tokens.length <= targetTokens) {
-        return [text];
-    }
-    let start = 0;
-    while (start < tokens.length) {
-        const end = Math.min(start + targetTokens, tokens.length);
-        const chunkTokens = tokens.slice(start, end);
-        const decoded = enc.decode(chunkTokens);
-        // Handle both string and Uint8Array return types
-        const chunkText = typeof decoded === 'string' ? decoded : new TextDecoder().decode(decoded);
-        chunks.push(chunkText);
-        if (end >= tokens.length)
-            break;
-        start = end - overlap;
-    }
-    return chunks;
-}
-async function embedTextOpenAI(text) {
-    const response = await fetch('https://api.openai.com/v1/embeddings', {
-        method: 'POST',
-        headers: {
-            'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
-            'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-            input: text,
-            model: 'text-embedding-3-small',
-        }),
-    });
-    if (!response.ok) {
-        const errText = await response.text();
-        throw new Error(`OpenAI embedding error: ${response.status} ${errText}`);
-    }
-    const data = await response.json();
-    return data.data?.[0]?.embedding || [];
-}
-async function embedTextCohere(text) {
-    const cohere = getCohereClient();
-    const response = await cohere.embed({
-        texts: [text],
-        model: 'embed-english-v3.0',
-        inputType: 'search_document',
-    });
-    const embeddings = Array.isArray(response.embeddings)
-        ? response.embeddings
-        : response.embeddings.embeddings || [];
-    return embeddings[0] || [];
-}
-async function embedText(text, retries = 3) {
-    const provider = detectEmbeddingProvider();
-    if (provider === 'none') {
-        throw new Error('No embedding API key found. Set OPENAI_API_KEY or COHERE_API_KEY to use index/ask/fix commands.');
-    }
-    for (let attempt = 0; attempt < retries; attempt++) {
-        try {
-            if (provider === 'openai') {
-                return await embedTextOpenAI(text);
-            }
-            else {
-                return await embedTextCohere(text);
-            }
-        }
-        catch (error) {
-            const isRateLimit = error?.status === 429 || error?.message?.includes('429') || error?.message?.includes('rate limit');
-            if (isRateLimit && attempt < retries - 1) {
-                const waitTime = Math.min(Math.pow(2, attempt) * 1000, 60000);
-                console.log(warning(`Rate limited, waiting ${waitTime / 1000}s before retry...`));
-                await new Promise(resolve => setTimeout(resolve, waitTime));
-                continue;
-            }
-            throw new Error(`Embedding error (${provider}): ${error instanceof Error ? error.message : String(error)}`);
-        }
-    }
-    throw new Error('Embedding failed after retries');
-}
-function cosineSimilarity(a, b) {
-    if (a.length !== b.length)
-        return 0;
-    let dotProduct = 0;
-    let normA = 0;
-    let normB = 0;
-    for (let i = 0; i < a.length; i++) {
-        dotProduct += a[i] * b[i];
-        normA += a[i] * a[i];
-        normB += b[i] * b[i];
-    }
-    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
-}
-export async function indexRepository(repoPath = process.cwd()) {
-    sectionHeader('Scanning Repository');
-    const files = getAllFiles(repoPath, repoPath);
-    console.log(info(`Found ${formatNumber(files.length)} files to index`));
-    newline();
-    // Initialize SQL.js
-    const SQL = await initSqlJs();
-    let db;
-    // Load existing database or create new one
-    if (existsSync(DB_PATH)) {
-        const buffer = readFileSync(DB_PATH);
-        db = new SQL.Database(buffer);
-    }
-    else {
-        db = new SQL.Database();
-    }
-    // Create table
-    db.run(`
-    CREATE TABLE IF NOT EXISTS embeddings (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      filepath TEXT NOT NULL,
-      chunkIndex INTEGER NOT NULL,
-      content TEXT NOT NULL,
-      embedding TEXT NOT NULL
-    )
-  `);
-    db.run(`
-    CREATE INDEX IF NOT EXISTS idx_filepath ON embeddings(filepath)
-  `);
-    // Clear existing data
-    db.run('DELETE FROM embeddings');
-    const insertStmt = db.prepare(`
-    INSERT INTO embeddings (filepath, chunkIndex, content, embedding)
-    VALUES (?, ?, ?, ?)
-  `);
-    let totalChunks = 0;
-    const fileResults = [];
-    // Create progress bar
-    const progressBar = createProgressBar(files.length, colors.dim('Processing') + ' |{bar}| ' + colors.cyan('{percentage}%') + ' | {value}/{total}');
-    sectionHeader('Processing Files');
-    for (let i = 0; i < files.length; i++) {
-        const filepath = files[i];
-        const relPath = relative(repoPath, filepath);
-        try {
-            const content = readFileSync(filepath, 'utf-8');
-            const chunks = chunkText(content, CHUNK_SIZE, OVERLAP);
-            let fileChunks = 0;
-            let fileErrors = 0;
-            for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
-                const chunk = chunks[chunkIdx];
-                try {
-                    const embedding = await embedText(chunk);
-                    insertStmt.run([
-                        relPath,
-                        chunkIdx,
-                        chunk,
-                        JSON.stringify(embedding),
-                    ]);
-                    fileChunks++;
-                    // Rate limiting: delay between chunks to stay under 40/min limit
-                    // 40 calls/min = 1.5s per call, use 2s to be safe
-                    await new Promise(resolve => setTimeout(resolve, 2000));
-                }
-                catch (err) {
-                    fileErrors++;
-                    // Continue with next chunk instead of failing entire file
-                }
-            }
-            totalChunks += fileChunks;
-            if (fileErrors === 0) {
-                fileResults.push({
-                    file: relPath,
-                    chunks: fileChunks,
-                    status: statusIndicator('success') + ' Done',
-                });
-            }
-            else {
-                fileResults.push({
-                    file: relPath,
-                    chunks: fileChunks,
-                    status: statusIndicator('warning') + ` Done (${fileErrors} errors)`,
-                });
-            }
-            progressBar.update(i + 1);
-        }
-        catch (err) {
-            fileResults.push({
-                file: relPath,
-                chunks: 0,
-                status: statusIndicator('error') + ' Failed',
-            });
-            progressBar.update(i + 1);
-        }
-    }
-    progressBar.stop();
-    insertStmt.free();
-    // Save database to file
-    const data = db.export();
-    const buffer = Buffer.from(data.buffer, data.byteOffset, data.byteLength);
-    writeFileSync(DB_PATH, buffer);
-    db.close();
-    newline();
-    sectionHeader('Indexing Results');
-    // Display file results table
-    const fileTable = createTable(['File', 'Chunks', 'Status'], fileResults.map(r => [
-        formatPath(r.file),
-        formatNumber(r.chunks),
-        r.status,
-    ]));
-    console.log(fileTable.toString());
-    newline();
-    // Summary
-    const successful = fileResults.filter(r => r.status.includes('Done')).length;
-    const failed = fileResults.filter(r => r.status.includes('Failed')).length;
-    sectionHeader('Summary');
-    console.log(createTable(['Metric', 'Value'], [
-        ['Total Files', formatNumber(files.length)],
-        ['Successful', statusIndicator('success') + ' ' + formatNumber(successful)],
-        ['Failed', failed > 0 ? statusIndicator('error') + ' ' + formatNumber(failed) : statusIndicator('success') + ' 0'],
-        ['Total Chunks', formatNumber(totalChunks)],
-        ['Database Size', colors.cyan(`${(buffer.length / 1024 / 1024).toFixed(2)} MB`)],
-    ]).toString());
-    newline();
-    console.log(success(`Indexing complete: ${formatNumber(totalChunks)} chunks indexed`));
-}
-/**
- * NEW: Parallel indexing with caching
- */
-export async function indexRepositoryParallel(repoPath, options = {}) {
-    const targetDir = repoPath || process.cwd();
-    // Find all code files
-    const files = getAllFiles(targetDir, targetDir);
-    if (files.length === 0) {
-        console.log(warning('No code files found to index.'));
-        return {
-            indexed: 0,
-            cached: 0,
-            totalChunks: 0,
-            timeMs: 0,
-        };
-    }
-    console.log(info(`Found ${formatNumber(files.length)} files to index`));
-    console.log(info('Using parallel processing with caching...'));
-    newline();
-    // Use parallel processor to get embeddings
-    // Auto-use parallel for repos with >100 files
-    const result = await indexParallel(files, {
-        maxConcurrency: options.maxConcurrency || (files.length > 100 ? 10 : 5),
-        useCache: options.useCache !== false,
-        cacheDir: join(targetDir, '.codehere-cache'),
-        sequential: files.length <= 100, // Only sequential for very small repos
-    });
-    // Now save to SQLite database
-    // Use per-repo database path for codebase isolation
-    const pathInfo = getEmbeddingsDbPath(targetDir);
-    const dbPath = pathInfo.dbPath;
-    console.log(info(`Database path: ${dbPath}`));
-    // Ensure cache directory exists (already created by getEmbeddingsDbPath, but double-check)
-    const dbDir = dirname(dbPath);
-    if (!existsSync(dbDir)) {
-        console.log(info(`Creating cache directory: ${dbDir}`));
-        mkdirSync(dbDir, { recursive: true });
-    }
-    console.log(info('Initializing SQLite database...'));
-    const SQL = await initSqlJs();
-    let db;
-    // Load existing database or create new one
-    if (existsSync(dbPath)) {
-        const buffer = readFileSync(dbPath);
-        db = new SQL.Database(buffer);
-    }
-    else {
-        db = new SQL.Database();
-    }
-    // Create table
-    db.run(`
-    CREATE TABLE IF NOT EXISTS embeddings (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      filepath TEXT NOT NULL,
-      chunkIndex INTEGER NOT NULL,
-      content TEXT NOT NULL,
-      embedding TEXT NOT NULL
-    )
-  `);
-    db.run(`
-    CREATE INDEX IF NOT EXISTS idx_filepath ON embeddings(filepath)
-  `);
-    // Clear existing data
-    db.run('DELETE FROM embeddings');
-    const insertStmt = db.prepare(`
-    INSERT INTO embeddings (filepath, chunkIndex, content, embedding)
-    VALUES (?, ?, ?, ?)
-  `);
-    // Load cached embeddings and write to database
-    const cacheDir = join(targetDir, '.codehere-cache');
-    let dbChunks = 0;
-    if (existsSync(cacheDir)) {
-        const cacheFiles = readdirSync(cacheDir).filter(f => f.endsWith('.json'));
-        console.log(info(`Loading ${cacheFiles.length} cache files into database...`));
-        for (const cacheFile of cacheFiles) {
-            try {
-                const cachePath = join(cacheDir, cacheFile);
-                const cacheEntry = JSON.parse(readFileSync(cachePath, 'utf-8'));
-                if (cacheEntry.chunks && Array.isArray(cacheEntry.chunks)) {
-                    // Handle both absolute and relative paths
-                    let relPath = cacheEntry.filepath || cacheFile.replace('.json', '').replace(/_/g, '/');
-                    if (relPath.startsWith('/') || relPath.startsWith(targetDir)) {
-                        // Absolute path - make it relative
-                        relPath = relative(targetDir, relPath);
-                    }
-                    // If still absolute or empty, use filename
-                    if (!relPath || relPath.startsWith('/')) {
-                        relPath = cacheFile.replace('.json', '').replace(/_/g, '/');
-                    }
-                    for (let i = 0; i < cacheEntry.chunks.length; i++) {
-                        const chunk = cacheEntry.chunks[i];
-                        if (chunk.content && chunk.embedding) {
-                            insertStmt.run([
-                                relPath,
-                                i,
-                                chunk.content,
-                                JSON.stringify(chunk.embedding),
-                            ]);
-                            dbChunks++;
-                        }
-                    }
-                }
-            }
-            catch (err) {
-                console.log(warning(`Error processing cache file ${cacheFile}: ${err instanceof Error ? err.message : String(err)}`));
-            }
-        }
-    }
-    else {
-        console.log(warning(`Cache directory not found: ${cacheDir}`));
-    }
-    insertStmt.free();
-    // Save database to file
-    try {
-        const data = db.export();
-        const buffer = Buffer.from(data.buffer, data.byteOffset, data.byteLength);
-        writeFileSync(dbPath, buffer);
-        db.close();
-        console.log(success(`Database saved to: ${dbPath}`));
-    }
-    catch (err) {
-        db.close();
-        throw new Error(`Failed to save database to ${dbPath}: ${err instanceof Error ? err.message : String(err)}`);
-    }
-    console.log(success(`Parallel indexing complete:`));
-    console.log(`  • Indexed: ${formatNumber(result.indexed)} files`);
-    console.log(`  • Cached: ${formatNumber(result.cached)} files`);
-    console.log(`  • Total chunks: ${formatNumber(result.totalChunks)}`);
-    console.log(`  • Saved to database: ${formatNumber(dbChunks)} chunks`);
-    console.log(`  • Database: ${dbPath}`);
-    console.log(`  • Time: ${formatNumber(result.timeMs / 1000)}s`);
-    return result;
-}
-//# sourceMappingURL=embed.js.map

package/dist/formatter.js

@@ -1,168 +0,0 @@
-/**
- * Terminal Markdown Formatter
- * Converts markdown to beautifully formatted terminal output
- */
-import { colors } from './ui.js';
-import { formatCodeBlockWithHighlight } from './infrastructure/ux/syntax-highlighter.js';
-/**
- * Format markdown text for terminal display
- */
-export function formatMarkdown(text) {
-    if (!text)
-        return '';
-    let formatted = text;
-    // Remove markdown code blocks but preserve content with syntax highlighting
-    formatted = formatted.replace(/```(\w*)\n?([\s\S]*?)```/g, (match, lang, code) => {
-        try {
-            // Use formatCodeBlock for better formatting with borders
-            return '\n' + formatCodeBlock(code.trim(), lang || 'text') + '\n';
-        }
-        catch {
-            return '\n' + colors.cyan(code.trim()) + '\n';
-        }
-    });
-    // Remove inline code but style it
-    formatted = formatted.replace(/`([^`]+)`/g, (match, code) => {
-        return colors.cyan(code);
-    });
-    // Convert headers to bold
-    formatted = formatted.replace(/^### (.*$)/gim, (match, text) => {
-        return '\n' + colors.bold(text) + '\n';
-    });
-    formatted = formatted.replace(/^## (.*$)/gim, (match, text) => {
-        return '\n' + colors.bold(text) + '\n';
-    });
-    formatted = formatted.replace(/^# (.*$)/gim, (match, text) => {
-        return '\n' + colors.bold(text) + '\n';
-    });
-    // Convert bold (**text** or __text__) to colored bold
-    // But preserve numbered lists in bold (e.g., **1. Title** -> 1. Title in bold)
-    formatted = formatted.replace(/\*\*(\d+)\. (.+?)\*\*/g, (match, num, text) => {
-        return colors.bold(`${num}. ${text}`);
-    });
-    formatted = formatted.replace(/\*\*(.+?)\*\*/g, (match, text) => {
-        return colors.bold(text);
-    });
-    formatted = formatted.replace(/__(.+?)__/g, (match, text) => {
-        return colors.bold(text);
-    });
-    // Convert italic (*text* or _text_) to dim
-    formatted = formatted.replace(/\*(.+?)\*/g, (match, text) => {
-        // Don't convert if it's part of **bold**
-        if (text.includes('*'))
-            return match;
-        return colors.dim(text);
-    });
-    formatted = formatted.replace(/_(.+?)_/g, (match, text) => {
-        // Don't convert if it's part of __bold__
-        if (text.includes('_'))
-            return match;
-        return colors.dim(text);
-    });
-    // Convert bullet points to clean bullets
-    formatted = formatted.replace(/^[\*\-\+] (.+)$/gim, (match, text) => {
-        return '  • ' + text.trim();
-    });
-    // Convert numbered lists - preserve numbering with proper indentation
-    // Handle both plain numbered lists and those that might have been bold
-    formatted = formatted.replace(/^(\d+)\. (.+)$/gim, (match, num, text) => {
-        // If text is already bold (from previous processing), keep it bold
-        const trimmed = text.trim();
-        return `  ${num}. ${trimmed}`;
-    });
-    // Clean up multiple newlines (max 2)
-    formatted = formatted.replace(/\n{3,}/g, '\n\n');
-    // Remove markdown links but keep text
-    formatted = formatted.replace(/\[([^\]]+)\]\([^\)]+\)/g, (match, text) => {
-        return text;
-    });
-    // Remove horizontal rules
-    formatted = formatted.replace(/^---+\s*$/gim, '');
-    // Clean up any remaining markdown artifacts
-    formatted = formatted.replace(/\[([^\]]+)\]\[[^\]]+\]/g, '$1'); // Reference links
-    formatted = formatted.replace(/!\[([^\]]*)\]\([^\)]+\)/g, '$1'); // Images
-    return formatted.trim();
-}
-/**
- * Format code block for terminal with syntax highlighting
- */
-export function formatCodeBlock(code, language, filepath) {
-    try {
-        return formatCodeBlockWithHighlight(code, language, filepath);
-    }
-    catch {
-        // Fallback to basic formatting if highlighter not available
-        const lines = code.split('\n');
-        const maxWidth = process.stdout.columns || 80;
-        const indent = '  ';
-        let output = colors.dim('┌─ Code ──────────────────────────────────────┐\n');
-        for (const line of lines) {
-            // Truncate very long lines
-            const displayLine = line.length > maxWidth - 10
-                ? line.substring(0, maxWidth - 13) + '...'
-                : line;
-            output += indent + colors.cyan(displayLine) + '\n';
-        }
-        output += colors.dim('└──────────────────────────────────────────────┘');
-        return output;
-    }
-}
-/**
- * Format list items nicely
- */
-export function formatList(items) {
-    return items.map(item => '  • ' + item.trim()).join('\n');
-}
-/**
- * Format response text with proper wrapping and formatting
- */
-export function formatResponse(text, maxWidth = 80) {
-    if (!text)
-        return '';
-    // First format markdown
-    let formatted = formatMarkdown(text);
-    // Word wrap to terminal width
-    const lines = formatted.split('\n');
-    const wrapped = [];
-    for (const line of lines) {
-        if (line.length <= maxWidth) {
-            wrapped.push(line);
-        }
-        else {
-            // Wrap long lines
-            const words = line.split(' ');
-            let currentLine = '';
-            for (const word of words) {
-                if ((currentLine + word).length <= maxWidth) {
-                    currentLine += (currentLine ? ' ' : '') + word;
-                }
-                else {
-                    if (currentLine) {
-                        wrapped.push(currentLine);
-                    }
-                    currentLine = word;
-                }
-            }
-            if (currentLine) {
-                wrapped.push(currentLine);
-            }
-        }
-    }
-    return wrapped.join('\n');
-}
-/**
- * Clean and format AI response for display
- */
-export function cleanResponse(response) {
-    if (!response)
-        return '';
-    let cleaned = response;
-    // Remove any leading/trailing markdown artifacts
-    cleaned = cleaned.trim();
-    // Remove markdown frontmatter if present
-    cleaned = cleaned.replace(/^---[\s\S]*?---\n/, '');
-    // Format the response
-    cleaned = formatResponse(cleaned);
-    return cleaned;
-}
-//# sourceMappingURL=formatter.js.map