codexa 1.0.0

package/bin/codexa.js

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require('../dist/cli.js');

package/dist/agent.js

@@ -0,0 +1,64 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.askQuestion = askQuestion;
+const node_path_1 = __importDefault(require("node:path"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const retriever_1 = require("./retriever");
+const models_1 = require("./models");
+const SYSTEM_PROMPT = `
+You are RepoSage.
+You answer questions about a codebase using ONLY the provided code snippets.
+
+Rules:
+- Use the CODE_SNIPPET sections only.
+- Do NOT hallucinate missing files.
+- If the context does not contain enough information, say:
+  "The provided context does not contain that information."
+- Keep answers short, direct, and technical.
+`;
+async function askQuestion(cwd, config, options) {
+    const { question, session = 'default' } = options;
+    const history = await loadHistory(config.historyDir, session);
+    const matches = await (0, retriever_1.retrieveContext)(question, config);
+    if (matches.length === 0) {
+        throw new Error('No chunks found. Run `agent ingest` first.');
+    }
+    const context = (0, retriever_1.formatContext)(matches);
+    const messages = [
+        { role: 'system', content: SYSTEM_PROMPT },
+        ...history,
+        {
+            role: 'user',
+            content: `CONTEXT:\n${context}\n\nQUESTION: ${question}\nANSWER:`,
+        },
+    ];
+    const llm = (0, models_1.createLLMClient)(config);
+    let finalAnswer = '';
+    const answer = await llm.generate(messages, {
+        stream: options.stream,
+        onToken: (token) => {
+            finalAnswer += token;
+            if (options.onToken)
+                options.onToken(token);
+        },
+    });
+    if (!options.stream) {
+        finalAnswer = answer;
+    }
+    return finalAnswer;
+}
+async function loadHistory(dir, id) {
+    await fs_extra_1.default.ensureDir(dir);
+    const file = node_path_1.default.join(dir, `${id}.json`);
+    if (!(await fs_extra_1.default.pathExists(file)))
+        return [];
+    return fs_extra_1.default.readJson(file);
+}
+// async function saveHistory(dir: string, id: string, messages: any[]) {
+//   await fs.ensureDir(dir);
+//   const file = path.join(dir, `${id}.json`);
+//   await fs.writeJson(file, messages, { spaces: 2 });
+// }

package/dist/chunker.js

@@ -0,0 +1,50 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chunkFile = chunkFile;
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const node_readline_1 = __importDefault(require("node:readline"));
+async function chunkFile(filePath, maxChunkSize, overlap) {
+    const chunks = [];
+    const fileStream = fs_extra_1.default.createReadStream(filePath, { encoding: 'utf8' });
+    const rl = node_readline_1.default.createInterface({
+        input: fileStream,
+        crlfDelay: Infinity,
+    });
+    const buffer = [];
+    let currentSize = 0;
+    let startLine = 1;
+    let lineNumber = 0;
+    for await (const line of rl) {
+        lineNumber += 1;
+        buffer.push(line);
+        currentSize += line.length;
+        const shouldFlush = currentSize >= maxChunkSize;
+        if (shouldFlush) {
+            const content = buffer.join('\n');
+            chunks.push({
+                filePath,
+                startLine,
+                endLine: lineNumber,
+                content,
+            });
+            const overlapStart = Math.max(0, buffer.length - overlap);
+            const overlapBuffer = buffer.slice(overlapStart);
+            startLine = lineNumber - overlapBuffer.length + 1;
+            buffer.length = 0;
+            buffer.push(...overlapBuffer);
+            currentSize = buffer.reduce((acc, curr) => acc + curr.length, 0);
+        }
+    }
+    if (buffer.length > 0) {
+        chunks.push({
+            filePath,
+            startLine,
+            endLine: lineNumber,
+            content: buffer.join('\n'),
+        });
+    }
+    return chunks;
+}

package/dist/cli.js

@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const ora_1 = __importDefault(require("ora"));
+const config_1 = require("./config");
+const ingest_1 = require("./ingest");
+const agent_1 = require("./agent");
+const logger_1 = require("./utils/logger");
+const program = new commander_1.Command();
+program
+    .name('codexa')
+    .description('Ask questions about any local repository from the command line.')
+    .version('0.1.0');
+program
+    .command('init')
+    .description('Create a local .codexarc.json with sensible defaults.')
+    .action(async () => {
+    const cwd = process.cwd();
+    await (0, config_1.ensureConfig)(cwd);
+    logger_1.log.success('Created .codexarc.json. Update it with your provider keys if needed.');
+});
+program
+    .command('ingest')
+    .option('-f, --force', 'clear the previous index before ingesting', false)
+    .description('Chunk the current repository and store embeddings locally.')
+    .action(async (options) => {
+    const cwd = process.cwd();
+    const config = await (0, config_1.ensureConfig)(cwd);
+    try {
+        await (0, ingest_1.ingestRepository)({ cwd, config, force: options.force });
+    }
+    catch (error) {
+        console.log('Ingestion failed.');
+        handleError(error);
+    }
+});
+program
+    .command('ask')
+    .description('Ask a natural-language question about the current repo.')
+    .argument('<question...>', 'Question to ask about the codebase.')
+    .option('-s, --session <name>', 'session identifier to keep conversation context', 'default')
+    .option('--no-stream', 'disable streaming output')
+    .action(async (question, options) => {
+    const cwd = process.cwd();
+    const config = await (0, config_1.loadConfig)(cwd);
+    const prompt = question.join(' ');
+    // Commander behavior:
+    //   default: stream = true
+    //   --no-stream => stream = false
+    const stream = options.stream !== false;
+    const spinner = (0, ora_1.default)('Extracting Response...').start();
+    try {
+        const answer = await (0, agent_1.askQuestion)(cwd, config, {
+            question: prompt,
+            session: options.session,
+            stream,
+            onToken: stream
+                ? (token) => {
+                    if (spinner.isSpinning)
+                        spinner.stop();
+                    process.stdout.write(token);
+                }
+                : undefined,
+            onStatusUpdate: (status) => {
+                if (!stream)
+                    spinner.text = status;
+            },
+        });
+        spinner.stop();
+        if (!stream) {
+            console.log('\n' + answer.trim() + '\n');
+        }
+        else {
+            console.log('\n');
+        }
+    }
+    catch (error) {
+        spinner.fail('Question failed.');
+        handleError(error);
+    }
+});
+program.parseAsync(process.argv).catch(handleError);
+function handleError(error) {
+    if (error instanceof Error) {
+        logger_1.log.error(error.message);
+    }
+    else {
+        console.error(error);
+    }
+    process.exitCode = 1;
+}

package/dist/config.js

@@ -0,0 +1,97 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ensureConfig = ensureConfig;
+exports.loadConfig = loadConfig;
+exports.saveConfig = saveConfig;
+const node_path_1 = __importDefault(require("node:path"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const dotenv_1 = __importDefault(require("dotenv"));
+dotenv_1.default.config();
+const CONFIG_FILENAME = '.codexarc.json';
+const DEFAULT_CONFIG = {
+    modelProvider: 'groq',
+    model: 'llama-3.1-8b-instant',
+    embeddingProvider: 'local',
+    embeddingModel: 'Xenova/all-MiniLM-L6-v2',
+    localModelUrl: 'http://localhost:11434',
+    localModelApiKey: '',
+    maxChunkSize: 300,
+    chunkOverlap: 30,
+    includeGlobs: [
+        '**/*.ts',
+        '**/*.tsx',
+        '**/*.js',
+        '**/*.jsx',
+        '**/*.py',
+        '**/*.go',
+        '**/*.rs',
+        '**/*.java',
+        '**/*.md',
+        '**/*.json',
+    ],
+    excludeGlobs: [
+        'node_modules/**',
+        '.git/**',
+        'dist/**',
+        'build/**',
+        '.codexa/**',
+        'package-lock.json',
+    ],
+    historyDir: '.codexa/sessions',
+    dbPath: '.codexa/index.db',
+    temperature: 0.2,
+    topK: 5,
+};
+async function ensureConfig(cwd) {
+    const configPath = node_path_1.default.join(cwd, CONFIG_FILENAME);
+    if (!(await fs_extra_1.default.pathExists(configPath))) {
+        await fs_extra_1.default.writeJson(configPath, DEFAULT_CONFIG, { spaces: 2 });
+    }
+    return loadConfig(cwd);
+}
+async function loadConfig(cwd) {
+    const configPath = node_path_1.default.join(cwd, CONFIG_FILENAME);
+    let config;
+    if (!(await fs_extra_1.default.pathExists(configPath))) {
+        config = { ...DEFAULT_CONFIG };
+    }
+    else {
+        config = {
+            ...DEFAULT_CONFIG,
+            ...(await fs_extra_1.default.readJson(configPath)),
+        };
+    }
+    return hydratePaths(cwd, config);
+}
+async function saveConfig(cwd, config) {
+    const configPath = node_path_1.default.join(cwd, CONFIG_FILENAME);
+    const dehydrated = dehydratePaths(cwd, config);
+    await fs_extra_1.default.writeJson(configPath, dehydrated, { spaces: 2 });
+}
+function hydratePaths(cwd, config) {
+    const clone = { ...config };
+    clone.dbPath = resolveDataPath(cwd, config.dbPath);
+    clone.historyDir = resolveDataPath(cwd, config.historyDir);
+    return clone;
+}
+function dehydratePaths(cwd, config) {
+    const clone = { ...config };
+    clone.dbPath = makeRelative(cwd, config.dbPath);
+    clone.historyDir = makeRelative(cwd, config.historyDir);
+    return clone;
+}
+function resolveDataPath(cwd, maybeRelative) {
+    if (node_path_1.default.isAbsolute(maybeRelative)) {
+        return maybeRelative;
+    }
+    return node_path_1.default.join(cwd, maybeRelative);
+}
+function makeRelative(cwd, maybeAbsolute) {
+    if (!node_path_1.default.isAbsolute(maybeAbsolute)) {
+        return maybeAbsolute;
+    }
+    return node_path_1.default.relative(cwd, maybeAbsolute);
+}

package/dist/db.js

@@ -0,0 +1,140 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VectorStore = void 0;
+const node_path_1 = __importDefault(require("node:path"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+function cosineSimilarity(a, b) {
+    if (a.length !== b.length) {
+        return 0;
+    }
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    const len = a.length;
+    for (let i = 0; i < len; i++) {
+        const ai = a[i];
+        const bi = b[i];
+        dot += ai * bi;
+        normA += ai * ai;
+        normB += bi * bi;
+    }
+    if (normA === 0 || normB === 0) {
+        return 0;
+    }
+    const sqrtNormA = Math.sqrt(normA);
+    const sqrtNormB = Math.sqrt(normB);
+    return dot / (sqrtNormA * sqrtNormB);
+}
+class VectorStore {
+    dbPath;
+    db = null;
+    constructor(dbPath) {
+        this.dbPath = dbPath;
+    }
+    init() {
+        if (this.db) {
+            return;
+        }
+        const dir = node_path_1.default.dirname(this.dbPath);
+        fs_extra_1.default.ensureDirSync(dir);
+        this.db = new better_sqlite3_1.default(this.dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.db
+            .prepare(`CREATE TABLE IF NOT EXISTS chunks (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          file_path TEXT NOT NULL,
+          start_line INTEGER NOT NULL,
+          end_line INTEGER NOT NULL,
+          content TEXT NOT NULL,
+          compressed TEXT,
+          embedding TEXT NOT NULL
+        )`)
+            .run();
+    }
+    clear() {
+        const db = this.connection;
+        db.prepare('DELETE FROM chunks').run();
+    }
+    insertChunks(chunks) {
+        const db = this.connection;
+        const insert = db.prepare(`INSERT INTO chunks (file_path, start_line, end_line, content, compressed, embedding)
+      VALUES (@filePath, @startLine, @endLine, @content, @compressed, @embedding)`);
+        const tx = db.transaction((rows) => {
+            rows.forEach((chunk) => {
+                if (!chunk.embedding) {
+                    throw new Error(`Chunk missing embedding for ${chunk.filePath}`);
+                }
+                insert.run({
+                    filePath: chunk.filePath,
+                    startLine: chunk.startLine,
+                    endLine: chunk.endLine,
+                    content: chunk.content,
+                    compressed: chunk.compressed ?? '',
+                    embedding: JSON.stringify(chunk.embedding),
+                });
+            });
+        });
+        tx(chunks);
+    }
+    search(queryEmbedding, topK) {
+        const db = this.connection;
+        const rows = db.prepare('SELECT * FROM chunks').all();
+        if (rows.length === 0) {
+            return [];
+        }
+        // if (rows.length > 1000) {
+        //   console.error(`Searching through ${rows.length} chunks...`);
+        // }
+        // min-heap approach: keep only the top K results
+        // avoids sorting all results
+        const topResults = [];
+        const minScore = { value: -Infinity };
+        for (const row of rows) {
+            const embedding = JSON.parse(row.embedding);
+            const score = cosineSimilarity(queryEmbedding, embedding);
+            if (topResults.length >= topK && score <= minScore.value) {
+                continue;
+            }
+            const result = {
+                filePath: row.file_path,
+                startLine: row.start_line,
+                endLine: row.end_line,
+                content: row.content,
+                compressed: row.compressed ?? '',
+                embedding,
+                score,
+            };
+            if (topResults.length < topK) {
+                topResults.push(result);
+                if (result.score < minScore.value) {
+                    minScore.value = result.score;
+                }
+                if (topResults.length === topK) {
+                    topResults.sort((a, b) => a.score - b.score);
+                    minScore.value = topResults[0].score;
+                }
+            }
+            else if (score > minScore.value) {
+                topResults[0] = result;
+                topResults.sort((a, b) => a.score - b.score);
+                minScore.value = topResults[0].score;
+            }
+        }
+        const finalResults = topResults.sort((a, b) => b.score - a.score);
+        // if (rows.length > 1000) {
+        //   console.error(`Search complete, returning top ${finalResults.length} results`);
+        // }
+        return finalResults;
+    }
+    get connection() {
+        if (!this.db) {
+            this.init();
+        }
+        return this.db;
+    }
+}
+exports.VectorStore = VectorStore;

package/dist/embeddings/index.js

@@ -0,0 +1,148 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createEmbedder = createEmbedder;
+const openai_1 = __importDefault(require("openai"));
+class OpenAIEmbedder {
+    model;
+    client;
+    constructor(model, apiKey) {
+        this.model = model;
+        if (!apiKey) {
+            throw new Error('OPENAI_API_KEY is required for the OpenAI embedding provider.');
+        }
+        this.client = new openai_1.default({
+            apiKey,
+        });
+    }
+    async embed(texts) {
+        const res = await this.client.embeddings.create({
+            model: this.model,
+            input: texts,
+        });
+        return res.data.map((item) => item.embedding);
+    }
+}
+class LocalEmbedder {
+    model;
+    pipeline;
+    pipelinePromise;
+    constructor(model) {
+        this.model = model;
+    }
+    async getPipeline() {
+        if (this.pipeline) {
+            return this.pipeline;
+        }
+        if (!this.pipelinePromise) {
+            this.pipelinePromise = (async () => {
+                try {
+                    const transformersModule = await Promise.resolve().then(() => __importStar(require('@xenova/transformers')));
+                    const { pipeline } = transformersModule;
+                    this.pipeline = await pipeline('feature-extraction', this.model);
+                    // console.log(`Pipeline created successfully with ${this.model}`);
+                    return this.pipeline;
+                }
+                catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    throw new Error(`Failed to load embedding model "${this.model}": ${errorMessage}`);
+                }
+            })();
+        }
+        return this.pipelinePromise;
+    }
+    async preload() {
+        await this.getPipeline();
+        console.error(`Embedding model preloaded\n`);
+    }
+    async embed(texts) {
+        const pipe = await this.getPipeline();
+        const embeddingPromises = texts.map(async (text) => {
+            const output = await pipe(text, {
+                pooling: 'mean',
+                normalize: true,
+            });
+            return this.extractEmbedding(output);
+        });
+        return Promise.all(embeddingPromises);
+    }
+    extractEmbedding(output) {
+        const data = output?.data;
+        if (data) {
+            if (Array.isArray(data)) {
+                return data;
+            }
+            else if (data instanceof Float32Array || data instanceof Float64Array) {
+                return Array.from(data);
+            }
+            else if (typeof data === 'object' && data !== null && 'tolist' in data) {
+                return data.tolist();
+            }
+            else {
+                return Array.from(data);
+            }
+        }
+        else if (Array.isArray(output)) {
+            return output;
+        }
+        else {
+            throw new Error(`Unexpected output format from embedding model`);
+        }
+    }
+}
+// Cache embedders to avoid reloading models on every query
+const embedderCache = new Map();
+function getCacheKey(config) {
+    return `${config.embeddingProvider}:${config.embeddingModel}`;
+}
+async function createEmbedder(config) {
+    const cacheKey = getCacheKey(config);
+    const cached = embedderCache.get(cacheKey);
+    if (cached) {
+        return cached;
+    }
+    let embedder;
+    if (config.embeddingProvider === 'local') {
+        embedder = new LocalEmbedder(config.embeddingModel);
+    }
+    else {
+        embedder = new OpenAIEmbedder(config.embeddingModel, process.env.OPENAI_API_KEY);
+    }
+    embedderCache.set(cacheKey, embedder);
+    return embedder;
+}

package/dist/ingest.js

@@ -0,0 +1,69 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ingestRepository = ingestRepository;
+const node_path_1 = __importDefault(require("node:path"));
+const globby_1 = require("globby");
+const chunker_1 = require("./chunker");
+const embeddings_1 = require("./embeddings");
+const db_1 = require("./db");
+const ora_1 = __importDefault(require("ora"));
+function compressText(text, cap = 450) {
+    return text
+        .replace(/\/\*[\s\S]*?\*\//g, '')
+        .replace(/\/\/.*/g, '')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .slice(0, cap);
+}
+function tick() {
+    return new Promise((resolve) => setImmediate(resolve));
+}
+async function ingestRepository({ cwd, config, force = false, }) {
+    const spinnerFiles = (0, ora_1.default)('Finding files...').start();
+    const files = await (0, globby_1.globby)(config.includeGlobs, {
+        cwd,
+        gitignore: true,
+        ignore: config.excludeGlobs,
+        absolute: true,
+        onlyFiles: true,
+    });
+    if (!files.length) {
+        spinnerFiles.fail('No matching files found.');
+        return;
+    }
+    spinnerFiles.succeed(`Found ${files.length} files`);
+    const spinnerChunk = (0, ora_1.default)('Chunking files...').start();
+    const chunks = [];
+    for (const file of files) {
+        const ch = await (0, chunker_1.chunkFile)(file, config.maxChunkSize, config.chunkOverlap);
+        ch.forEach((c) => (c.filePath = node_path_1.default.relative(cwd, c.filePath)));
+        chunks.push(...ch);
+        await tick();
+    }
+    spinnerChunk.succeed(`Chunked files (${chunks.length} chunks)`);
+    const spinnerCompress = (0, ora_1.default)('Compressing chunks...').start();
+    chunks.forEach((c) => (c.compressed = compressText(c.content)));
+    spinnerCompress.succeed('Compression complete');
+    const spinnerEmbed = (0, ora_1.default)('Embedding chunks...').start();
+    const embedder = await (0, embeddings_1.createEmbedder)(config);
+    const batchSize = 32;
+    for (let i = 0; i < chunks.length; i += batchSize) {
+        const batch = chunks.slice(i, i + batchSize);
+        const texts = batch.map((c) => c.compressed);
+        const vectors = await embedder.embed(texts);
+        batch.forEach((c, idx) => (c.embedding = vectors[idx]));
+        await tick();
+    }
+    spinnerEmbed.succeed('Embedding complete');
+    const spinnerStore = (0, ora_1.default)('Storing chunks...').start();
+    const store = new db_1.VectorStore(config.dbPath);
+    store.init();
+    if (force)
+        store.clear();
+    store.insertChunks(chunks);
+    spinnerStore.succeed('Stored successfully');
+    (0, ora_1.default)().succeed('Ingestion complete!');
+}