@axplusb/kepler 0.0.1 → 1.0.1

package/src/context/ast-parser.mjs

@@ -0,0 +1,298 @@
+/**
+ * AST Parser — Extract structured code knowledge from source files.
+ *
+ * Multi-language regex-based extraction (no tree-sitter dependency).
+ * Returns function signatures, class definitions, imports, exports.
+ *
+ * This is Phase 3 of the Investigative Funnel:
+ * Raw code (2000 tokens) → Structured summary (200 tokens)
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+/**
+ * Analyze a source file and return structured knowledge.
+ * @param {string} filePath - absolute path
+ * @param {Object} [options]
+ * @param {number} [options.startLine] - optional line range
+ * @param {number} [options.endLine]
+ * @returns {{ success: boolean, summary: string, structure: Object }}
+ */
+export function analyzeCode(filePath, { startLine, endLine } = {}) {
+  try {
+    let content = fs.readFileSync(filePath, 'utf-8');
+    const totalLines = content.split('\n').length;
+
+    if (startLine || endLine) {
+      const lines = content.split('\n');
+      const start = (startLine || 1) - 1;
+      const end = endLine || lines.length;
+      content = lines.slice(start, end).join('\n');
+    }
+
+    const ext = path.extname(filePath).toLowerCase();
+    const relPath = filePath; // caller should make relative if needed
+
+    let structure;
+    if (['.ts', '.tsx', '.js', '.jsx', '.mjs'].includes(ext)) {
+      structure = parseJavaScriptTypeScript(content);
+    } else if (ext === '.py') {
+      structure = parsePython(content);
+    } else if (ext === '.go') {
+      structure = parseGo(content);
+    } else if (['.rs'].includes(ext)) {
+      structure = parseRust(content);
+    } else {
+      structure = parseGeneric(content);
+    }
+
+    structure.file = path.basename(filePath);
+    structure.lines = totalLines;
+    structure.language = ext.replace('.', '');
+
+    // Build concise summary
+    const summary = buildSummary(structure);
+
+    return { success: true, summary, structure };
+  } catch (err) {
+    return { success: false, summary: `Error analyzing ${filePath}: ${err.message}`, structure: {} };
+  }
+}
+
+// ── JavaScript / TypeScript ────────────────────────────────
+
+function parseJavaScriptTypeScript(content) {
+  const imports = [];
+  const exports = [];
+  const functions = [];
+  const classes = [];
+  const interfaces = [];
+  const types = [];
+  const lines = content.split('\n');
+
+  for (let i = 0; i < lines.length; i++) {
+    const trimmed = lines[i].trim();
+    const lineNum = i + 1;
+
+    // Imports
+    const importMatch = trimmed.match(/^import\s+(?:\{([^}]+)\}|(\w+))\s+from\s+['"]([^'"]+)['"]/);
+    if (importMatch) {
+      imports.push({ names: (importMatch[1] || importMatch[2] || '').trim(), from: importMatch[3] });
+      continue;
+    }
+
+    // Exports
+    if (trimmed.startsWith('export default')) {
+      exports.push('default');
+    } else if (trimmed.match(/^export\s+(const|let|var|function|class|interface|type|async)/)) {
+      const nameMatch = trimmed.match(/^export\s+(?:async\s+)?(?:const|let|var|function|class|interface|type)\s+(\w+)/);
+      if (nameMatch) exports.push(nameMatch[1]);
+    }
+
+    // Functions
+    const fnMatch = trimmed.match(/^(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(([^)]*)\)/);
+    if (fnMatch) {
+      functions.push({ name: fnMatch[1], params: fnMatch[2].trim(), line: lineNum });
+      continue;
+    }
+
+    // Arrow functions
+    const arrowMatch = trimmed.match(/^(?:export\s+)?(?:const|let)\s+(\w+)\s*=\s*(?:async\s+)?\(([^)]*)\)\s*(?::\s*\w+)?\s*=>/);
+    if (arrowMatch) {
+      functions.push({ name: arrowMatch[1], params: arrowMatch[2].trim(), line: lineNum });
+      continue;
+    }
+
+    // Classes
+    const classMatch = trimmed.match(/^(?:export\s+)?class\s+(\w+)(?:\s+extends\s+(\w+))?/);
+    if (classMatch) {
+      classes.push({ name: classMatch[1], extends: classMatch[2] || null, line: lineNum });
+      continue;
+    }
+
+    // Methods
+    const methodMatch = trimmed.match(/^(?:async\s+)?(\w+)\s*\(([^)]*)\)\s*(?::\s*\w+)?\s*\{/);
+    if (methodMatch && !['if', 'for', 'while', 'switch', 'catch'].includes(methodMatch[1])) {
+      functions.push({ name: methodMatch[1], params: methodMatch[2].trim(), line: lineNum, isMethod: true });
+    }
+
+    // Interfaces
+    const ifaceMatch = trimmed.match(/^(?:export\s+)?interface\s+(\w+)/);
+    if (ifaceMatch) interfaces.push({ name: ifaceMatch[1], line: lineNum });
+
+    // Types
+    const typeMatch = trimmed.match(/^(?:export\s+)?type\s+(\w+)\s*=/);
+    if (typeMatch) types.push({ name: typeMatch[1], line: lineNum });
+  }
+
+  return { imports, exports, functions, classes, interfaces, types };
+}
+
+// ── Python ─────────────────────────────────────────────────
+
+function parsePython(content) {
+  const imports = [];
+  const functions = [];
+  const classes = [];
+  const decorators = [];
+  const lines = content.split('\n');
+
+  for (let i = 0; i < lines.length; i++) {
+    const trimmed = lines[i].trim();
+    const lineNum = i + 1;
+
+    const importMatch = trimmed.match(/^(?:from\s+(\S+)\s+)?import\s+(.+)/);
+    if (importMatch) {
+      imports.push({ from: importMatch[1] || '', names: importMatch[2].trim() });
+      continue;
+    }
+
+    if (trimmed.startsWith('@')) {
+      decorators.push(trimmed);
+      continue;
+    }
+
+    const fnMatch = trimmed.match(/^(?:async\s+)?def\s+(\w+)\s*\(([^)]*)\)/);
+    if (fnMatch) {
+      functions.push({ name: fnMatch[1], params: fnMatch[2].trim(), line: lineNum });
+      continue;
+    }
+
+    const classMatch = trimmed.match(/^class\s+(\w+)(?:\(([^)]*)\))?/);
+    if (classMatch) {
+      classes.push({ name: classMatch[1], bases: classMatch[2] || '', line: lineNum });
+    }
+  }
+
+  return { imports, functions, classes, decorators };
+}
+
+// ── Go ─────────────────────────────────────────────────────
+
+function parseGo(content) {
+  const imports = [];
+  const functions = [];
+  const structs = [];
+
+  for (const line of content.split('\n')) {
+    const trimmed = line.trim();
+
+    const importMatch = trimmed.match(/^import\s+"([^"]+)"/);
+    if (importMatch) imports.push(importMatch[1]);
+
+    const fnMatch = trimmed.match(/^func\s+(?:\((\w+)\s+\*?(\w+)\)\s+)?(\w+)\s*\(([^)]*)\)/);
+    if (fnMatch) {
+      functions.push({
+        name: fnMatch[3],
+        params: fnMatch[4]?.trim() || '',
+        receiver: fnMatch[2] || null,
+      });
+    }
+
+    const structMatch = trimmed.match(/^type\s+(\w+)\s+struct/);
+    if (structMatch) structs.push(structMatch[1]);
+  }
+
+  return { imports, functions, structs };
+}
+
+// ── Rust ───────────────────────────────────────────────────
+
+function parseRust(content) {
+  const functions = [];
+  const structs = [];
+  const impls = [];
+
+  for (const line of content.split('\n')) {
+    const trimmed = line.trim();
+
+    const fnMatch = trimmed.match(/^(?:pub\s+)?(?:async\s+)?fn\s+(\w+)\s*\(([^)]*)\)/);
+    if (fnMatch) functions.push({ name: fnMatch[1], params: fnMatch[2].trim() });
+
+    const structMatch = trimmed.match(/^(?:pub\s+)?struct\s+(\w+)/);
+    if (structMatch) structs.push(structMatch[1]);
+
+    const implMatch = trimmed.match(/^impl\s+(?:<[^>]+>\s+)?(\w+)/);
+    if (implMatch) impls.push(implMatch[1]);
+  }
+
+  return { functions, structs, impls };
+}
+
+// ── Generic fallback ───────────────────────────────────────
+
+function parseGeneric(content) {
+  const functions = [];
+  const lines = content.split('\n');
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    // Try common patterns
+    const fnMatch = trimmed.match(/^(?:(?:pub|public|private|protected|static|async|export)\s+)*(?:function|def|fn|func)\s+(\w+)/);
+    if (fnMatch) functions.push({ name: fnMatch[1] });
+  }
+
+  return { functions };
+}
+
+// ── Summary Builder ────────────────────────────────────────
+
+function buildSummary(structure) {
+  const parts = [];
+  parts.push(`${structure.file} (${structure.lines} lines, ${structure.language})`);
+  parts.push(`Use read_file with start_line/end_line to read specific sections.`);
+  parts.push('');
+
+  if (structure.classes?.length) {
+    for (const cls of structure.classes) {
+      const ext = cls.extends ? ` extends ${cls.extends}` : (cls.bases ? `(${cls.bases})` : '');
+      const ln = cls.line ? ` [line ${cls.line}]` : '';
+      parts.push(`  class ${cls.name}${ext}${ln}`);
+    }
+  }
+
+  if (structure.interfaces?.length) {
+    const ifaceList = structure.interfaces.map(i =>
+      typeof i === 'string' ? i : `${i.name} [line ${i.line}]`
+    ).join(', ');
+    parts.push(`  interfaces: ${ifaceList}`);
+  }
+
+  if (structure.types?.length) {
+    const typeList = structure.types.map(t =>
+      typeof t === 'string' ? t : `${t.name} [line ${t.line}]`
+    ).join(', ');
+    parts.push(`  types: ${typeList}`);
+  }
+
+  if (structure.functions?.length) {
+    for (const fn of structure.functions.slice(0, 20)) {
+      const receiver = fn.receiver ? `(${fn.receiver}) ` : '';
+      const method = fn.isMethod ? '  ' : '';
+      const ln = fn.line ? ` [line ${fn.line}]` : '';
+      parts.push(`${method}  ${receiver}${fn.name}(${fn.params || ''})${ln}`);
+    }
+    if (structure.functions.length > 20) {
+      parts.push(`  ... and ${structure.functions.length - 20} more functions`);
+    }
+  }
+
+  if (structure.structs?.length) {
+    parts.push(`  structs: ${structure.structs.join(', ')}`);
+  }
+
+  if (structure.imports?.length) {
+    const importSummary = structure.imports.slice(0, 5).map(i =>
+      i.from ? `${i.from}` : (typeof i === 'string' ? i : i.names)
+    ).join(', ');
+    const more = structure.imports.length > 5 ? ` +${structure.imports.length - 5} more` : '';
+    parts.push(`  imports: ${importSummary}${more}`);
+  }
+
+  if (structure.exports?.length) {
+    parts.push(`  exports: ${structure.exports.join(', ')}`);
+  }
+
+  return parts.join('\n');
+}

package/src/context/bm25.mjs

@@ -0,0 +1,85 @@
+/**
+ * BM25 Search — T20: Lightweight text search index (~100 LOC).
+ * Parameters: k1=1.2, b=0.75 (standard BM25 tuning).
+ */
+
+export class BM25Index {
+    constructor(k1 = 1.2, b = 0.75) {
+        this.k1 = k1;
+        this.b = b;
+        this.docs = [];      // [{ id, tokens, length }]
+        this.df = new Map();  // term → doc frequency
+        this.avgDl = 0;
+        this.N = 0;
+    }
+
+    /** Tokenize text into lowercase terms. */
+    static tokenize(text) {
+        return text.toLowerCase().match(/[a-z_][a-z0-9_]*/g) || [];
+    }
+
+    /** Add a document to the index. */
+    addDocument(id, text) {
+        const tokens = BM25Index.tokenize(text);
+        const tf = new Map();
+        for (const t of tokens) tf.set(t, (tf.get(t) || 0) + 1);
+        this.docs.push({ id, tf, length: tokens.length });
+
+        for (const term of tf.keys()) {
+            this.df.set(term, (this.df.get(term) || 0) + 1);
+        }
+        this.N = this.docs.length;
+        this.avgDl = this.docs.reduce((s, d) => s + d.length, 0) / this.N;
+    }
+
+    /** Build index from array of { id, text } objects. */
+    buildIndex(documents) {
+        this.docs = [];
+        this.df = new Map();
+        for (const doc of documents) {
+            this.addDocument(doc.id, doc.text);
+        }
+    }
+
+    /** Search for query, return top-K results sorted by BM25 score. */
+    search(query, topK = 10) {
+        const queryTokens = BM25Index.tokenize(query);
+        if (queryTokens.length === 0) return [];
+
+        const scores = [];
+
+        for (const doc of this.docs) {
+            let score = 0;
+            for (const qt of queryTokens) {
+                const tf = doc.tf.get(qt) || 0;
+                if (tf === 0) continue;
+                const dfVal = this.df.get(qt) || 0;
+                const idf = Math.log((this.N - dfVal + 0.5) / (dfVal + 0.5) + 1);
+                const tfNorm = (tf * (this.k1 + 1)) / (tf + this.k1 * (1 - this.b + this.b * doc.length / this.avgDl));
+                score += idf * tfNorm;
+            }
+            if (score > 0) scores.push({ id: doc.id, score });
+        }
+
+        return scores.sort((a, b) => b.score - a.score).slice(0, topK);
+    }
+
+    /** Serialize index to JSON. */
+    toJSON() {
+        return {
+            k1: this.k1, b: this.b, N: this.N, avgDl: this.avgDl,
+            docs: this.docs.map(d => ({ id: d.id, tf: Object.fromEntries(d.tf), length: d.length })),
+            df: Object.fromEntries(this.df),
+        };
+    }
+
+    /** Load index from JSON. */
+    static fromJSON(data) {
+        const idx = new BM25Index(data.k1, data.b);
+        idx.N = data.N;
+        idx.avgDl = data.avgDl;
+        idx.df = new Map(Object.entries(data.df));
+        idx.docs = data.docs.map(d => ({ id: d.id, tf: new Map(Object.entries(d.tf)), length: d.length }));
+        return idx;
+    }
+}

package/src/context/retriever.mjs

@@ -0,0 +1,270 @@
+/**
+ * Context Retriever — T20: Unified context retrieval with BM25.
+ * Indexes project files and retrieves relevant chunks for LLM context.
+ */
+
+import { BM25Index } from './bm25.mjs';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { indexDir as getIndexDir } from '../core/paths.mjs';
+
+const IGNORED_DIRS = new Set(['.git', 'node_modules', '.orca', '__pycache__', '.venv', 'venv', 'dist', 'build', '.next']);
+const CODE_EXTS = new Set(['.js', '.mjs', '.ts', '.tsx', '.py', '.go', '.rs', '.java', '.rb', '.php', '.c', '.cpp', '.h', '.css', '.html', '.json', '.yaml', '.yml', '.toml', '.md', '.sh']);
+const MAX_FILE_SIZE = 100_000; // 100KB
+const CHUNK_LINES = 50;
+const CHUNK_OVERLAP = 10;
+
+export class ContextRetriever {
+    constructor(projectDir = process.cwd()) {
+        this.projectDir = projectDir;
+        this.indexDir = getIndexDir(projectDir);
+        this.index = null;
+        this.chunkTexts = new Map(); // id → original text content
+    }
+
+    /** Build or rebuild the search index. */
+    async buildIndex() {
+        const files = this._scanFiles(this.projectDir);
+        const documents = [];
+
+        for (const filePath of files) {
+            try {
+                const content = fs.readFileSync(filePath, 'utf-8');
+                const relPath = path.relative(this.projectDir, filePath);
+                const chunks = this._chunkFile(content, relPath);
+                documents.push(...chunks);
+            } catch { /* skip unreadable files */ }
+        }
+
+        this.index = new BM25Index();
+        this.index.buildIndex(documents);
+
+        // Store chunk texts for retrieval
+        this.chunkTexts = new Map();
+        for (const doc of documents) {
+            this.chunkTexts.set(doc.id, doc.text);
+        }
+
+        // Persist index + chunk texts
+        if (!fs.existsSync(this.indexDir)) fs.mkdirSync(this.indexDir, { recursive: true });
+        fs.writeFileSync(path.join(this.indexDir, 'bm25.json'), JSON.stringify(this.index.toJSON()));
+        fs.writeFileSync(path.join(this.indexDir, 'chunks.json'), JSON.stringify(Object.fromEntries(this.chunkTexts)));
+
+        return { fileCount: files.length, chunkCount: documents.length };
+    }
+
+    /**
+     * Incrementally update index for a single changed file.
+     * Re-chunks the file and replaces its entries in the BM25 index.
+     * ~5-50ms per file — safe to call after every edit/write.
+     */
+    updateFile(filePath) {
+        if (!this.index) {
+            if (!this.loadIndex()) return false;
+        }
+
+        const absPath = path.resolve(filePath);
+        const relPath = path.relative(this.projectDir, absPath);
+
+        // Remove old chunks for this file
+        const oldIds = new Set();
+        for (const doc of this.index.docs) {
+            if (doc.id === relPath || doc.id.startsWith(relPath + ':')) {
+                oldIds.add(doc.id);
+            }
+        }
+
+        // Collect remaining documents (excluding old chunks for this file)
+        const remainingDocs = [];
+        for (const doc of this.index.docs) {
+            if (!oldIds.has(doc.id)) {
+                // Reconstruct text from tf map for rebuild
+                remainingDocs.push({ id: doc.id, text: this.chunkTexts.get(doc.id) || '' });
+            }
+        }
+
+        // Remove old chunk texts
+        for (const id of oldIds) this.chunkTexts.delete(id);
+
+        // Re-chunk if file still exists (delete = just remove)
+        const newChunks = [];
+        if (fs.existsSync(absPath)) {
+            try {
+                const content = fs.readFileSync(absPath, 'utf-8');
+                newChunks.push(...this._chunkFile(content, relPath));
+            } catch { /* skip unreadable */ }
+        }
+
+        // Rebuild index from remaining + new chunks
+        // This is fast (~5-20ms) because BM25 build is O(n) on token count
+        const allDocs = [...remainingDocs, ...newChunks];
+        this.index = new BM25Index();
+        this.index.buildIndex(allDocs);
+
+        // Update chunk texts
+        for (const chunk of newChunks) {
+            this.chunkTexts.set(chunk.id, chunk.text);
+        }
+
+        // Persist updated index
+        try {
+            if (!fs.existsSync(this.indexDir)) fs.mkdirSync(this.indexDir, { recursive: true });
+            fs.writeFileSync(path.join(this.indexDir, 'bm25.json'), JSON.stringify(this.index.toJSON()));
+            fs.writeFileSync(path.join(this.indexDir, 'chunks.json'), JSON.stringify(Object.fromEntries(this.chunkTexts)));
+        } catch { /* best-effort persist */ }
+
+        return true;
+    }
+
+    /** Load persisted index. */
+    loadIndex() {
+        const indexPath = path.join(this.indexDir, 'bm25.json');
+        const chunksPath = path.join(this.indexDir, 'chunks.json');
+        if (!fs.existsSync(indexPath)) return false;
+        try {
+            const data = JSON.parse(fs.readFileSync(indexPath, 'utf-8'));
+            this.index = BM25Index.fromJSON(data);
+
+            // Load chunk texts if available
+            if (fs.existsSync(chunksPath)) {
+                const chunks = JSON.parse(fs.readFileSync(chunksPath, 'utf-8'));
+                this.chunkTexts = new Map(Object.entries(chunks));
+            }
+            return true;
+        } catch {
+            return false;
+        }
+    }
+
+    /** Retrieve relevant context chunks for a query, with full text. */
+    retrieve(query, topK = 10) {
+        if (!this.index) {
+            if (!this.loadIndex()) return [];
+        }
+        const results = this.index.search(query, topK);
+        // Attach chunk text to results
+        return results.map(r => ({
+            ...r,
+            text: this.chunkTexts.get(r.id) || `[File: ${r.id}]`,
+        }));
+    }
+
+    /** Scan project files respecting .gitignore-like patterns. */
+    _scanFiles(dir, depth = 0) {
+        if (depth > 15) return [];
+        const results = [];
+        let entries;
+        try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; }
+
+        for (const entry of entries) {
+            if (entry.name.startsWith('.') && IGNORED_DIRS.has(entry.name)) continue;
+            if (IGNORED_DIRS.has(entry.name)) continue;
+
+            const fullPath = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                results.push(...this._scanFiles(fullPath, depth + 1));
+            } else if (entry.isFile()) {
+                const ext = path.extname(entry.name);
+                if (!CODE_EXTS.has(ext)) continue;
+                try {
+                    const stat = fs.statSync(fullPath);
+                    if (stat.size > MAX_FILE_SIZE) continue;
+                } catch { continue; }
+                results.push(fullPath);
+            }
+        }
+        return results;
+    }
+
+    /**
+     * Chunk a file by AST boundaries (functions/classes) with line-based fallback.
+     * AST-aware chunks contain complete functions/classes — not arbitrary 50-line blocks.
+     */
+    _chunkFile(content, relPath) {
+        const lines = content.split('\n');
+
+        // Small files: single chunk
+        if (lines.length <= CHUNK_LINES) {
+            return [{ id: relPath, text: `${relPath}\n${content}` }];
+        }
+
+        // Try AST-aware chunking: split at function/class boundaries
+        const boundaries = this._findASTBoundaries(lines);
+
+        if (boundaries.length > 1) {
+            // AST-aware: chunk at function/class boundaries
+            const chunks = [];
+            for (const { name, startLine, endLine } of boundaries) {
+                const chunk = lines.slice(startLine, endLine).join('\n');
+                const id = `${relPath}:${startLine + 1}:${name}`;
+                chunks.push({ id, text: `${relPath}:${startLine + 1} (${name})\n${chunk}` });
+            }
+            return chunks;
+        }
+
+        // Fallback: line-based chunking for non-code files
+        const chunks = [];
+        for (let i = 0; i < lines.length; i += (CHUNK_LINES - CHUNK_OVERLAP)) {
+            const chunk = lines.slice(i, i + CHUNK_LINES).join('\n');
+            chunks.push({ id: `${relPath}:${i + 1}`, text: `${relPath}:${i + 1}\n${chunk}` });
+        }
+        return chunks;
+    }
+
+    /**
+     * Find function/class boundaries using regex patterns.
+     * Returns array of { name, startLine, endLine }.
+     */
+    _findASTBoundaries(lines) {
+        const boundaries = [];
+        const patterns = [
+            // Python: def/class (indentation-based)
+            /^(?:async\s+)?def\s+(\w+)\s*\(/,
+            /^class\s+(\w+)/,
+            // JS/TS: function, class, export function
+            /^(?:export\s+)?(?:async\s+)?function\s+(\w+)/,
+            /^(?:export\s+)?class\s+(\w+)/,
+            // JS/TS: const fn = arrow
+            /^(?:export\s+)?(?:const|let)\s+(\w+)\s*=\s*(?:async\s+)?\(/,
+            // Go: func
+            /^func\s+(?:\([^)]*\)\s+)?(\w+)\s*\(/,
+            // Rust: fn, struct, impl
+            /^(?:pub\s+)?(?:async\s+)?fn\s+(\w+)/,
+            /^(?:pub\s+)?struct\s+(\w+)/,
+        ];
+
+        let currentStart = 0;
+        let currentName = '_top_level';
+
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i].trimStart();
+            for (const pattern of patterns) {
+                const match = line.match(pattern);
+                if (match) {
+                    // Close previous boundary
+                    if (i > currentStart) {
+                        boundaries.push({
+                            name: currentName,
+                            startLine: currentStart,
+                            endLine: i,
+                        });
+                    }
+                    currentStart = i;
+                    currentName = match[1] || match[2] || 'unknown';
+                    break;
+                }
+            }
+        }
+
+        // Close last boundary
+        if (lines.length > currentStart) {
+            boundaries.push({
+                name: currentName,
+                startLine: currentStart,
+                endLine: lines.length,
+            });
+        }
+
+        return boundaries;
+    }
+}