@bahulam/code 0.1.1

package/src/context/skeleton.mjs

@@ -0,0 +1,134 @@
+/**
+ * Project Skeleton — lightweight codebase overview for LLM context.
+ *
+ * Generates a compact representation of the project:
+ * - File tree (directories + file names)
+ * - Function/class signatures extracted via regex (not full AST)
+ *
+ * Designed to be ~500-1000 tokens — gives the model a "map" so it knows
+ * where to look without reading every file.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+const IGNORED_DIRS = new Set(['.git', 'node_modules', '.bahulam', '__pycache__', '.venv', 'venv', 'dist', 'build', '.next', '.cache', 'coverage', '.tox']);
+const CODE_EXTS = new Set(['.js', '.mjs', '.ts', '.tsx', '.py', '.go', '.rs', '.java', '.rb', '.c', '.cpp', '.h']);
+const MAX_FILE_SIZE = 200_000;
+
+// Regex patterns for function/class signatures (multi-language)
+const SIGNATURE_PATTERNS = [
+  // Python: def/class/async def
+  /^(?:async\s+)?(?:def|class)\s+(\w+)\s*[\(:].*$/gm,
+  // JS/TS: function, class, export function, const fn =
+  /^(?:export\s+)?(?:async\s+)?(?:function\s+(\w+)|class\s+(\w+))/gm,
+  // Go: func
+  /^func\s+(?:\(.*?\)\s+)?(\w+)\s*\(/gm,
+  // Rust: fn, struct, impl
+  /^(?:pub\s+)?(?:fn|struct|impl)\s+(\w+)/gm,
+];
+
+/**
+ * Build a project skeleton — file tree + key signatures.
+ * @param {string} projectDir
+ * @param {Object} [options]
+ * @param {number} [options.maxFiles=200] — max files to include
+ * @param {number} [options.maxChars=4000] — max total chars (~1000 tokens)
+ * @returns {string} — skeleton text for LLM context
+ */
+export function buildProjectSkeleton(projectDir, { maxFiles = 200, maxChars = 4000 } = {}) {
+  const files = scanFiles(projectDir, 0, maxFiles);
+  if (files.length === 0) return '';
+
+  const parts = [];
+  parts.push(`Project: ${path.basename(projectDir)} (${files.length} source files)`);
+  parts.push('');
+
+  // Group by directory
+  const dirs = new Map();
+  for (const f of files) {
+    const rel = path.relative(projectDir, f);
+    const dir = path.dirname(rel);
+    if (!dirs.has(dir)) dirs.set(dir, []);
+    dirs.get(dir).push(rel);
+  }
+
+  // File tree
+  parts.push('## File Tree');
+  for (const [dir, dirFiles] of dirs) {
+    parts.push(`${dir}/`);
+    for (const f of dirFiles) {
+      parts.push(`  ${path.basename(f)}`);
+    }
+  }
+
+  // Key signatures (top-level functions/classes)
+  parts.push('');
+  parts.push('## Key Signatures');
+
+  let sigCount = 0;
+  for (const f of files) {
+    if (sigCount > 50) break; // Cap signatures
+    try {
+      const content = fs.readFileSync(f, 'utf-8');
+      const rel = path.relative(projectDir, f);
+      const sigs = extractSignatures(content);
+      if (sigs.length > 0) {
+        parts.push(`${rel}: ${sigs.join(', ')}`);
+        sigCount += sigs.length;
+      }
+    } catch { /* skip */ }
+  }
+
+  let skeleton = parts.join('\n');
+  if (skeleton.length > maxChars) {
+    skeleton = skeleton.slice(0, maxChars) + '\n... (truncated)';
+  }
+  return skeleton;
+}
+
+/**
+ * Extract function/class names from source code via regex.
+ */
+function extractSignatures(content) {
+  const names = new Set();
+  for (const pattern of SIGNATURE_PATTERNS) {
+    pattern.lastIndex = 0;
+    let match;
+    while ((match = pattern.exec(content)) !== null) {
+      // Take the first non-null capture group
+      const name = match[1] || match[2] || match[3];
+      if (name && !name.startsWith('_')) names.add(name);
+    }
+  }
+  return Array.from(names).slice(0, 10);
+}
+
+/**
+ * Scan project for source files.
+ */
+function scanFiles(dir, depth = 0, maxFiles = 200) {
+  if (depth > 10) return [];
+  const results = [];
+  let entries;
+  try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; }
+
+  for (const entry of entries) {
+    if (results.length >= maxFiles) break;
+    if (entry.name.startsWith('.') || IGNORED_DIRS.has(entry.name)) continue;
+
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      results.push(...scanFiles(fullPath, depth + 1, maxFiles - results.length));
+    } else if (entry.isFile()) {
+      const ext = path.extname(entry.name);
+      if (!CODE_EXTS.has(ext)) continue;
+      try {
+        const stat = fs.statSync(fullPath);
+        if (stat.size > MAX_FILE_SIZE) continue;
+      } catch { continue; }
+      results.push(fullPath);
+    }
+  }
+  return results;
+}

package/src/context/symbol-indexer.mjs

@@ -0,0 +1,375 @@
+/**
+ * Symbol Indexer — AST-based code search using tree-sitter.
+ *
+ * Parses source files into symbols (functions, classes, methods) with
+ * signatures and line numbers. Indexes symbols in BM25 for search.
+ *
+ * Memory efficient: stores symbol signatures (~50 chars) not file chunks
+ * (~2000 chars). One tree-sitter parse per file, O(n) on file size.
+ *
+ * Usage:
+ *   const indexer = new SymbolIndexer();
+ *   await indexer.init();  // load WASM grammars once
+ *   indexer.indexFile('/path/to/file.py', content);
+ *   const results = indexer.search('find_ordering_name');
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { BM25Index } from './bm25.mjs';
+
+const GRAMMAR_DIR = new URL('./grammars/', import.meta.url).pathname;
+
+const LANG_MAP = {
+    '.py': 'python',
+    '.js': 'javascript',
+    '.mjs': 'javascript',
+    '.jsx': 'javascript',
+    '.ts': 'typescript',
+    '.tsx': 'typescript',
+};
+
+/**
+ * @typedef {Object} Symbol
+ * @property {string} name
+ * @property {string} kind - 'function' | 'class' | 'method'
+ * @property {string} file - relative path
+ * @property {number} line
+ * @property {number} endLine
+ * @property {string} signature - e.g., "def find_ordering_name(self, name, opts)"
+ * @property {string} [parent] - parent class name if method
+ * @property {string} [docstring] - first line of docstring
+ */
+
+export class SymbolIndexer {
+    constructor() {
+        this._Parser = null;
+        this._languages = {};     // ext → Language
+        this._symbols = [];       // all extracted symbols
+        this._symbolMap = new Map(); // id → Symbol
+        this._bm25 = new BM25Index();
+        this._initialized = false;
+    }
+
+    /**
+     * Load tree-sitter WASM runtime + grammars. Call once per session.
+     * Lazy — only loads grammars for languages actually encountered.
+     */
+    async init() {
+        if (this._initialized) return;
+        try {
+            const TreeSitter = (await import('web-tree-sitter')).default;
+            await TreeSitter.init();
+            this._Parser = new TreeSitter();
+            this._TreeSitter = TreeSitter;
+            this._initialized = true;
+        } catch (e) {
+            // Fallback: tree-sitter not available, use regex parser
+            this._initialized = false;
+        }
+    }
+
+    async _getLanguage(ext) {
+        if (this._languages[ext]) return this._languages[ext];
+        const langName = LANG_MAP[ext];
+        if (!langName || !this._TreeSitter) return null;
+
+        // Try bundled WASM from tree-sitter-wasms package
+        const wasmPaths = [
+            path.join(GRAMMAR_DIR, `tree-sitter-${langName}.wasm`),
+        ];
+
+        // Also check node_modules
+        try {
+            const modPath = new URL(`../../node_modules/tree-sitter-wasms/out/tree-sitter-${langName}.wasm`, import.meta.url).pathname;
+            wasmPaths.push(modPath);
+        } catch { /* ignore */ }
+
+        for (const p of wasmPaths) {
+            try {
+                if (fs.existsSync(p)) {
+                    const lang = await this._TreeSitter.Language.load(p);
+                    this._languages[ext] = lang;
+                    return lang;
+                }
+            } catch { /* try next */ }
+        }
+        return null;
+    }
+
+    /**
+     * Index a single file. Extracts symbols and adds to BM25.
+     * @param {string} relPath - relative path (used as ID)
+     * @param {string} content - file content
+     */
+    async indexFile(relPath, content) {
+        const ext = path.extname(relPath).toLowerCase();
+        let symbols;
+
+        const lang = await this._getLanguage(ext);
+        if (lang && this._Parser) {
+            this._Parser.setLanguage(lang);
+            const tree = this._Parser.parse(content);
+            symbols = this._extractSymbols(tree.rootNode, relPath, ext);
+            tree.delete();
+        } else {
+            symbols = this._regexExtract(relPath, content, ext);
+        }
+
+        for (const sym of symbols) {
+            const id = `${sym.file}:${sym.line}:${sym.name}`;
+            this._symbols.push(sym);
+            this._symbolMap.set(id, sym);
+
+            // BM25 document: name + signature + parent + docstring
+            const text = [
+                sym.name,
+                sym.signature || '',
+                sym.parent ? `${sym.parent}.${sym.name}` : '',
+                sym.docstring || '',
+                sym.file,
+            ].join(' ');
+            this._bm25.addDocument(id, text);
+        }
+    }
+
+    /**
+     * Search for symbols matching a query.
+     * @param {string} query
+     * @param {number} [topK=10]
+     * @returns {Array<{symbol: Symbol, score: number}>}
+     */
+    search(query, topK = 10) {
+        const results = this._bm25.search(query, topK);
+        return results.map(r => ({
+            symbol: this._symbolMap.get(r.id),
+            score: r.score,
+            id: r.id,
+        })).filter(r => r.symbol);
+    }
+
+    /**
+     * Format search results for the agent.
+     */
+    formatResults(results) {
+        if (!results.length) return '';
+        return results.map(r => {
+            const s = r.symbol;
+            const parent = s.parent ? `${s.parent}.` : '';
+            const doc = s.docstring ? `  "${s.docstring}"` : '';
+            return `${s.file}:${s.line}  ${parent}${s.signature || s.name}${doc}`;
+        }).join('\n');
+    }
+
+    get symbolCount() { return this._symbols.length; }
+
+    // ── Tree-sitter extraction ──
+
+    _extractSymbols(node, file, ext) {
+        const symbols = [];
+        const langName = LANG_MAP[ext];
+
+        if (langName === 'python') {
+            this._walkPython(node, file, symbols, null);
+        } else if (langName === 'javascript' || langName === 'typescript') {
+            this._walkJS(node, file, symbols, null);
+        }
+        return symbols;
+    }
+
+    _walkPython(node, file, symbols, parentClass) {
+        for (let i = 0; i < node.childCount; i++) {
+            const child = node.child(i);
+            const type = child.type;
+
+            if (type === 'class_definition') {
+                const nameNode = child.childForFieldName('name');
+                const name = nameNode?.text || '';
+                const bases = child.childForFieldName('superclasses')?.text || '';
+                symbols.push({
+                    name, kind: 'class', file,
+                    line: child.startPosition.row + 1,
+                    endLine: child.endPosition.row + 1,
+                    signature: `class ${name}${bases ? `(${bases})` : ''}`,
+                    docstring: this._pyDocstring(child),
+                });
+                // Recurse into class body for methods
+                const body = child.childForFieldName('body');
+                if (body) this._walkPython(body, file, symbols, name);
+            }
+
+            else if (type === 'function_definition') {
+                const nameNode = child.childForFieldName('name');
+                const name = nameNode?.text || '';
+                const params = child.childForFieldName('parameters')?.text || '()';
+                const returnType = child.childForFieldName('return_type')?.text || '';
+                const sig = `def ${name}${params}${returnType ? ' -> ' + returnType : ''}`;
+                symbols.push({
+                    name,
+                    kind: parentClass ? 'method' : 'function',
+                    file,
+                    line: child.startPosition.row + 1,
+                    endLine: child.endPosition.row + 1,
+                    signature: sig,
+                    parent: parentClass || undefined,
+                    docstring: this._pyDocstring(child),
+                });
+            }
+
+            else if (type === 'decorated_definition') {
+                // Unwrap decorator to get the actual definition
+                for (let j = 0; j < child.childCount; j++) {
+                    const inner = child.child(j);
+                    if (inner.type === 'function_definition' || inner.type === 'class_definition') {
+                        this._walkPython(child, file, symbols, parentClass);
+                        break;
+                    }
+                }
+            }
+
+            else {
+                // Recurse for module-level statements
+                if (!parentClass && child.childCount > 0) {
+                    this._walkPython(child, file, symbols, parentClass);
+                }
+            }
+        }
+    }
+
+    _pyDocstring(defNode) {
+        const body = defNode.childForFieldName('body');
+        if (!body || body.childCount === 0) return '';
+        const first = body.child(0);
+        if (first?.type === 'expression_statement') {
+            const expr = first.child(0);
+            if (expr?.type === 'string' || expr?.type === 'concatenated_string') {
+                const raw = expr.text;
+                // Extract first line of docstring
+                const content = raw.replace(/^['"`]{1,3}/, '').replace(/['"`]{1,3}$/, '');
+                const firstLine = content.split('\n')[0].trim();
+                return firstLine.slice(0, 120);
+            }
+        }
+        return '';
+    }
+
+    _walkJS(node, file, symbols, parentClass) {
+        for (let i = 0; i < node.childCount; i++) {
+            const child = node.child(i);
+            const type = child.type;
+
+            if (type === 'class_declaration' || type === 'class') {
+                const nameNode = child.childForFieldName('name');
+                const name = nameNode?.text || '';
+                symbols.push({
+                    name, kind: 'class', file,
+                    line: child.startPosition.row + 1,
+                    endLine: child.endPosition.row + 1,
+                    signature: `class ${name}`,
+                });
+                const body = child.childForFieldName('body');
+                if (body) this._walkJS(body, file, symbols, name);
+            }
+
+            else if (type === 'function_declaration' || type === 'method_definition') {
+                const nameNode = child.childForFieldName('name');
+                const name = nameNode?.text || '';
+                const params = child.childForFieldName('parameters')?.text || '()';
+                symbols.push({
+                    name,
+                    kind: parentClass ? 'method' : 'function',
+                    file,
+                    line: child.startPosition.row + 1,
+                    endLine: child.endPosition.row + 1,
+                    signature: `${parentClass ? '' : 'function '}${name}${params}`,
+                    parent: parentClass || undefined,
+                });
+            }
+
+            else if (type === 'export_statement' || type === 'lexical_declaration') {
+                this._walkJS(child, file, symbols, parentClass);
+            }
+
+            else if (child.childCount > 0 && !parentClass) {
+                this._walkJS(child, file, symbols, parentClass);
+            }
+        }
+    }
+
+    // ── Regex fallback (no tree-sitter) ──
+
+    _regexExtract(file, content, ext) {
+        const symbols = [];
+        const lines = content.split('\n');
+        let currentClass = null;
+
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            const trimmed = line.trim();
+            const lineNum = i + 1;
+            const indent = line.length - line.trimStart().length;
+
+            // Python
+            if (ext === '.py') {
+                const classMatch = trimmed.match(/^class\s+(\w+)(?:\(([^)]*)\))?/);
+                if (classMatch) {
+                    currentClass = classMatch[1];
+                    symbols.push({
+                        name: currentClass, kind: 'class', file, line: lineNum,
+                        signature: `class ${currentClass}${classMatch[2] ? `(${classMatch[2]})` : ''}`,
+                    });
+                    continue;
+                }
+                const fnMatch = trimmed.match(/^(?:async\s+)?def\s+(\w+)\s*\(([^)]*)\)/);
+                if (fnMatch) {
+                    const isMethod = indent >= 4 && currentClass;
+                    symbols.push({
+                        name: fnMatch[1],
+                        kind: isMethod ? 'method' : 'function',
+                        file, line: lineNum,
+                        signature: `def ${fnMatch[1]}(${fnMatch[2]})`,
+                        parent: isMethod ? currentClass : undefined,
+                    });
+                    continue;
+                }
+                if (indent === 0 && !trimmed.startsWith('#') && trimmed) {
+                    currentClass = null;
+                }
+            }
+
+            // JS/TS
+            if (['.js', '.mjs', '.ts', '.tsx', '.jsx'].includes(ext)) {
+                const fnMatch = trimmed.match(/^(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(([^)]*)\)/);
+                if (fnMatch) {
+                    symbols.push({ name: fnMatch[1], kind: 'function', file, line: lineNum, signature: `function ${fnMatch[1]}(${fnMatch[2]})` });
+                }
+                const classMatch = trimmed.match(/^(?:export\s+)?class\s+(\w+)/);
+                if (classMatch) {
+                    symbols.push({ name: classMatch[1], kind: 'class', file, line: lineNum, signature: `class ${classMatch[1]}` });
+                }
+            }
+        }
+        return symbols;
+    }
+
+    // ── Serialization ──
+
+    toJSON() {
+        return {
+            symbols: this._symbols,
+            bm25: this._bm25.toJSON(),
+        };
+    }
+
+    static fromJSON(data) {
+        const indexer = new SymbolIndexer();
+        indexer._initialized = true; // don't need tree-sitter for search
+        indexer._symbols = data.symbols || [];
+        indexer._bm25 = BM25Index.fromJSON(data.bm25);
+        for (const sym of indexer._symbols) {
+            const id = `${sym.file}:${sym.line}:${sym.name}`;
+            indexer._symbolMap.set(id, sym);
+        }
+        return indexer;
+    }
+}

package/src/core/agent-history.mjs

@@ -0,0 +1,111 @@
+/**
+ * Structured live history for backend continuity.
+ *
+ * The terminal display history is intentionally human-friendly. This builder
+ * keeps the backend payload provider-shaped: assistant text/tool_use blocks,
+ * followed by user tool_result blocks, appended in the order they happened.
+ */
+
+const DEFAULT_MAX_TOOL_RESULT_CHARS = 200_000;
+
+function asString(value) {
+    if (value == null) return '';
+    if (typeof value === 'string') return value;
+    try {
+        return JSON.stringify(value);
+    } catch {
+        return String(value);
+    }
+}
+
+function maybeTruncate(text, maxChars) {
+    const value = asString(text);
+    if (!maxChars || value.length <= maxChars) return value;
+    return `${value.slice(0, maxChars)}\n\n[Bahulam Code truncated this tool result from ${value.length} to ${maxChars} characters for live session continuity.]`;
+}
+
+function mergeTextBlock(blocks, text) {
+    if (!text) return;
+    const last = blocks[blocks.length - 1];
+    if (last?.type === 'text') {
+        last.text += text;
+    } else {
+        blocks.push({ type: 'text', text });
+    }
+}
+
+export class AgentHistoryTurnBuilder {
+    constructor({ maxToolResultChars = DEFAULT_MAX_TOOL_RESULT_CHARS } = {}) {
+        this.maxToolResultChars = maxToolResultChars;
+        this.messages = [];
+        this.assistantBlocks = [];
+        this.toolUseIds = new Set();
+        this.toolResultIds = new Set();
+    }
+
+    addAssistantText(text) {
+        mergeTextBlock(this.assistantBlocks, asString(text));
+    }
+
+    addToolUse(data = {}) {
+        if (data.internal || data.sub_agent) return false;
+        const id = data.call_id || data.request_id || data.id;
+        const name = data.tool || data.name;
+        if (!id || !name) return false;
+        this.assistantBlocks.push({
+            type: 'tool_use',
+            id,
+            name,
+            input: data.args || data.input || {},
+        });
+        this.toolUseIds.add(id);
+        return true;
+    }
+
+    addToolResult(data = {}) {
+        if (data.internal || data.sub_agent) return false;
+        const id = data.call_id || data._callId || data.request_id || data.id || data.tool_use_id;
+        if (this.toolResultIds.has(id)) return false;
+        if (!id) return false;
+        if (!this.toolUseIds.has(id)) {
+            const name = data.tool || data.name;
+            if (!name) return false;
+            this.assistantBlocks.push({
+                type: 'tool_use',
+                id,
+                name,
+                input: data.args || data.input || {},
+            });
+            this.toolUseIds.add(id);
+        }
+
+        this.flushAssistant();
+        this.messages.push({
+            role: 'user',
+            content: [{
+                type: 'tool_result',
+                tool_use_id: id,
+            content: maybeTruncate(data.llm_content ?? data.output ?? data.result ?? data.message ?? '', this.maxToolResultChars),
+                ...(data.success === false || data.is_error ? { is_error: true } : {}),
+            }],
+        });
+        this.toolResultIds.add(id);
+        return true;
+    }
+
+    flushAssistant() {
+        const blocks = this.assistantBlocks.filter(block => {
+            if (block.type === 'text') return Boolean(block.text);
+            return true;
+        });
+        if (!blocks.length) return false;
+        this.messages.push({ role: 'assistant', content: blocks });
+        this.assistantBlocks = [];
+        return true;
+    }
+
+    finish() {
+        this.flushAssistant();
+        return this.messages;
+    }
+}