@aiderdesk/tree-sitter-utils 0.1.0

package/README.md

@@ -0,0 +1,151 @@
+# @aiderdesk/tree-sitter-utils
+
+A comprehensive tree-sitter library for code analysis, symbol extraction, and repository mapping.
+
+## Features
+
+- 🌳 **Tree-Sitter Based**: Uses tree-sitter for accurate AST parsing
+- 📦 **On-Demand WASM Loading**: Automatically downloads language WASM files when needed
+- 🔍 **Symbol Extraction**: Extract functions, classes, variables, and other symbols from code
+- 🗺️ **Repository Mapping**: Generate comprehensive repository maps with PageRank-based symbol ranking
+- 💾 **Smart Caching**: SQLite-based caching for fast repeated analysis
+- 🌐 **Multi-Language Support**: Supports 18+ programming languages out of the box
+
+## Installation
+
+```bash
+npm install @aiderdesk/tree-sitter-utils
+```
+
+## Quick Start
+
+### Extract Symbols from Files
+
+```typescript
+import { extractSymbols } from '@aiderdesk/tree-sitter-utils';
+
+const symbols = await extractSymbols([
+  '/path/to/file.ts',
+  '/path/to/another-file.py'
+]);
+
+console.log(symbols);
+// [
+//   { name: 'MyClass', kind: 'def', line: 10, rel_fname: 'file.ts', ... },
+//   { name: 'myFunction', kind: 'def', line: 25, rel_fname: 'file.ts', ... }
+// ]
+```
+
+### Generate Repository Map
+
+```typescript
+import { getRepoMap } from '@aiderdesk/tree-sitter-utils';
+
+const map = await getRepoMap('/path/to/project', {
+  maxLines: 1000,
+  excludePatterns: ['node_modules/**', 'dist/**']
+});
+
+console.log(map);
+// src/index.ts:
+// ├── class MyClass
+// ├── function myFunction
+// ...
+```
+
+### Advanced Usage with TreeSitterAnalyzer
+
+```typescript
+import { TreeSitterAnalyzer } from '@aiderdesk/tree-sitter-utils';
+
+const analyzer = new TreeSitterAnalyzer();
+
+// Initialize (optional - lazy initialization supported)
+await analyzer.initialize();
+
+// Get repository map
+const repoMap = await analyzer.getRepoMap('/path/to/project', {
+  maxLines: 500,
+  mentionedFiles: new Set(['src/important.ts']),
+  mentionedIdents: new Set(['ImportantClass'])
+});
+
+// Extract symbols
+const symbols = await analyzer.extractSymbols(['/path/to/file.ts'], {
+  useCache: true
+});
+
+// Parse single file
+const parseResult = await analyzer.parseFile('/path/to/file.ts');
+```
+
+## Supported Languages
+
+- Python
+- JavaScript / JSX
+- TypeScript / TSX
+- Java
+- Go
+- Rust
+- C / C++
+- C#
+- Ruby
+- PHP
+- Scala
+- Dart
+- OCaml / OCaml Interface
+- Haskell
+- Julia
+
+## Configuration Options
+
+### RepoMapOptions
+
+```typescript
+interface RepoMapOptions {
+  root: string;                    // Root directory of the project
+  files?: string[];                // Specific files to analyze (optional)
+  excludePatterns?: string[];      // Glob patterns to exclude
+  mentionedFiles?: Set<string>;    // Files to prioritize in ranking
+  mentionedIdents?: Set<string>;   // Identifiers to prioritize
+  chatFiles?: Set<string>;         // Files to exclude from output
+  maxLines?: number;               // Maximum lines in output
+}
+```
+
+### SymbolOptions
+
+```typescript
+interface SymbolOptions {
+  useCache?: boolean;              // Use SQLite cache (default: true)
+  rootDir?: string;                // Root directory for relative paths
+}
+```
+
+## How It Works
+
+1. **On-Demand WASM Loading**: When a language is first encountered, the library automatically downloads its WASM file from unpkg and caches it in `~/.cache/aiderdesk/tree-sitter/wasm/`
+
+2. **Tree-Sitter Parsing**: Uses tree-sitter queries (`.scm` files) to extract definitions and references from source code
+
+3. **PageRank Algorithm**: Builds a dependency graph and applies PageRank to rank symbols by importance
+
+4. **Smart Caching**: SQLite-based caching tracks file modification times to avoid re-parsing unchanged files
+
+## API Reference
+
+### Convenience Functions
+
+- `extractSymbols(filePaths: string[], options?: SymbolOptions): Promise<Symbol[]>`
+- `getRepoMap(root: string, options?: RepoMapOptions): Promise<string>`
+
+### TreeSitterAnalyzer Class
+
+- `initialize(): Promise<void>` - Initialize the analyzer
+- `getRepoMap(options: RepoMapOptions): Promise<string>` - Generate repository map
+- `extractSymbols(filePaths: string[], options?: SymbolOptions): Promise<Symbol[]>` - Extract symbols
+- `parseFile(filePath: string): Promise<ParseResult>` - Parse single file
+
+## License
+
+MIT

package/dist/cache-manager.d.ts

@@ -0,0 +1,16 @@
+import { Tag } from './types';
+export declare class CacheManager {
+    private db;
+    private dbPath;
+    constructor(dbPath?: string);
+    initialize(): Promise<void>;
+    getFileCache(filePath: string): Promise<{
+        tags: Tag[];
+        mtime: number;
+    } | null>;
+    setFileCache(filePath: string, mtime: number, tags: Tag[]): Promise<void>;
+    getFileMtime(filePath: string): Promise<number | null>;
+    clearCache(): Promise<void>;
+    close(): void;
+}
+//# sourceMappingURL=cache-manager.d.ts.map

package/dist/cache-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache-manager.d.ts","sourceRoot":"","sources":["../src/cache-manager.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,GAAG,EAAE,MAAM,SAAS,CAAC;AAmB9B,qBAAa,YAAY;IACvB,OAAO,CAAC,EAAE,CAA6B;IACvC,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,MAAM;IAIrB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAuB3B,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,GAAG,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAsB9E,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAuBzE,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAUtD,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAcjC,KAAK,IAAI,IAAI;CAOd"}

package/dist/constants.d.ts

@@ -0,0 +1,4 @@
+export declare const RESOURCES_DIR: string;
+export declare const QUERIES_DIR: string;
+export declare const WASM_DIR: string;
+//# sourceMappingURL=constants.d.ts.map

package/dist/constants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAeA,eAAO,MAAM,aAAa,QAAwC,CAAC;AACnE,eAAO,MAAM,WAAW,QAAsC,CAAC;AAC/D,eAAO,MAAM,QAAQ,QAAmC,CAAC"}

package/dist/file-utils.d.ts

@@ -0,0 +1,2 @@
+export declare const getAllFiles: (baseDir: string, excludePatterns?: string[]) => Promise<string[]>;
+//# sourceMappingURL=file-utils.d.ts.map

package/dist/file-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../src/file-utils.ts"],"names":[],"mappings":"AAsFA,eAAO,MAAM,WAAW,GAAU,SAAS,MAAM,EAAE,kBAAiB,MAAM,EAAO,KAAG,OAAO,CAAC,MAAM,EAAE,CAsBnG,CAAC"}

package/dist/graph-builder.d.ts

@@ -0,0 +1,13 @@
+import Graph from 'graphology';
+import { Tag } from './types';
+export declare class GraphBuilder {
+    buildDependencyGraph(tags: Tag[]): {
+        graph: Graph;
+        defines: Map<string, Set<string>>;
+        references: Map<string, string[]>;
+    };
+    private isSnakeCase;
+    private isKebabCase;
+    private isCamelCase;
+}
+//# sourceMappingURL=graph-builder.d.ts.map

package/dist/graph-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-builder.d.ts","sourceRoot":"","sources":["../src/graph-builder.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,YAAY,CAAC;AAE/B,OAAO,EAAE,GAAG,EAAE,MAAM,SAAS,CAAC;AAG9B,qBAAa,YAAY;IACvB,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG;QACjC,KAAK,EAAE,KAAK,CAAC;QACb,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAClC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;KACnC;IAiFD,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,WAAW;CAGpB"}

package/dist/index.d.ts

@@ -0,0 +1,27 @@
+export type { Tag, Symbol, RepoMapOptions, SymbolOptions, GraphEdge, RankedDefinition, ParseResult, LanguageConfig, Logger } from './types';
+export { TreeSitterAnalyzer } from './tree-sitter-analyzer';
+export { TreeSitterParser } from './tree-sitter-parser';
+export { SymbolExtractor } from './symbol-extractor';
+export { GraphBuilder } from './graph-builder';
+export { PageRankCalculator } from './pagerank';
+export { TreeRenderer } from './tree-renderer';
+export { CacheManager } from './cache-manager';
+export { detectLanguage, isLanguageSupported, getSupportedExtensions, getSupportedLanguages } from './language-detector';
+export { downloadWasmForLanguage, getLanguageWasmConfig, getSupportedLanguagesForWasm } from './wasm-downloader';
+export { setLogger, getLogger } from './logger';
+export { getAllFiles } from './file-utils';
+/**
+ * Extract symbols from files
+ * @param filePaths - Array of file paths to extract symbols from
+ * @param options - Symbol extraction options
+ * @returns Array of symbols
+ */
+export declare function extractSymbols(filePaths: string[], options?: import('./types').SymbolOptions): Promise<import('./types').Symbol[]>;
+/**
+ * Generate a repository map
+ * @param root - Root directory of the repository
+ * @param options - Repository map options
+ * @returns Formatted repository map string
+ */
+export declare function getRepoMap(root: string, options?: Omit<import('./types').RepoMapOptions, 'root'>): Promise<string>;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EAAE,GAAG,EAAE,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,EAAE,gBAAgB,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAG5I,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACzH,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AACjH,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAe3C;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,OAAO,SAAS,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,SAAS,EAAE,MAAM,EAAE,CAAC,CAGxI;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,SAAS,EAAE,cAAc,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAGxH"}