lance-context 0.1.0 → 1.0.1

package/dist/search/indexer.d.ts

@@ -1,39 +1,275 @@
 import type { EmbeddingBackend } from '../embeddings/index.js';
+import { type ConceptCluster, type ClusteringResult, type ClusteringOptions } from './clustering.js';
+/**
+ * Represents a chunk of code that has been indexed.
+ * Each chunk contains a portion of a source file with its location metadata.
+ */
 export interface CodeChunk {
+    /** Unique identifier for this chunk (format: filepath:startLine-endLine) */
     id: string;
-    filePath: string;
+    /** Relative path to the source file from the project root */
+    filepath: string;
+    /** The actual source code content of this chunk */
     content: string;
+    /** Starting line number in the source file (1-indexed) */
     startLine: number;
+    /** Ending line number in the source file (1-indexed) */
     endLine: number;
+    /** Programming language of the source code */
     language: string;
+    /** Vector embedding for semantic search (populated during indexing) */
     embedding?: number[];
+    /** Type of code symbol (function, class, method, etc.) - from AST chunking */
+    symbolType?: 'function' | 'class' | 'method' | 'interface' | 'type' | 'variable' | 'import' | 'other';
+    /** Name of the code symbol (e.g., 'UserService', 'MyClass.constructor') - from AST chunking */
+    symbolName?: string;
 }
+/**
+ * Status information about the code index.
+ */
 export interface IndexStatus {
+    /** Whether the codebase has been indexed */
     indexed: boolean;
+    /** Number of files that have been indexed */
     fileCount: number;
+    /** Total number of code chunks in the index */
     chunkCount: number;
+    /** ISO timestamp of the last index update, or null if never indexed */
     lastUpdated: string | null;
+    /** Path to the LanceDB index directory */
     indexPath: string;
+    /** Name of the embedding backend used */
+    embeddingBackend?: string;
+    /** Model identifier used for embeddings */
+    embeddingModel?: string;
+    /** Whether index corruption was detected */
+    corrupted?: boolean;
+    /** Description of detected corruption */
+    corruptionReason?: string;
+    /** Whether the current embedding backend differs from the indexed backend */
+    backendMismatch?: boolean;
+    /** Description of the backend mismatch */
+    backendMismatchReason?: string;
+    /** Whether the index is currently being rebuilt due to backend change */
+    isIndexing?: boolean;
+}
+/**
+ * Options for searching similar code.
+ */
+export interface SearchSimilarOptions {
+    /** File path to find similar code for (relative to project root) */
+    filepath?: string;
+    /** Starting line number (1-indexed, requires filepath) */
+    startLine?: number;
+    /** Ending line number (1-indexed, requires filepath) */
+    endLine?: number;
+    /** Code snippet to find similar code for (alternative to filepath) */
+    code?: string;
+    /** Maximum number of results to return (default: 10) */
+    limit?: number;
+    /** Minimum similarity score threshold 0-1 (default: 0) */
+    threshold?: number;
+    /** Exclude the source chunk from results (default: true) */
+    excludeSelf?: boolean;
+}
+/**
+ * Result from similar code search, includes similarity score.
+ */
+export interface SimilarCodeResult extends CodeChunk {
+    /** Similarity score from 0 to 1 (1 = identical) */
+    similarity: number;
 }
 /**
- * Code indexer using LanceDB for vector storage
+ * Options for searching code.
  */
+export interface SearchOptions {
+    /** Natural language query to search for */
+    query: string;
+    /** Maximum number of results to return (default: 10) */
+    limit?: number;
+    /** Glob pattern to filter results by file path (e.g., "src/\**", "!test/\**") */
+    pathPattern?: string;
+    /** Filter results to specific languages (e.g., ["typescript", "javascript"]) */
+    languages?: string[];
+}
+/**
+ * Progress information during indexing operations.
+ * Used to report status to callers via the progress callback.
+ */
+export interface IndexProgress {
+    /** Current phase of the indexing process */
+    phase: 'scanning' | 'chunking' | 'embedding' | 'storing' | 'complete';
+    /** Current progress count within the phase */
+    current: number;
+    /** Total items to process in the current phase */
+    total: number;
+    /** Human-readable status message */
+    message: string;
+}
+/**
+ * Callback function for receiving indexing progress updates.
+ */
+export type ProgressCallback = (progress: IndexProgress) => void;
+/**
+ * Summary of the codebase structure and concept areas
+ */
+export interface CodebaseSummary {
+    /** Total number of files indexed */
+    totalFiles: number;
+    /** Total number of code chunks */
+    totalChunks: number;
+    /** Languages detected in the codebase */
+    languages: {
+        language: string;
+        fileCount: number;
+        chunkCount: number;
+    }[];
+    /** Discovered concept clusters */
+    concepts: ConceptCluster[];
+    /** Quality score for the clustering (silhouette score, -1 to 1) */
+    clusteringQuality: number;
+    /** Timestamp when summary was generated */
+    generatedAt: string;
+}
 export declare class CodeIndexer {
     private db;
     private table;
+    private metadataTable;
     private embeddingBackend;
     private indexPath;
     private projectPath;
+    private config;
+    /** LRU cache for query embeddings to avoid recomputing identical queries */
+    private queryEmbeddingCache;
     constructor(projectPath: string, embeddingBackend: EmbeddingBackend);
     initialize(): Promise<void>;
+    private get metadataPath();
+    /**
+     * Save index metadata to disk
+     */
+    private saveIndexMetadata;
+    /**
+     * Load index metadata from disk
+     */
+    private loadIndexMetadata;
+    /**
+     * Get the modification time of a file
+     */
+    private getFileMtime;
+    /**
+     * Get stored metadata for all indexed files
+     */
+    private getStoredMetadata;
+    /**
+     * Detect which files have been added, modified, or deleted
+     */
+    private detectFileChanges;
+    /**
+     * Save metadata for indexed files
+     */
+    private saveFileMetadata;
     getStatus(): Promise<IndexStatus>;
-    indexCodebase(patterns?: string[], excludePatterns?: string[]): Promise<{
+    /**
+     * Check if the current embedding backend differs from the one used to create the index.
+     * Returns mismatch status and reason if mismatched.
+     */
+    private checkBackendMismatch;
+    /**
+     * Validate index integrity by checking metadata consistency.
+     * Returns corruption status and reason if corrupted.
+     */
+    private validateIndexIntegrity;
+    indexCodebase(patterns?: string[], excludePatterns?: string[], forceReindex?: boolean, onProgress?: ProgressCallback, autoRepair?: boolean): Promise<{
         filesIndexed: number;
         chunksCreated: number;
+        incremental: boolean;
+        repaired?: boolean;
     }>;
+    /**
+     * Perform a full reindex of all files
+     */
+    private indexFull;
+    /**
+     * Perform incremental indexing - only process changed files
+     */
+    private indexIncremental;
+    /**
+     * Generate embeddings for chunks in batches
+     */
+    private embedChunks;
     private chunkFile;
+    /**
+     * Chunk a file using AST-aware parsing
+     */
+    private chunkFileWithAST;
+    /**
+     * Chunk a file using tree-sitter AST parsing (Python, Go, Rust, Java, Kotlin)
+     */
+    private chunkFileWithTreeSitter;
+    /**
+     * Chunk a file using line-based splitting (fallback)
+     */
+    private chunkFileByLines;
     private getLanguage;
+    /**
+     * Get query embedding from cache or compute it.
+     * Uses LRU eviction when cache is full.
+     */
+    private getQueryEmbedding;
+    /**
+     * Check if a filepath matches a glob pattern.
+     * Supports negation patterns starting with '!'.
+     */
+    private matchesPathPattern;
+    /**
+     * Search with options object
+     */
+    search(options: SearchOptions): Promise<CodeChunk[]>;
+    /**
+     * Search with query string and optional limit (backward compatible)
+     */
     search(query: string, limit?: number): Promise<CodeChunk[]>;
+    /**
+     * Calculate keyword match score for hybrid search
+     */
+    private calculateKeywordScore;
+    /**
+     * Find code chunks semantically similar to a given code snippet or file location.
+     * This is useful for finding duplicate logic, similar implementations, or related code.
+     */
+    searchSimilar(options: SearchSimilarOptions): Promise<SimilarCodeResult[]>;
     clearIndex(): Promise<void>;
+    private get clusteringMetadataPath();
+    /**
+     * Clear clustering metadata file
+     */
+    private clearClusteringMetadata;
+    /**
+     * Save clustering result to metadata file
+     */
+    private saveClusteringMetadata;
+    /**
+     * Load clustering result from metadata file
+     */
+    private loadClusteringMetadata;
+    /**
+     * Cluster the indexed codebase into semantic concept areas.
+     * Uses k-means clustering on embeddings to discover related code groups.
+     */
+    clusterConcepts(options?: ClusteringOptions): Promise<ClusteringResult>;
+    /**
+     * List all discovered concept clusters.
+     * Returns cached clustering result if available, otherwise clusters first.
+     */
+    listConcepts(forceRecluster?: boolean): Promise<ConceptCluster[]>;
+    /**
+     * Search for code within a specific concept cluster.
+     * Returns chunks that belong to the specified cluster, optionally filtered by query.
+     */
+    searchByConcept(conceptId: number, query?: string, limit?: number): Promise<CodeChunk[]>;
+    /**
+     * Generate a comprehensive summary of the codebase including concept areas.
+     */
+    summarizeCodebase(clusteringOptions?: ClusteringOptions): Promise<CodebaseSummary>;
 }
 //# sourceMappingURL=indexer.d.ts.map

package/dist/search/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/search/indexer.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE/D,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB;AAKD;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,EAAE,CAAmC;IAC7C,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;gBAEhB,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,gBAAgB;IAM7D,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC;IA0BjC,aAAa,CACjB,QAAQ,GAAE,MAAM,EAAoF,EACpG,eAAe,GAAE,MAAM,EAAsE,GAC5F,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;YA+D7C,SAAS;IA2BvB,OAAO,CAAC,WAAW;IA2Bb,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAsB/D,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;CAOlC"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/search/indexer.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAa/D,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EAEvB,MAAM,iBAAiB,CAAC;AAezB;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,4EAA4E;IAC5E,EAAE,EAAE,MAAM,CAAC;IACX,6DAA6D;IAC7D,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,OAAO,EAAE,MAAM,CAAC;IAChB,0DAA0D;IAC1D,SAAS,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,OAAO,EAAE,MAAM,CAAC;IAChB,8CAA8C;IAC9C,QAAQ,EAAE,MAAM,CAAC;IACjB,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,8EAA8E;IAC9E,UAAU,CAAC,EACP,UAAU,GACV,OAAO,GACP,QAAQ,GACR,WAAW,GACX,MAAM,GACN,UAAU,GACV,QAAQ,GACR,OAAO,CAAC;IACZ,+FAA+F;IAC/F,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,4CAA4C;IAC5C,OAAO,EAAE,OAAO,CAAC;IACjB,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;IACnB,uEAAuE;IACvE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,2CAA2C;IAC3C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,4CAA4C;IAC5C,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,6EAA6E;IAC7E,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,0CAA0C;IAC1C,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,yEAAyE;IACzE,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AA2BD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAkB,SAAQ,SAAS;IAClD,mDAAmD;IACnD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iFAAiF;IACjF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gFAAgF;IAChF,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,4CAA4C;IAC5C,KAAK,EAAE,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,CAAC;IACtE,8CAA8C;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;AAEjE;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,SAAS,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACzE,kCAAkC;IAClC,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,mEAAmE;IACnE,iBAAiB,EAAE,MAAM,CAAC;IAC1B,2CAA2C;IAC3C,WAAW,EAAE,MAAM,CAAC;CACrB;AAwCD,qBAAa,WAAW;IACtB,OAAO,CAAC,EAAE,CAAmC;IAC7C,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,MAAM,CAAmC;IACjD,4EAA4E;IAC5E,OAAO,CAAC,mBAAmB,CAAoC;gBAEnD,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,gBAAgB;IAM7D,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAMjC,OAAO,KAAK,YAAY,GAEvB;IAED;;OAEG;YACW,iBAAiB;IAwB/B;;OAEG;YACW,iBAAiB;IAS/B;;OAEG;YACW,YAAY;IAK1B;;OAEG;YACW,iBAAiB;IAe/B;;OAEG;YACW,iBAAiB;IAoC/B;;OAEG;YACW,gBAAgB;IAsBxB,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC;IA2CvC;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IA2C5B;;;OAGG;YACW,sBAAsB;IAsC9B,aAAa,CACjB,QAAQ,CAAC,EAAE,MAAM,EAAE,EACnB,eAAe,CAAC,EAAE,MAAM,EAAE,EAC1B,YAAY,GAAE,OAAe,EAC7B,UAAU,CAAC,EAAE,gBAAgB,EAC7B,UAAU,GAAE,OAAe,GAC1B,OAAO,CAAC;QACT,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,OAAO,CAAC;QACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IAsFF;;OAEG;YACW,SAAS;IAsEvB;;OAEG;YACW,gBAAgB;IAsH9B;;OAEG;YACW,WAAW;YAuBX,SAAS;IAkCvB;;OAEG;YACW,gBAAgB;IAoB9B;;OAEG;YACW,uBAAuB;IAoBrC;;OAEG;YACW,gBAAgB;IA+B9B,OAAO,CAAC,WAAW;IA2BnB;;;OAGG;YACW,iBAAiB;IA0B/B;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAQ1B;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAC1D;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAyEjE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAsC7B;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAuG1E,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAYjC,OAAO,KAAK,sBAAsB,GAEjC;IAED;;OAEG;YACW,uBAAuB;IAQrC;;OAEG;YACW,sBAAsB;IAYpC;;OAEG;YACW,sBAAsB;IAepC;;;OAGG;IACG,eAAe,CAAC,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA8BjF;;;OAGG;IACG,YAAY,CAAC,cAAc,GAAE,OAAe,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAY9E;;;OAGG;IACG,eAAe,CACnB,SAAS,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,EACd,KAAK,GAAE,MAAW,GACjB,OAAO,CAAC,SAAS,EAAE,CAAC;IAyEvB;;OAEG;IACG,iBAAiB,CAAC,iBAAiB,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;CA6DzF"}