lmgrep 0.1.0 → 0.1.2

package/dist/index.d.ts

@@ -0,0 +1,32 @@
+import { type Embedder } from "./lib/embedder.js";
+import type { BuildOptions, Chunker, Logger, LmgrepConfig, RepairResult, SearchOptions, SearchResult, StatusInfo } from "./lib/types.js";
+export type { BuildOptions, Chunk, Chunker, FileChange, FileEntry, IndexedChunk, Logger, LmgrepConfig, ProgressEvent, RepairResult, SearchOptions, SearchResult, StatusInfo, } from "./lib/types.js";
+export { consoleLogger, silentLogger } from "./lib/types.js";
+export type { Embedder } from "./lib/embedder.js";
+export { AISDKEmbedder, EmbeddingAbortError } from "./lib/embedder.js";
+export { TreeSitterChunker } from "./lib/chunker/index.js";
+export { Store, findIndexedAncestor, getDbPath, getLegacyDbPath, resolveProject, discoverIndexedProjects, writeProjectMetadata, readProjectMetadata, extractModelFamily, acquireDbLock, releaseDbLock, isDbLocked, discoverRunningProcesses, } from "./lib/store.js";
+export type { ProjectMetadata, RunningProcess } from "./lib/store.js";
+export { startWatcher } from "./lib/serve.js";
+export { loadConfig, getConfigDir, getGlobalConfigPath } from "./lib/config.js";
+export interface CreateIndexOptions {
+    cwd: string;
+    config?: Partial<LmgrepConfig>;
+    embedder?: Embedder;
+    chunker?: Chunker;
+    logger?: Logger;
+}
+export interface LmgrepIndex {
+    readonly cwd: string;
+    readonly config: LmgrepConfig;
+    build(opts?: BuildOptions): Promise<{
+        succeeded: number;
+        failed: number;
+    }>;
+    search(query: string, opts?: SearchOptions): Promise<SearchResult[]>;
+    repair(dry?: boolean): Promise<RepairResult>;
+    watch(): Promise<void>;
+    status(): Promise<StatusInfo>;
+    close(): Promise<void>;
+}
+export declare function createIndex(options: CreateIndexOptions): Promise<LmgrepIndex>;

package/dist/index.js

@@ -0,0 +1,180 @@
+import { loadConfig } from "./lib/config.js";
+import { TreeSitterChunker } from "./lib/chunker/index.js";
+import { AISDKEmbedder } from "./lib/embedder.js";
+import { build } from "./lib/build.js";
+import { repair } from "./lib/repair.js";
+import { serve } from "./lib/serve.js";
+import { resolve } from "node:path";
+import { Store, findIndexedAncestor, resolveProject, readProjectMetadata, getDbPath, extractModelFamily, } from "./lib/store.js";
+import { consoleLogger } from "./lib/types.js";
+export { consoleLogger, silentLogger } from "./lib/types.js";
+export { AISDKEmbedder, EmbeddingAbortError } from "./lib/embedder.js";
+export { TreeSitterChunker } from "./lib/chunker/index.js";
+export { Store, findIndexedAncestor, getDbPath, getLegacyDbPath, resolveProject, discoverIndexedProjects, writeProjectMetadata, readProjectMetadata, extractModelFamily, acquireDbLock, releaseDbLock, isDbLocked, discoverRunningProcesses, } from "./lib/store.js";
+export { startWatcher } from "./lib/serve.js";
+export { loadConfig, getConfigDir, getGlobalConfigPath } from "./lib/config.js";
+export async function createIndex(options) {
+    const { cwd } = options;
+    // Resolve config: user overrides > file config > defaults
+    const fileConfig = loadConfig(cwd);
+    const config = { ...fileConfig, ...options.config };
+    const store = Store.forProject(cwd);
+    const embedder = options.embedder ?? new AISDKEmbedder(config);
+    const chunker = options.chunker ?? new TreeSitterChunker();
+    const logger = options.logger ?? consoleLogger;
+    return {
+        cwd,
+        config,
+        async build(opts) {
+            return build(cwd, store, config, embedder, chunker, opts, logger);
+        },
+        async search(query, opts = {}) {
+            let queryVector = await embedder.embedQuery(query);
+            // Check model compatibility with the index
+            const dbPath = getDbPath(cwd);
+            const meta = readProjectMetadata(dbPath);
+            if (meta) {
+                // Hard error: dimension mismatch
+                if (meta.dimensions != null && queryVector.length !== meta.dimensions) {
+                    throw new Error(`Dimension mismatch: index has ${meta.dimensions}-dim vectors but your model produces ${queryVector.length}-dim. ` +
+                        `These embeddings are incompatible.`);
+                }
+                // Advisory: model family mismatch
+                if (meta.model) {
+                    const indexFamily = extractModelFamily(meta.model);
+                    const queryFamily = extractModelFamily(config.model);
+                    if (indexFamily !== queryFamily) {
+                        logger.info(`Warning: index was built with "${meta.model}" (${indexFamily}) ` +
+                            `but searching with "${config.model}" (${queryFamily}). ` +
+                            `Results may be degraded if these are different model families.`);
+                    }
+                }
+            }
+            // Subtract the --not vector to push away unwanted results
+            if (opts.not) {
+                const notVector = await embedder.embedQuery(opts.not);
+                queryVector = queryVector.map((v, i) => v - notVector[i] * 0.5);
+            }
+            // Determine which stores to search
+            const targets = resolveSearchTargets(cwd, store, opts);
+            const limit = opts.limit ?? 25;
+            // Search all targets
+            let results = [];
+            for (const target of targets) {
+                const targetResults = await target.store.search(queryVector, limit, target.filePrefix, opts.type);
+                // Tag results from foreign projects with their root
+                if (target.projectRoot) {
+                    for (const r of targetResults) {
+                        r.filePath = `${target.projectRoot}/${r.filePath}`;
+                    }
+                }
+                results.push(...targetResults);
+            }
+            // When searching multiple targets, sort by score and trim
+            if (targets.length > 1) {
+                results.sort((a, b) => b.score - a.score);
+                results = results.slice(0, limit);
+            }
+            // Post-filter by language (file extension)
+            if (opts.language && opts.language.length > 0) {
+                const exts = new Set(opts.language.map((l) => (l.startsWith(".") ? l : `.${l}`)));
+                results = results.filter((r) => exts.has(r.filePath.slice(r.filePath.lastIndexOf("."))));
+            }
+            if (opts.minScore != null) {
+                results = results.filter((r) => r.score >= opts.minScore);
+            }
+            // Close any foreign stores we opened
+            for (const target of targets) {
+                if (target.store !== store) {
+                    await target.store.close();
+                }
+            }
+            return results;
+        },
+        async repair(dry = false) {
+            return repair(cwd, store, chunker, dry, logger);
+        },
+        async watch() {
+            return serve(cwd, store, config, embedder, chunker, logger);
+        },
+        async status() {
+            const ancestor = findIndexedAncestor(cwd);
+            const projectRoot = ancestor ? ancestor.root : cwd;
+            const prefix = ancestor?.prefix ?? "";
+            const statusStore = projectRoot === cwd ? store : Store.forProject(projectRoot);
+            const files = await statusStore.getIndexedFiles();
+            const hashes = await statusStore.getIndexedHashes();
+            let totalChunks = 0;
+            for (const [, h] of files)
+                totalChunks += h.length;
+            let embeddingOk = false;
+            let embeddingLatencyMs;
+            try {
+                const start = Date.now();
+                await embedder.embedQuery("test");
+                embeddingLatencyMs = Date.now() - start;
+                embeddingOk = true;
+            }
+            catch { }
+            // Read index metadata for model/dimensions info
+            const meta = readProjectMetadata(getDbPath(projectRoot));
+            return {
+                projectRoot,
+                prefix,
+                config,
+                fileCount: files.size,
+                chunkCount: totalChunks,
+                uniqueHashes: hashes.size,
+                embeddingOk,
+                embeddingLatencyMs,
+                indexModel: meta?.model,
+                indexDimensions: meta?.dimensions,
+            };
+        },
+        async close() {
+            await store.close();
+        },
+    };
+}
+function resolveSearchTargets(cwd, localStore, opts) {
+    // --across: search multiple projects
+    if (opts.across && opts.across.length > 0) {
+        const targets = [
+            { store: localStore, filePrefix: opts.filePrefix },
+        ];
+        for (const p of opts.across) {
+            const abs = resolve(cwd, p);
+            const { root } = resolveProject(abs);
+            targets.push({
+                store: Store.forProject(abs),
+                filePrefix: opts.filePrefix,
+                projectRoot: root,
+            });
+        }
+        return targets;
+    }
+    // --project: search a single foreign project
+    if (opts.project) {
+        const abs = resolve(cwd, opts.project);
+        const { root } = resolveProject(abs);
+        return [
+            {
+                store: Store.forProject(abs),
+                filePrefix: opts.filePrefix,
+                projectRoot: root,
+            },
+        ];
+    }
+    // Default: search local project, resolving ancestor prefix
+    let filePrefix = opts.filePrefix;
+    let searchStore = localStore;
+    const ancestor = findIndexedAncestor(cwd);
+    if (ancestor?.prefix) {
+        searchStore = Store.forProject(ancestor.root);
+        filePrefix = filePrefix
+            ? `${ancestor.prefix}/${filePrefix}`
+            : ancestor.prefix;
+    }
+    return [{ store: searchStore, filePrefix }];
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAiB,MAAM,mBAAmB,CAAC;AACjE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EACN,KAAK,EACL,mBAAmB,EACnB,cAAc,EACd,mBAAmB,EACnB,SAAS,EACT,kBAAkB,GAClB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAkB/C,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG7D,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EACN,KAAK,EACL,mBAAmB,EACnB,SAAS,EACT,eAAe,EACf,cAAc,EACd,uBAAuB,EACvB,oBAAoB,EACpB,mBAAmB,EACnB,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,UAAU,EACV,wBAAwB,GACxB,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAsBhF,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,OAA2B;IAE3B,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAExB,0DAA0D;IAC1D,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IACnC,MAAM,MAAM,GAAiB,EAAE,GAAG,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAElE,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,iBAAiB,EAAE,CAAC;IAC3D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;IAE/C,OAAO;QACN,GAAG;QACH,MAAM;QAEN,KAAK,CAAC,KAAK,CAAC,IAAmB;YAC9B,OAAO,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QACnE,CAAC;QAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAsB,EAAE;YACnD,IAAI,WAAW,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAEnD,2CAA2C;YAC3C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,IAAI,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,IAAI,EAAE,CAAC;gBACV,iCAAiC;gBACjC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,IAAI,WAAW,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;oBACvE,MAAM,IAAI,KAAK,CACd,iCAAiC,IAAI,CAAC,UAAU,wCAAwC,WAAW,CAAC,MAAM,QAAQ;wBAClH,oCAAoC,CACpC,CAAC;gBACH,CAAC;gBAED,kCAAkC;gBAClC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;oBAChB,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnD,MAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACrD,IAAI,WAAW,KAAK,WAAW,EAAE,CAAC;wBACjC,MAAM,CAAC,IAAI,CACV,kCAAkC,IAAI,CAAC,KAAK,MAAM,WAAW,IAAI;4BACjE,uBAAuB,MAAM,CAAC,KAAK,MAAM,WAAW,KAAK;4BACzD,gEAAgE,CAChE,CAAC;oBACH,CAAC;gBACF,CAAC;YACF,CAAC;YAED,0DAA0D;YAC1D,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBACd,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACtD,WAAW,GAAG,WAAW,CAAC,GAAG,CAC5B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,CAChC,CAAC;YACH,CAAC;YAED,mCAAmC;YACnC,MAAM,OAAO,GAAG,oBAAoB,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YAE/B,qBAAqB;YACrB,IAAI,OAAO,GAAmB,EAAE,CAAC;YACjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM,CAC9C,WAAW,EACX,KAAK,EACL,MAAM,CAAC,UAAU,EACjB,IAAI,CAAC,IAAI,CACT,CAAC;gBAEF,oDAAoD;gBACpD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;oBACxB,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;wBAC/B,CAAC,CAAC,QAAQ,GAAG,GAAG,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;oBACpD,CAAC;gBACF,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;YAChC,CAAC;YAED,0DAA0D;YAC1D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;gBAC1C,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACnC,CAAC;YAED,2CAA2C;YAC3C,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/C,MAAM,IAAI,GAAG,IAAI,GAAG,CACnB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAC3D,CAAC;gBACF,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CACvD,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;gBAC3B,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,QAAS,CAAC,CAAC;YAC5D,CAAC;YAED,qCAAqC;YACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;oBAC5B,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC5B,CAAC;YACF,CAAC;YAED,OAAO,OAAO,CAAC;QAChB,CAAC;QAED,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,KAAK;YACvB,OAAO,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC;QAED,KAAK,CAAC,KAAK;YACV,OAAO,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7D,CAAC;QAED,KAAK,CAAC,MAAM;YACX,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAC1C,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;YACnD,MAAM,MAAM,GAAG,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAC;YACtC,MAAM,WAAW,GAChB,WAAW,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;YAE7D,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,eAAe,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,gBAAgB,EAAE,CAAC;YAEpD,IAAI,WAAW,GAAG,CAAC,CAAC;YACpB,KAAK,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK;gBAAE,WAAW,IAAI,CAAC,CAAC,MAAM,CAAC;YAEnD,IAAI,WAAW,GAAG,KAAK,CAAC;YACxB,IAAI,kBAAsC,CAAC;YAC3C,IAAI,CAAC;gBACJ,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACzB,MAAM,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;gBAClC,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBACxC,WAAW,GAAG,IAAI,CAAC;YACpB,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;YAEV,gDAAgD;YAChD,MAAM,IAAI,GAAG,mBAAmB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC;YAEzD,OAAO;gBACN,WAAW;gBACX,MAAM;gBACN,MAAM;gBACN,SAAS,EAAE,KAAK,CAAC,IAAI;gBACrB,UAAU,EAAE,WAAW;gBACvB,YAAY,EAAE,MAAM,CAAC,IAAI;gBACzB,WAAW;gBACX,kBAAkB;gBAClB,UAAU,EAAE,IAAI,EAAE,KAAK;gBACvB,eAAe,EAAE,IAAI,EAAE,UAAU;aACjC,CAAC;QACH,CAAC;QAED,KAAK,CAAC,KAAK;YACV,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;KACD,CAAC;AACH,CAAC;AAWD,SAAS,oBAAoB,CAC5B,GAAW,EACX,UAAiB,EACjB,IAAmB;IAEnB,qCAAqC;IACrC,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAmB;YAC/B,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE;SAClD,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;YACrC,OAAO,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI;aACjB,CAAC,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QACrC,OAAO;YACN;gBACC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI;aACjB;SACD,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,IAAI,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;IACjC,IAAI,WAAW,GAAG,UAAU,CAAC;IAC7B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAC1C,IAAI,QAAQ,EAAE,MAAM,EAAE,CAAC;QACtB,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC9C,UAAU,GAAG,UAAU;YACtB,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,UAAU,EAAE;YACpC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;IACpB,CAAC;IAED,OAAO,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;AAC7C,CAAC"}

package/dist/lib/build.d.ts

@@ -0,0 +1,7 @@
+import { type Embedder } from "./embedder.js";
+import { type Store } from "./store.js";
+import type { Chunker, Logger, LmgrepConfig, BuildOptions } from "./types.js";
+export declare function build(cwd: string, store: Store, config: LmgrepConfig, embedder: Embedder, chunker: Chunker, opts?: BuildOptions, logger?: Logger): Promise<{
+    succeeded: number;
+    failed: number;
+}>;

package/dist/lib/build.js

@@ -0,0 +1,222 @@
+import { ResilientEmbedder, EmbeddingAbortError, } from "./embedder.js";
+import { walkFiles, detectChanges, filterByMtime } from "./scanner.js";
+import { writeProjectMetadata } from "./store.js";
+import { consoleLogger } from "./types.js";
+function parseDuration(s) {
+    const match = s.match(/^(\d+)\s*(s|m|h|d)$/);
+    if (!match)
+        throw new Error(`Invalid duration "${s}". Use e.g. 10m, 2h, 1d`);
+    const n = Number.parseInt(match[1], 10);
+    const unit = match[2];
+    const multipliers = {
+        s: 1000,
+        m: 60_000,
+        h: 3_600_000,
+        d: 86_400_000,
+    };
+    return n * multipliers[unit];
+}
+export async function build(cwd, store, config, embedder, chunker, opts = {}, logger = consoleLogger) {
+    const log = logger.info.bind(logger);
+    const emit = opts.onProgress;
+    if (opts.reset) {
+        log("Resetting index...");
+        await store.reset();
+    }
+    // 1. Scan
+    let files;
+    if (opts.files && opts.files.length > 0) {
+        files = opts.files;
+        log(`Processing ${files.length} targeted files`);
+    }
+    else {
+        files = walkFiles(cwd, config.ignore, config.extensions);
+        if (opts.since) {
+            const cutoff = Date.now() - parseDuration(opts.since);
+            const before = files.length;
+            files = filterByMtime(files, cwd, cutoff);
+            log(`Found ${files.length} files modified in the last ${opts.since} (out of ${before})`);
+        }
+        else {
+            log(`Found ${files.length} files`);
+        }
+    }
+    emit?.({ phase: "scan", current: files.length, total: files.length });
+    // 2. Change detection
+    const storedHashes = await store.getFileHashes();
+    const { changed, currentHashes } = detectChanges(files, storedHashes, cwd, opts.force);
+    log(`${changed.length} files changed out of ${files.length}`);
+    if (changed.length === 0) {
+        log("No changes detected. Index is up to date.");
+        return { succeeded: 0, failed: 0 };
+    }
+    // 2b. Skip files whose content hash is already known (indexed on another branch)
+    const knownHashes = await store.filterKnownFileHashes(changed.map((f) => f.hash));
+    const alreadyKnown = [];
+    const trulyChanged = [];
+    for (const f of changed) {
+        if (knownHashes.has(f.hash)) {
+            alreadyKnown.push(f);
+        }
+        else {
+            trulyChanged.push(f);
+        }
+    }
+    if (alreadyKnown.length > 0) {
+        // Register these files on the current branch without re-chunking
+        const hashEntries = alreadyKnown.map((f) => ({
+            filePath: f.path,
+            fileHash: f.hash,
+        }));
+        await store.upsertFileHashes(hashEntries);
+        log(`${alreadyKnown.length} files already indexed (content known from other branches)`);
+    }
+    if (trulyChanged.length === 0) {
+        log("No new content to index.");
+        return { succeeded: 0, failed: 0 };
+    }
+    // 3. Chunk changed files
+    const changedPaths = trulyChanged.map((f) => f.path);
+    const allChunks = [];
+    for (let i = 0; i < changedPaths.length; i++) {
+        try {
+            const chunks = await chunker.chunk(changedPaths[i], cwd);
+            allChunks.push(...chunks);
+        }
+        catch {
+            // skip files that fail to parse
+        }
+        if ((i + 1) % 1000 === 0 || i === changedPaths.length - 1) {
+            emit?.({
+                phase: "chunk",
+                current: i + 1,
+                total: changedPaths.length,
+                message: `${allChunks.length} chunks`,
+            });
+            log(`Chunking: ${i + 1}/${changedPaths.length} files, ${allChunks.length} chunks so far`);
+        }
+    }
+    if (allChunks.length === 0) {
+        log("No chunks produced. Index is up to date.");
+        return { succeeded: 0, failed: 0 };
+    }
+    if (opts.dry) {
+        for (const f of changedPaths)
+            log(`  ${f}`);
+        return { succeeded: 0, failed: 0 };
+    }
+    // 4. Delete old chunks for changed files
+    await store.deleteChunksByFiles(changedPaths);
+    // 5. Filter out already-indexed chunks (by hash, queried in DB)
+    const existingHashes = await store.filterExistingChunkHashes(allChunks.map((c) => c.hash));
+    let newChunks = allChunks.filter((c) => !existingHashes.has(c.hash));
+    const alreadyIndexed = allChunks.length - newChunks.length;
+    // Filter oversized chunks
+    let skippedOversize = 0;
+    if (config.maxTokens) {
+        const before = newChunks.length;
+        newChunks = newChunks.filter((c) => {
+            const est = Math.ceil((c.context.length + c.content.length) / 4);
+            return est <= config.maxTokens;
+        });
+        skippedOversize = before - newChunks.length;
+    }
+    log(`${changedPaths.length} files changed, ${allChunks.length} chunks total, ${newChunks.length} to embed` +
+        ` (${alreadyIndexed} already indexed${skippedOversize > 0 ? `, ${skippedOversize} oversized` : ""})`);
+    if (newChunks.length === 0) {
+        const hashEntries = changedPaths
+            .filter((fp) => currentHashes.has(fp))
+            .map((fp) => ({
+            filePath: fp,
+            fileHash: currentHashes.get(fp),
+        }));
+        if (hashEntries.length > 0)
+            await store.upsertFileHashes(hashEntries);
+        log("All chunks already indexed.");
+        return { succeeded: 0, failed: 0 };
+    }
+    // 6. Embed and store
+    const resilient = new ResilientEmbedder(embedder, config, {
+        onBatchStart(batchNum, total) {
+            emit?.({ phase: "embed", current: batchNum, total });
+        },
+        onBatchDone(batchNum, succeeded, failed) {
+            log(`Batch ${batchNum}: ${succeeded} ok / ${failed} err / ${newChunks.length} total`);
+        },
+        onReload(attempt, max) {
+            log(`Consecutive failures — reloading model (attempt ${attempt}/${max})...`);
+        },
+    });
+    const texts = newChunks.map((c) => `${c.context}\n${c.content}`);
+    let vectors;
+    let failedIndices;
+    let aborted = false;
+    try {
+        const result = await resilient.embedBatched(texts);
+        vectors = result.vectors;
+        failedIndices = result.failedIndices;
+    }
+    catch (err) {
+        if (err instanceof EmbeddingAbortError) {
+            aborted = true;
+            vectors = err.vectors;
+            failedIndices = err.failedIndices;
+            logger.error(err.message);
+        }
+        else {
+            throw err;
+        }
+    }
+    // Store successful chunks
+    const indexed = [];
+    const failedFiles = new Set();
+    let embeddingDimensions;
+    for (let i = 0; i < newChunks.length; i++) {
+        if (vectors[i] !== null) {
+            indexed.push({ ...newChunks[i], vector: vectors[i] });
+            if (embeddingDimensions === undefined) {
+                embeddingDimensions = vectors[i].length;
+            }
+        }
+        else {
+            failedFiles.add(newChunks[i].filePath);
+        }
+    }
+    if (indexed.length > 0) {
+        const STORE_BATCH = 500;
+        for (let i = 0; i < indexed.length; i += STORE_BATCH) {
+            await store.addChunks(indexed.slice(i, i + STORE_BATCH));
+            emit?.({
+                phase: "store",
+                current: Math.min(i + STORE_BATCH, indexed.length),
+                total: indexed.length,
+            });
+        }
+    }
+    // Save file hashes for files with no failures
+    const hashEntries = changedPaths
+        .filter((fp) => currentHashes.has(fp) && !failedFiles.has(fp))
+        .map((fp) => ({ filePath: fp, fileHash: currentHashes.get(fp) }));
+    if (hashEntries.length > 0)
+        await store.upsertFileHashes(hashEntries);
+    const succeeded = indexed.length;
+    const failed = failedIndices.size;
+    if (aborted) {
+        if (succeeded > 0) {
+            log(`Saved ${succeeded} chunks before aborting.`);
+        }
+        log(`Embedding failed. ${failed} chunks could not be embedded. ` +
+            `Fix your embedding provider and run \`lmgrep index\` to resume.`);
+    }
+    else {
+        log(`Done: ${succeeded} chunks indexed from ${changedPaths.length} files` +
+            (failed > 0 ? ` (${failed} failed)` : ""));
+    }
+    // Update project metadata (preserve original model/dimensions as baseline)
+    writeProjectMetadata(cwd, {
+        model: config.model,
+        dimensions: embeddingDimensions,
+    });
+    return { succeeded, failed };
+}
+//# sourceMappingURL=build.js.map

package/dist/lib/build.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/lib/build.ts"],"names":[],"mappings":"AAAA,OAAO,EAEN,iBAAiB,EACjB,mBAAmB,GACnB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAc,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAS9D,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,SAAS,aAAa,CAAC,CAAS;IAC/B,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IAC7C,IAAI,CAAC,KAAK;QACT,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,yBAAyB,CAAC,CAAC;IAClE,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,WAAW,GAA2B;QAC3C,CAAC,EAAE,IAAI;QACP,CAAC,EAAE,MAAM;QACT,CAAC,EAAE,SAAS;QACZ,CAAC,EAAE,UAAU;KACb,CAAC;IACF,OAAO,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAC1B,GAAW,EACX,KAAY,EACZ,MAAoB,EACpB,QAAkB,EAClB,OAAgB,EAChB,OAAqB,EAAE,EACvB,SAAiB,aAAa;IAE9B,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAE7B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAChB,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAC1B,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,UAAU;IACV,IAAI,KAAe,CAAC;IACpB,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACnB,GAAG,CAAC,cAAc,KAAK,CAAC,MAAM,iBAAiB,CAAC,CAAC;IAClD,CAAC;SAAM,CAAC;QACP,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QAEzD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YAC5B,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;YAC1C,GAAG,CACF,SAAS,KAAK,CAAC,MAAM,+BAA+B,IAAI,CAAC,KAAK,YAAY,MAAM,GAAG,CACnF,CAAC;QACH,CAAC;aAAM,CAAC;YACP,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC;QACpC,CAAC;IACF,CAAC;IAED,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAEtE,sBAAsB;IACtB,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;IACjD,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,aAAa,CAC/C,KAAK,EACL,YAAY,EACZ,GAAG,EACH,IAAI,CAAC,KAAK,CACV,CAAC;IAEF,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,yBAAyB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAE9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,GAAG,CAAC,2CAA2C,CAAC,CAAC;QACjD,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,iFAAiF;IACjF,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,qBAAqB,CACpD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC1B,CAAC;IACF,MAAM,YAAY,GAAmB,EAAE,CAAC;IACxC,MAAM,YAAY,GAAmB,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACP,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACF,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,iEAAiE;QACjE,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5C,QAAQ,EAAE,CAAC,CAAC,IAAI;YAChB,QAAQ,EAAE,CAAC,CAAC,IAAI;SAChB,CAAC,CAAC,CAAC;QACJ,MAAM,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAC1C,GAAG,CACF,GAAG,YAAY,CAAC,MAAM,4DAA4D,CAClF,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,GAAG,CAAC,0BAA0B,CAAC,CAAC;QAChC,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,yBAAyB;IACzB,MAAM,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrD,MAAM,SAAS,GAAY,EAAE,CAAC;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACzD,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACR,gCAAgC;QACjC,CAAC;QAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3D,IAAI,EAAE,CAAC;gBACN,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,CAAC,GAAG,CAAC;gBACd,KAAK,EAAE,YAAY,CAAC,MAAM;gBAC1B,OAAO,EAAE,GAAG,SAAS,CAAC,MAAM,SAAS;aACrC,CAAC,CAAC;YACH,GAAG,CACF,aAAa,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,gBAAgB,CACpF,CAAC;QACH,CAAC;IACF,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,GAAG,CAAC,0CAA0C,CAAC,CAAC;QAChD,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACd,KAAK,MAAM,CAAC,IAAI,YAAY;YAAE,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5C,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,yCAAyC;IACzC,MAAM,KAAK,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;IAE9C,gEAAgE;IAChE,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,yBAAyB,CAC3D,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC5B,CAAC;IACF,IAAI,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;IAE3D,0BAA0B;IAC1B,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;QAChC,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACjE,OAAO,GAAG,IAAI,MAAM,CAAC,SAAU,CAAC;QACjC,CAAC,CAAC,CAAC;QACH,eAAe,GAAG,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;IAC7C,CAAC;IAED,GAAG,CACF,GAAG,YAAY,CAAC,MAAM,mBAAmB,SAAS,CAAC,MAAM,kBAAkB,SAAS,CAAC,MAAM,WAAW;QACrG,KAAK,cAAc,mBAAmB,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,eAAe,YAAY,CAAC,CAAC,CAAC,EAAE,GAAG,CACrG,CAAC;IAEF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,YAAY;aAC9B,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;aACrC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACb,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,CAAE;SAChC,CAAC,CAAC,CAAC;QACL,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;QACtE,GAAG,CAAC,6BAA6B,CAAC,CAAC;QACnC,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,qBAAqB;IACrB,MAAM,SAAS,GAAG,IAAI,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE;QACzD,YAAY,CAAC,QAAQ,EAAE,KAAK;YAC3B,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM;YACtC,GAAG,CACF,SAAS,QAAQ,KAAK,SAAS,SAAS,MAAM,UAAU,SAAS,CAAC,MAAM,QAAQ,CAChF,CAAC;QACH,CAAC;QACD,QAAQ,CAAC,OAAO,EAAE,GAAG;YACpB,GAAG,CACF,mDAAmD,OAAO,IAAI,GAAG,MAAM,CACvE,CAAC;QACH,CAAC;KACD,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IAEjE,IAAI,OAA4B,CAAC;IACjC,IAAI,aAA0B,CAAC;IAC/B,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QACnD,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QACzB,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IACtC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,IAAI,GAAG,YAAY,mBAAmB,EAAE,CAAC;YACxC,OAAO,GAAG,IAAI,CAAC;YACf,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;YACtB,aAAa,GAAG,GAAG,CAAC,aAAa,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC;aAAM,CAAC;YACP,MAAM,GAAG,CAAC;QACX,CAAC;IACF,CAAC;IAED,0BAA0B;IAC1B,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,IAAI,mBAAuC,CAAC;IAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;YACvD,IAAI,mBAAmB,KAAK,SAAS,EAAE,CAAC;gBACvC,mBAAmB,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;YAC1C,CAAC;QACF,CAAC;aAAM,CAAC;YACP,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACxC,CAAC;IACF,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,WAAW,GAAG,GAAG,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;YACtD,MAAM,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC;YACzD,IAAI,EAAE,CAAC;gBACN,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC;gBAClD,KAAK,EAAE,OAAO,CAAC,MAAM;aACrB,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,8CAA8C;IAC9C,MAAM,WAAW,GAAG,YAAY;SAC9B,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;SAC7D,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC,CAAC;IACpE,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAEtE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC;IAElC,IAAI,OAAO,EAAE,CAAC;QACb,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,GAAG,CAAC,SAAS,SAAS,0BAA0B,CAAC,CAAC;QACnD,CAAC;QACD,GAAG,CACF,qBAAqB,MAAM,iCAAiC;YAC3D,iEAAiE,CAClE,CAAC;IACH,CAAC;SAAM,CAAC;QACP,GAAG,CACF,SAAS,SAAS,wBAAwB,YAAY,CAAC,MAAM,QAAQ;YACpE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAC1C,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,oBAAoB,CAAC,GAAG,EAAE;QACzB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,mBAAmB;KAC/B,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC"}

package/dist/{chunker → lib/chunker}/context.d.ts

@@ -13,10 +13,6 @@ export interface ChunkContext {
 }
 /** Walk up from a node to collect typed parent scopes */
 export declare function extractScope(node: Parser.SyntaxNode, langConfig: LanguageConfig): ScopeEntry[];
-/** Extract import module names from the root of the tree */
-export declare function extractImports(tree: Parser.Tree, langConfig: LanguageConfig): string[];
-/** Extract sibling signatures from the same parent scope */
-export declare function extractSiblingSignatures(node: Parser.SyntaxNode, langConfig: LanguageConfig): string[];
 /** Extract leading comments and decorators immediately before a node */
 export declare function extractLeadingComment(node: Parser.SyntaxNode, source: string): string | null;
 /** Classify a chunk's structural role based on its AST node type */

package/dist/{chunker → lib/chunker}/context.js

@@ -1,4 +1,3 @@
-/** Map AST node types to human-readable scope kinds */
 const SCOPE_KIND_MAP = {
     // TypeScript / JavaScript
     class_declaration: "class",
@@ -29,7 +28,6 @@ const SCOPE_KIND_MAP = {
     // Generic
     ContainerDecl: "container",
 };
-/** AST node types that represent type/structure definitions */
 const DEFINITION_TYPES = new Set([
     "class_declaration",
     "class_definition",
@@ -47,11 +45,9 @@ const DEFINITION_TYPES = new Set([
     "type_definition",
     "struct_declaration",
     "protocol_declaration",
-    "class_definition",
     "trait_definition",
     "ContainerDecl",
 ]);
-/** AST node types that typically orchestrate / glue logic */
 const ORCHESTRATION_TYPES = new Set([
     "export_statement",
     "decorated_definition",
@@ -72,80 +68,11 @@ export function extractScope(node, langConfig) {
     }
     return scopes;
 }
-/** Extract import module names from the root of the tree */
-export function extractImports(tree, langConfig) {
-    const modules = [];
-    for (const child of tree.rootNode.children) {
-        if (langConfig.importTypes.includes(child.type)) {
-            const mod = extractModuleName(child);
-            if (mod && !modules.includes(mod)) {
-                modules.push(mod);
-            }
-        }
-    }
-    return modules;
-}
-/** Pull out just the module/package name from an import node */
-function extractModuleName(node) {
-    // Look for string children (the module specifier)
-    const stringNode = findDescendant(node, (n) => n.type === "string" ||
-        n.type === "string_literal" ||
-        n.type === "interpreted_string_literal");
-    if (stringNode) {
-        // Strip quotes
-        return stringNode.text.replace(/^["']|["']$/g, "");
-    }
-    // Python: import foo.bar / from foo.bar import baz
-    const dottedName = findDescendant(node, (n) => n.type === "dotted_name" || n.type === "module_name");
-    if (dottedName)
-        return dottedName.text;
-    // Rust: use crate::foo::bar
-    const path = findDescendant(node, (n) => n.type === "scoped_identifier" || n.type === "use_list");
-    if (path)
-        return path.text;
-    // C/C++: #include <foo.h> or "foo.h"
-    const sysLib = findDescendant(node, (n) => n.type === "system_lib_string" || n.type === "string_literal");
-    if (sysLib)
-        return sysLib.text.replace(/^[<"]|[>"]$/g, "");
-    return null;
-}
-function findDescendant(node, predicate) {
-    for (const child of node.children) {
-        if (predicate(child))
-            return child;
-        const found = findDescendant(child, predicate);
-        if (found)
-            return found;
-    }
-    return null;
-}
-/** Extract sibling signatures from the same parent scope */
-export function extractSiblingSignatures(node, langConfig) {
-    const parent = node.parent;
-    if (!parent)
-        return [];
-    const siblings = [];
-    for (const child of parent.children) {
-        if (child.id === node.id)
-            continue;
-        if (!langConfig.chunkTypes.includes(child.type))
-            continue;
-        const name = extractNodeName(child);
-        if (name) {
-            const firstLine = child.text.split("\n")[0].trim();
-            if (firstLine.length < 200) {
-                siblings.push(firstLine);
-            }
-        }
-    }
-    return siblings.slice(0, 10);
-}
 /** Extract leading comments and decorators immediately before a node */
 export function extractLeadingComment(node, source) {
     const lines = source.split("\n");
     const nodeStartLine = node.startPosition.row;
     const collected = [];
-    // Walk backwards from the line before the node
     for (let i = nodeStartLine - 1; i >= 0 && i >= nodeStartLine - 10; i--) {
         const line = lines[i].trim();
         if (line.startsWith("//") ||
@@ -161,7 +88,6 @@ export function extractLeadingComment(node, source) {
             collected.unshift(lines[i]);
         }
         else if (line === "") {
-            // Allow one blank line gap
             if (collected.length > 0)
                 break;
         }
@@ -187,7 +113,9 @@ export function buildContextString(ctx) {
     lines.push(`[file: ${ctx.filePath}]`);
     lines.push(`[role: ${ctx.role}]`);
     if (ctx.scope.length > 0) {
-        const scopeStr = ctx.scope.map((s) => `${s.kind} ${s.name}`).join(" > ");
+        const scopeStr = ctx.scope
+            .map((s) => `${s.kind} ${s.name}`)
+            .join(" > ");
         lines.push(`[scope: ${scopeStr}]`);
     }
     if (ctx.leadingComment) {
@@ -195,7 +123,6 @@ export function buildContextString(ctx) {
     }
     return lines.join("\n");
 }
-/** Try to extract a name identifier from an AST node */
 function extractNodeName(node) {
     const nameNode = node.childForFieldName("name") ??
         node.children.find((c) => c.type === "identifier" || c.type === "type_identifier");

package/dist/lib/chunker/context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.js","sourceRoot":"","sources":["../../../src/lib/chunker/context.ts"],"names":[],"mappings":"AAiBA,MAAM,cAAc,GAA2B;IAC9C,0BAA0B;IAC1B,iBAAiB,EAAE,OAAO;IAC1B,UAAU,EAAE,OAAO;IACnB,qBAAqB,EAAE,WAAW;IAClC,MAAM,EAAE,QAAQ;IAChB,qBAAqB,EAAE,WAAW;IAClC,SAAS;IACT,gBAAgB,EAAE,OAAO;IACzB,OAAO;IACP,SAAS,EAAE,MAAM;IACjB,UAAU,EAAE,OAAO;IACnB,QAAQ,EAAE,KAAK;IACf,KAAK;IACL,gBAAgB,EAAE,MAAM;IACxB,OAAO;IACP,KAAK,EAAE,OAAO;IACd,UAAU;IACV,gBAAgB,EAAE,QAAQ;IAC1B,eAAe,EAAE,OAAO;IACxB,oBAAoB,EAAE,WAAW;IACjC,QAAQ;IACR,kBAAkB,EAAE,QAAQ;IAC5B,qBAAqB,EAAE,WAAW;IAClC,QAAQ;IACR,iBAAiB,EAAE,QAAQ;IAC3B,gBAAgB,EAAE,OAAO;IACzB,UAAU;IACV,aAAa,EAAE,WAAW;CAC1B,CAAC;AAEF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAChC,mBAAmB;IACnB,kBAAkB;IAClB,uBAAuB;IACvB,wBAAwB;IACxB,kBAAkB;IAClB,aAAa;IACb,WAAW;IACX,YAAY;IACZ,WAAW;IACX,kBAAkB;IAClB,kBAAkB;IAClB,iBAAiB;IACjB,gBAAgB;IAChB,iBAAiB;IACjB,oBAAoB;IACpB,sBAAsB;IACtB,kBAAkB;IAClB,eAAe;CACf,CAAC,CAAC;AAEH,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IACnC,kBAAkB;IAClB,sBAAsB;CACtB,CAAC,CAAC;AAEH,yDAAyD;AACzD,MAAM,UAAU,YAAY,CAC3B,IAAuB,EACvB,UAA0B;IAE1B,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;IAC1B,OAAO,OAAO,EAAE,CAAC;QAChB,IAAI,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;gBAC1D,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAChC,CAAC;QACF,CAAC;QACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAC1B,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,qBAAqB,CACpC,IAAuB,EACvB,MAAc;IAEd,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;IAC7C,MAAM,SAAS,GAAa,EAAE,CAAC;IAE/B,KAAK,IAAI,CAAC,GAAG,aAAa,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,aAAa,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QACxE,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7B,IACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACrB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACrB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACrB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACrB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EACrB,CAAC;YACF,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;aAAM,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YACxB,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM;QACjC,CAAC;aAAM,CAAC;YACP,MAAM;QACP,CAAC;IACF,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;AACpC,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,YAAY,CAAC,IAAuB;IACnD,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC;IACzD,IAAI,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,eAAe,CAAC;IAC/D,OAAO,gBAAgB,CAAC;AACzB,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,kBAAkB,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC;IACtC,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC;IAElC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK;aACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;aACjC,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,WAAW,QAAQ,GAAG,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,cAAc,GAAG,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,eAAe,CAAC,IAAuB;IAC/C,MAAM,QAAQ,GACb,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,IAAI,CACjB,CAAC,CAAoB,EAAE,EAAE,CACxB,CAAC,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,CAAC,IAAI,KAAK,iBAAiB,CACxD,CAAC;IACH,OAAO,QAAQ,EAAE,IAAI,CAAC;AACvB,CAAC"}

package/dist/lib/chunker/index.d.ts

@@ -0,0 +1,7 @@
+import type { Chunk, Chunker } from "../types.js";
+/** Chunk a single file into context-enriched chunks */
+export declare function chunkFile(filePath: string, cwd: string): Promise<Chunk[]>;
+/** Default chunker using tree-sitter with sliding-window fallback */
+export declare class TreeSitterChunker implements Chunker {
+    chunk(filePath: string, cwd: string): Promise<Chunk[]>;
+}