lmgrep 0.1.18 → 0.2.0

package/dist/lib/chunker/index.js

@@ -1,209 +0,0 @@
-import { createHash } from "node:crypto";
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
-import { Language, Parser } from "web-tree-sitter";
-import { buildContextString, classifyRole, extractLeadingComment, extractScope, } from "./context.js";
-import { getLanguageForFile, getWasmPath, } from "./languages.js";
-const MAX_CHUNK_TOKENS = 8192;
-let parserInstance;
-const loadedLanguages = new Map();
-async function getParser() {
-    if (!parserInstance) {
-        await Parser.init();
-        parserInstance = new Parser();
-    }
-    return parserInstance;
-}
-async function getLanguage(langConfig) {
-    const cached = loadedLanguages.get(langConfig.id);
-    if (cached)
-        return cached;
-    const wasmPath = getWasmPath(langConfig);
-    if (!wasmPath)
-        return undefined;
-    const lang = await Language.load(wasmPath);
-    loadedLanguages.set(langConfig.id, lang);
-    return lang;
-}
-/** Chunk a single file into context-enriched chunks */
-export async function chunkFile(filePath, cwd) {
-    const langConfig = getLanguageForFile(filePath);
-    if (!langConfig)
-        return fallbackChunk(filePath, cwd);
-    const parser = await getParser();
-    const language = await getLanguage(langConfig);
-    if (!language)
-        return fallbackChunk(filePath, cwd);
-    parser.setLanguage(language);
-    const absolutePath = join(cwd, filePath);
-    const source = readFileSync(absolutePath, "utf-8");
-    const tree = parser.parse(source);
-    if (!tree)
-        return fallbackChunk(filePath, cwd);
-    if (langConfig.id === "markdown") {
-        const md = markdownChunks(tree.rootNode, filePath, source);
-        return md.length > 0 ? md : fallbackChunk(filePath, cwd);
-    }
-    const chunks = [];
-    collectChunks(tree.rootNode, langConfig, filePath, source, chunks);
-    if (chunks.length === 0) {
-        return fallbackChunk(filePath, cwd);
-    }
-    return chunks;
-}
-/**
- * Markdown-specific chunking. The block grammar's `section` nodes nest by
- * heading level, so emitting them whole collapses a doc into one chunk. Instead
- * we split at every heading the grammar reports (so a `#` inside a fenced code
- * block isn't mistaken for one) — each chunk spans a heading and its body up to
- * the next heading. Result: complete, non-overlapping, heading-granular chunks.
- */
-function markdownChunks(root, filePath, source) {
-    const lines = source.split("\n");
-    const headingRows = [];
-    const visit = (node) => {
-        if (node.type === "atx_heading" || node.type === "setext_heading") {
-            headingRows.push(node.startPosition.row);
-        }
-        for (const child of node.children)
-            visit(child);
-    };
-    visit(root);
-    // Split points: start of file plus each heading row, sorted and deduped.
-    const starts = [...new Set([0, ...headingRows])].sort((a, b) => a - b);
-    const chunks = [];
-    for (let i = 0; i < starts.length; i++) {
-        const start = starts[i];
-        const end = i + 1 < starts.length ? starts[i + 1] : lines.length;
-        const content = lines.slice(start, end).join("\n");
-        if (content.trim().length === 0)
-            continue;
-        const headingText = (lines[start] ?? "").replace(/^\s*#+\s*/, "").trim();
-        const name = headingText.slice(0, 80) || `lines_${start + 1}_${end}`;
-        const hash = createHash("sha256")
-            .update(content)
-            .digest("hex")
-            .slice(0, 16);
-        chunks.push({
-            id: `${filePath}:${start}:${hash}`,
-            filePath,
-            startLine: start + 1,
-            endLine: end,
-            type: "section",
-            name,
-            content,
-            context: `[file: ${filePath}]`,
-            hash,
-        });
-    }
-    return chunks;
-}
-function collectChunks(node, langConfig, filePath, source, chunks) {
-    if (langConfig.chunkTypes.includes(node.type)) {
-        const content = node.text;
-        // If too large, recurse into children instead
-        const estimatedTokens = Math.ceil(content.length / 4);
-        if (estimatedTokens > MAX_CHUNK_TOKENS &&
-            hasChunkableDescendants(node, langConfig)) {
-            for (const child of node.children) {
-                collectChunks(child, langConfig, filePath, source, chunks);
-            }
-            return;
-        }
-        const name = extractNodeName(node) ?? `anonymous_${node.startPosition.row}`;
-        // Skip very small chunks
-        if (content.split("\n").length < 2 && content.length < 50) {
-            return;
-        }
-        const scope = extractScope(node, langConfig);
-        const leadingComment = extractLeadingComment(node, source);
-        const role = classifyRole(node);
-        const context = buildContextString({
-            filePath,
-            scope,
-            leadingComment,
-            role,
-        });
-        const hash = createHash("sha256")
-            .update(content)
-            .digest("hex")
-            .slice(0, 16);
-        chunks.push({
-            id: `${filePath}:${node.startPosition.row}:${hash}`,
-            filePath,
-            startLine: node.startPosition.row + 1,
-            endLine: node.endPosition.row + 1,
-            type: node.type,
-            name,
-            content,
-            context,
-            hash,
-        });
-        return;
-    }
-    for (const child of node.children) {
-        collectChunks(child, langConfig, filePath, source, chunks);
-    }
-}
-function hasChunkableDescendants(node, langConfig) {
-    for (const child of node.children) {
-        if (langConfig.chunkTypes.includes(child.type))
-            return true;
-        if (hasChunkableDescendants(child, langConfig))
-            return true;
-    }
-    return false;
-}
-function extractNodeName(node) {
-    const nameNode = node.childForFieldName("name") ??
-        node.children.find((c) => c.type === "identifier" ||
-            c.type === "type_identifier" ||
-            // nix bindings name their target via an attrpath (e.g. `outputs = ...`)
-            c.type === "attrpath");
-    return nameNode?.text;
-}
-/**
- * Fallback: chunk by sliding window for files with no grammar or no chunkable
- * structure (e.g. linear shell scripts). A small window overlap keeps a passage
- * spanning a window boundary recallable; it's kept light (20%) because the
- * extra rows are pure index bloat now that search dedups overlapping results.
- */
-function fallbackChunk(filePath, cwd) {
-    const absolutePath = join(cwd, filePath);
-    const source = readFileSync(absolutePath, "utf-8");
-    const lines = source.split("\n");
-    if (lines.length === 0)
-        return [];
-    const WINDOW = 50;
-    const STRIDE = 40;
-    const chunks = [];
-    for (let i = 0; i < lines.length; i += STRIDE) {
-        const slice = lines.slice(i, i + WINDOW);
-        const content = slice.join("\n");
-        if (content.trim().length === 0)
-            continue;
-        const hash = createHash("sha256")
-            .update(content)
-            .digest("hex")
-            .slice(0, 16);
-        chunks.push({
-            id: `${filePath}:${i}:${hash}`,
-            filePath,
-            startLine: i + 1,
-            endLine: Math.min(i + WINDOW, lines.length),
-            type: "block",
-            name: `lines_${i + 1}_${Math.min(i + WINDOW, lines.length)}`,
-            content,
-            context: `[file: ${filePath}]`,
-            hash,
-        });
-    }
-    return chunks;
-}
-/** Default chunker using tree-sitter with sliding-window fallback */
-export class TreeSitterChunker {
-    async chunk(filePath, cwd) {
-        return chunkFile(filePath, cwd);
-    }
-}
-//# sourceMappingURL=index.js.map

package/dist/lib/chunker/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/lib/chunker/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAa,MAAM,iBAAiB,CAAC;AAE9D,OAAO,EACN,kBAAkB,EAClB,YAAY,EACZ,qBAAqB,EACrB,YAAY,GACZ,MAAM,cAAc,CAAC;AACtB,OAAO,EAEN,kBAAkB,EAClB,WAAW,GACX,MAAM,gBAAgB,CAAC;AAExB,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,IAAI,cAAkC,CAAC;AACvC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAoB,CAAC;AAEpD,KAAK,UAAU,SAAS;IACvB,IAAI,CAAC,cAAc,EAAE,CAAC;QACrB,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,cAAc,GAAG,IAAI,MAAM,EAAE,CAAC;IAC/B,CAAC;IACD,OAAO,cAAc,CAAC;AACvB,CAAC;AAED,KAAK,UAAU,WAAW,CACzB,UAA0B;IAE1B,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAClD,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAE1B,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;IACzC,IAAI,CAAC,QAAQ;QAAE,OAAO,SAAS,CAAC;IAEhC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3C,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IACzC,OAAO,IAAI,CAAC;AACb,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,SAAS,CAC9B,QAAgB,EAChB,GAAW;IAEX,MAAM,UAAU,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,CAAC,UAAU;QAAE,OAAO,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,CAAC;IAC/C,IAAI,CAAC,QAAQ;QAAE,OAAO,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAEnD,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAE7B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAClC,IAAI,CAAC,IAAI;QAAE,OAAO,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAE/C,IAAI,UAAU,CAAC,EAAE,KAAK,UAAU,EAAE,CAAC;QAClC,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC3D,OAAO,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,IAAU,EAAE,QAAgB,EAAE,MAAc;IACnE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEjC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,CAAC,IAAU,EAAQ,EAAE;QAClC,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACnE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAC1C,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ;YAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IACjD,CAAC,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,CAAC;IAEZ,yEAAyE;IACzE,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvE,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;QACjE,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAE1C,MAAM,WAAW,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACzE,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,SAAS,KAAK,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;QACrE,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;aAC/B,MAAM,CAAC,OAAO,CAAC;aACf,MAAM,CAAC,KAAK,CAAC;aACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEf,MAAM,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,EAAE;YAClC,QAAQ;YACR,SAAS,EAAE,KAAK,GAAG,CAAC;YACpB,OAAO,EAAE,GAAG;YACZ,IAAI,EAAE,SAAS;YACf,IAAI;YACJ,OAAO;YACP,OAAO,EAAE,UAAU,QAAQ,GAAG;YAC9B,IAAI;SACJ,CAAC,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CACrB,IAAU,EACV,UAA0B,EAC1B,QAAgB,EAChB,MAAc,EACd,MAAe;IAEf,IAAI,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC;QAE1B,8CAA8C;QAC9C,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtD,IACC,eAAe,GAAG,gBAAgB;YAClC,uBAAuB,CAAC,IAAI,EAAE,UAAU,CAAC,EACxC,CAAC;YACF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnC,aAAa,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YAC5D,CAAC;YACD,OAAO;QACR,CAAC;QAED,MAAM,IAAI,GACT,eAAe,CAAC,IAAI,CAAC,IAAI,aAAa,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC;QAEhE,yBAAyB;QACzB,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC3D,OAAO;QACR,CAAC;QAED,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAC7C,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAEhC,MAAM,OAAO,GAAG,kBAAkB,CAAC;YAClC,QAAQ;YACR,KAAK;YACL,cAAc;YACd,IAAI;SACJ,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;aAC/B,MAAM,CAAC,OAAO,CAAC;aACf,MAAM,CAAC,KAAK,CAAC;aACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEf,MAAM,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,GAAG,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,IAAI,IAAI,EAAE;YACnD,QAAQ;YACR,SAAS,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC;YACrC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC;YACjC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI;YACJ,OAAO;YACP,OAAO;YACP,IAAI;SACJ,CAAC,CAAC;QACH,OAAO;IACR,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnC,aAAa,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5D,CAAC;AACF,CAAC;AAED,SAAS,uBAAuB,CAC/B,IAAU,EACV,UAA0B;IAE1B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnC,IAAI,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QAC5D,IAAI,uBAAuB,CAAC,KAAK,EAAE,UAAU,CAAC;YAAE,OAAO,IAAI,CAAC;IAC7D,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,eAAe,CAAC,IAAU;IAClC,MAAM,QAAQ,GACb,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,IAAI,CACjB,CAAC,CAAO,EAAE,EAAE,CACX,CAAC,CAAC,IAAI,KAAK,YAAY;YACvB,CAAC,CAAC,IAAI,KAAK,iBAAiB;YAC5B,wEAAwE;YACxE,CAAC,CAAC,IAAI,KAAK,UAAU,CACtB,CAAC;IACH,OAAO,QAAQ,EAAE,IAAI,CAAC;AACvB,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,QAAgB,EAAE,GAAW;IACnD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAE1C,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;aAC/B,MAAM,CAAC,OAAO,CAAC;aACf,MAAM,CAAC,KAAK,CAAC;aACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEf,MAAM,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,GAAG,QAAQ,IAAI,CAAC,IAAI,IAAI,EAAE;YAC9B,QAAQ;YACR,SAAS,EAAE,CAAC,GAAG,CAAC;YAChB,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC;YAC3C,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,SAAS,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE;YAC5D,OAAO;YACP,OAAO,EAAE,UAAU,QAAQ,GAAG;YAC9B,IAAI;SACJ,CAAC,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED,qEAAqE;AACrE,MAAM,OAAO,iBAAiB;IAC7B,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,GAAW;QACxC,OAAO,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC;CACD"}

package/dist/lib/chunker/languages.d.ts

@@ -1,15 +0,0 @@
-export type LanguageId = "javascript" | "typescript" | "tsx" | "python" | "rust" | "go" | "ruby" | "c" | "cpp" | "swift" | "json" | "yaml" | "toml" | "lua" | "scala" | "zig" | "bash" | "html" | "css" | "java" | "kotlin" | "php" | "c_sharp" | "vue" | "markdown" | "nix";
-export interface LanguageConfig {
-    id: LanguageId;
-    extensions: string[];
-    wasmFile: string;
-    /** AST node types that represent top-level or nestable definitions */
-    chunkTypes: string[];
-    /** AST node types that represent import/require statements */
-    importTypes: string[];
-    /** AST node types that represent class/struct/module wrappers */
-    scopeTypes: string[];
-}
-export declare const LANGUAGES: LanguageConfig[];
-export declare function getLanguageForFile(filePath: string): LanguageConfig | undefined;
-export declare function getWasmPath(lang: LanguageConfig): string | undefined;

package/dist/lib/chunker/languages.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"languages.js","sourceRoot":"","sources":["../../../src/lib/chunker/languages.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AA0C/C,MAAM,CAAC,MAAM,SAAS,GAAqB;IAC1C;QACC,EAAE,EAAE,YAAY;QAChB,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;QAC3B,QAAQ,EAAE,6BAA6B;QACvC,UAAU,EAAE;YACX,sBAAsB;YACtB,gBAAgB;YAChB,mBAAmB;YACnB,mBAAmB;YACnB,kBAAkB;YAClB,qBAAqB;SACrB;QACD,WAAW,EAAE,CAAC,kBAAkB,EAAE,iBAAiB,CAAC;QACpD,UAAU,EAAE,CAAC,mBAAmB,EAAE,YAAY,CAAC;KAC/C;IACD;QACC,EAAE,EAAE,YAAY;QAChB,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,6BAA6B;QACvC,UAAU,EAAE;YACX,sBAAsB;YACtB,gBAAgB;YAChB,mBAAmB;YACnB,mBAAmB;YACnB,uBAAuB;YACvB,wBAAwB;YACxB,kBAAkB;YAClB,kBAAkB;YAClB,qBAAqB;SACrB;QACD,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,QAAQ;YACR,uBAAuB;SACvB;KACD;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE;YACX,sBAAsB;YACtB,gBAAgB;YAChB,mBAAmB;YACnB,mBAAmB;YACnB,uBAAuB;YACvB,wBAAwB;YACxB,kBAAkB;YAClB,kBAAkB;YAClB,qBAAqB;SACrB;QACD,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,QAAQ;YACR,uBAAuB;SACvB;KACD;IACD;QACC,EAAE,EAAE,QAAQ;QACZ,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,yBAAyB;QACnC,UAAU,EAAE;YACX,qBAAqB;YACrB,kBAAkB;YAClB,sBAAsB;SACtB;QACD,WAAW,EAAE,CAAC,kBAAkB,EAAE,uBAAuB,CAAC;QAC1D,UAAU,EAAE,CAAC,kBAAkB,EAAE,QAAQ,CAAC;KAC1C;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE;YACX,eAAe;YACf,WAAW;YACX,aAAa;YACb,WAAW;YACX,YAAY;YACZ,WAAW;YACX,YAAY;YACZ,aAAa;YACb,kBAAkB;SAClB;QACD,WAAW,EAAE,CAAC,iBAAiB,CAAC;QAChC,UAAU,EAAE,CAAC,WAAW,EAAE,YAAY,EAAE,UAAU,CAAC;KACnD;IACD;QACC,EAAE,EAAE,IAAI;QACR,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,qBAAqB;QAC/B,UAAU,EAAE;YACX,sBAAsB;YACtB,oBAAoB;YACpB,kBAAkB;YAClB,mBAAmB;YACnB,iBAAiB;SACjB;QACD,WAAW,EAAE,CAAC,oBAAoB,CAAC;QACnC,UAAU,EAAE,CAAC,kBAAkB,CAAC;KAChC;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,QAAQ,EAAE,kBAAkB,EAAE,OAAO,EAAE,QAAQ,CAAC;QAC7D,WAAW,EAAE,CAAC,MAAM,CAAC;QACrB,UAAU,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC;KAC/B;IACD;QACC,EAAE,EAAE,GAAG;QACP,UAAU,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;QACxB,QAAQ,EAAE,oBAAoB;QAC9B,UAAU,EAAE;YACX,qBAAqB;YACrB,kBAAkB;YAClB,gBAAgB;YAChB,iBAAiB;YACjB,aAAa;SACb;QACD,WAAW,EAAE,CAAC,iBAAiB,CAAC;QAChC,UAAU,EAAE,CAAC,kBAAkB,CAAC;KAChC;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;QACnC,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE;YACX,qBAAqB;YACrB,iBAAiB;YACjB,kBAAkB;YAClB,sBAAsB;YACtB,sBAAsB;YACtB,gBAAgB;SAChB;QACD,WAAW,EAAE,CAAC,iBAAiB,EAAE,mBAAmB,CAAC;QACrD,UAAU,EAAE;YACX,iBAAiB;YACjB,kBAAkB;YAClB,sBAAsB;SACtB;KACD;IACD;QACC,EAAE,EAAE,OAAO;QACX,UAAU,EAAE,CAAC,QAAQ,CAAC;QACtB,QAAQ,EAAE,wBAAwB;QAClC,yEAAyE;QACzE,uEAAuE;QACvE,UAAU,EAAE;YACX,sBAAsB;YACtB,mBAAmB;YACnB,sBAAsB;SACtB;QACD,WAAW,EAAE,CAAC,oBAAoB,CAAC;QACnC,UAAU,EAAE,CAAC,mBAAmB,CAAC;KACjC;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,QAAQ,CAAC;KACtB;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;QAC7B,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,oBAAoB,CAAC;QAClC,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,eAAe,CAAC;KAC7B;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;QAC7B,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,OAAO,CAAC;KACrB;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,2EAA2E;QAC3E,UAAU,EAAE,CAAC,sBAAsB,EAAE,qBAAqB,CAAC;QAC3D,WAAW,EAAE,CAAC,MAAM,CAAC;QACrB,UAAU,EAAE,EAAE;KACd;IACD;QACC,EAAE,EAAE,OAAO;QACX,UAAU,EAAE,CAAC,QAAQ,CAAC;QACtB,QAAQ,EAAE,wBAAwB;QAClC,UAAU,EAAE;YACX,qBAAqB;YACrB,gBAAgB;YAChB,gBAAgB;YAChB,kBAAkB;YAClB,mBAAmB;YACnB,kBAAkB;SAClB;QACD,WAAW,EAAE,CAAC,oBAAoB,CAAC;QACnC,UAAU,EAAE;YACX,kBAAkB;YAClB,mBAAmB;YACnB,kBAAkB;SAClB;KACD;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE;YACX,sBAAsB;YACtB,sBAAsB;YACtB,kBAAkB;SAClB;QACD,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,oBAAoB,EAAE,kBAAkB,EAAE,mBAAmB,CAAC;KAC3E;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC;QAC5B,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,qBAAqB,CAAC;QACnC,WAAW,EAAE,CAAC,SAAS,CAAC;QACxB,UAAU,EAAE,EAAE;KACd;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,SAAS,EAAE,gBAAgB,EAAE,eAAe,CAAC;QAC1D,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,SAAS,CAAC;KACvB;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;QAC7B,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE,CAAC,UAAU,EAAE,iBAAiB,EAAE,qBAAqB,CAAC;QAClE,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,UAAU,EAAE,EAAE;KACd;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,kBAAkB;YAClB,oBAAoB;YACpB,yBAAyB;YACzB,oBAAoB;SACpB;QACD,WAAW,EAAE,CAAC,oBAAoB,CAAC;QACnC,UAAU,EAAE,CAAC,mBAAmB,EAAE,uBAAuB,CAAC;KAC1D;IACD;QACC,EAAE,EAAE,QAAQ;QACZ,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,yBAAyB;QACnC,UAAU,EAAE;YACX,mBAAmB;YACnB,oBAAoB;YACpB,sBAAsB;YACtB,sBAAsB;SACtB;QACD,WAAW,EAAE,CAAC,eAAe,CAAC;QAC9B,UAAU,EAAE,CAAC,mBAAmB,EAAE,oBAAoB,CAAC;KACvD;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,mBAAmB;YACnB,kBAAkB;YAClB,qBAAqB;YACrB,oBAAoB;SACpB;QACD,WAAW,EAAE,CAAC,2BAA2B,CAAC;QAC1C,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,mBAAmB;SACnB;KACD;IACD;QACC,EAAE,EAAE,SAAS;QACb,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,yBAAyB;QACnC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,oBAAoB;YACpB,kBAAkB;YAClB,oBAAoB;YACpB,oBAAoB;YACpB,yBAAyB;YACzB,sBAAsB;SACtB;QACD,WAAW,EAAE,CAAC,iBAAiB,CAAC;QAChC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,oBAAoB;YACpB,uBAAuB;SACvB;KACD;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE,CAAC,gBAAgB,EAAE,kBAAkB,EAAE,eAAe,CAAC;QACnE,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,EAAE;KACd;IACD;QACC,EAAE,EAAE,UAAU;QACd,UAAU,EAAE,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,CAAC;QACxC,QAAQ,EAAE,2BAA2B;QACrC,wEAAwE;QACxE,yEAAyE;QACzE,wEAAwE;QACxE,UAAU,EAAE,CAAC,SAAS,CAAC;QACvB,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,SAAS,CAAC;KACvB;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE,CAAC,SAAS,CAAC;QACvB,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,oBAAoB,EAAE,aAAa,CAAC;KACjD;CACD,CAAC;AAEF,MAAM,SAAS,GAAG,IAAI,GAAG,EAA0B,CAAC;AACpD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACnC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC1B,CAAC;AACF,CAAC;AAED,MAAM,UAAU,kBAAkB,CACjC,QAAgB;IAEhB,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;IACtD,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAoB;IAC/C,6EAA6E;IAC7E,6EAA6E;IAC7E,2EAA2E;IAC3E,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC/E,IAAI,CAAC;QACJ,OAAO,OAAO,CAAC,OAAO,CAAC,kBAAkB,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC"}

package/dist/lib/config.d.ts

@@ -1,12 +0,0 @@
-import type { LmgrepConfig } from "./types.js";
-/**
- * Return the XDG-compliant global config directory for lmgrep.
- *   Linux:  $XDG_CONFIG_HOME/lmgrep  or  ~/.config/lmgrep
- *   macOS:  ~/Library/Application Support/lmgrep
- */
-export declare function getConfigDir(): string;
-/**
- * Return the path to the global config file.
- */
-export declare function getGlobalConfigPath(): string;
-export declare function loadConfig(cwd: string): LmgrepConfig;

package/dist/lib/config.js

@@ -1,83 +0,0 @@
-import { readFileSync, existsSync } from "node:fs";
-import { homedir, platform } from "node:os";
-import { join } from "node:path";
-import { parse } from "yaml";
-import { z } from "zod";
-const ConfigSchema = z.object({
-    model: z
-        .string()
-        .regex(/^.+:.+$/, 'Model must be in "provider:model" format (e.g. "ollama:nomic-embed-text")'),
-    provider: z.string().optional(),
-    baseURL: z.string().url().optional(),
-    local: z.boolean().optional(),
-    batchSize: z.number().int().positive().default(100),
-    dimensions: z.number().int().positive().optional(),
-    queryPrefix: z.string().optional(),
-    documentPrefix: z.string().optional(),
-    maxTokens: z.number().int().positive().optional(),
-    ignore: z.array(z.string()).optional(),
-    extensions: z
-        .object({
-        include: z.array(z.string()).optional(),
-        exclude: z.array(z.string()).optional(),
-    })
-        .optional(),
-});
-const CONFIG_FILE = "config.yml";
-const DEFAULTS = {
-    batchSize: 100,
-};
-/**
- * Return the XDG-compliant global config directory for lmgrep.
- *   Linux:  $XDG_CONFIG_HOME/lmgrep  or  ~/.config/lmgrep
- *   macOS:  ~/Library/Application Support/lmgrep
- */
-export function getConfigDir() {
-    if (platform() === "darwin") {
-        return join(homedir(), "Library", "Application Support", "lmgrep");
-    }
-    const xdg = process.env.XDG_CONFIG_HOME;
-    return join(xdg || join(homedir(), ".config"), "lmgrep");
-}
-/**
- * Return the path to the global config file.
- */
-export function getGlobalConfigPath() {
-    return join(getConfigDir(), CONFIG_FILE);
-}
-function tryLoadFile(path) {
-    if (!existsSync(path))
-        return undefined;
-    const raw = readFileSync(path, "utf-8");
-    const parsed = parse(raw);
-    if (parsed == null || typeof parsed !== "object")
-        return undefined;
-    return ConfigSchema.partial().parse(parsed);
-}
-export function loadConfig(cwd) {
-    // Project-local config (.lmgrep.yml in project root)
-    for (const name of [".lmgrep.yml", ".lmgrep.yaml"]) {
-        const found = tryLoadFile(join(cwd, name));
-        if (found)
-            return validateConfig({ ...DEFAULTS, ...found });
-    }
-    // Global config (XDG config dir)
-    const globalConfig = tryLoadFile(getGlobalConfigPath());
-    if (globalConfig)
-        return validateConfig({ ...DEFAULTS, ...globalConfig });
-    // Legacy: ~/.lmgrep.yml (for backwards compat)
-    for (const name of [".lmgrep.yml", ".lmgrep.yaml"]) {
-        const found = tryLoadFile(join(homedir(), name));
-        if (found)
-            return validateConfig({ ...DEFAULTS, ...found });
-    }
-    throw new Error("No configuration found. Run `lmgrep init` to set up your embedding model.");
-}
-function validateConfig(config) {
-    if (!config.model) {
-        throw new Error("No model configured. Set `model` in .lmgrep.yml (e.g. `model: ollama:nomic-embed-text`).\n" +
-            "Run `lmgrep init` to auto-detect your setup.");
-    }
-    return config;
-}
-//# sourceMappingURL=config.js.map

package/dist/lib/config.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,KAAK,EAAE,CAAC;SACN,MAAM,EAAE;SACR,KAAK,CAAC,SAAS,EAAE,2EAA2E,CAAC;IAC/F,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACpC,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC7B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IACnD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAClD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACrC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtC,UAAU,EAAE,CAAC;SACX,MAAM,CAAC;QACP,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;QACvC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;KACvC,CAAC;SACD,QAAQ,EAAE;CACZ,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,YAAY,CAAC;AAEjC,MAAM,QAAQ,GAAqD;IAClE,SAAS,EAAE,GAAG;CACd,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,YAAY;IAC3B,IAAI,QAAQ,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,qBAAqB,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC;IACD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IACxC,OAAO,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC,EAAE,QAAQ,CAAC,CAAC;AAC1D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB;IAClC,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,WAAW,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAChC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,MAAM,IAAI,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IACnE,OAAO,YAAY,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAW;IACrC,qDAAqD;IACrD,KAAK,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QAC3C,IAAI,KAAK;YAAE,OAAO,cAAc,CAAC,EAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,iCAAiC;IACjC,MAAM,YAAY,GAAG,WAAW,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACxD,IAAI,YAAY;QAAE,OAAO,cAAc,CAAC,EAAE,GAAG,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAE1E,+CAA+C;IAC/C,KAAK,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;QACjD,IAAI,KAAK;YAAE,OAAO,cAAc,CAAC,EAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,IAAI,KAAK,CACd,2EAA2E,CAC3E,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACtB,MAA+C;IAE/C,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACd,4FAA4F;YAC3F,8CAA8C,CAC/C,CAAC;IACH,CAAC;IACD,OAAO,MAAsB,CAAC;AAC/B,CAAC"}

package/dist/lib/embedder.d.ts

@@ -1,65 +0,0 @@
-import type { LmgrepConfig } from "./types.js";
-export interface Embedder {
-    embed(texts: string[]): Promise<number[][]>;
-    embedQuery(query: string): Promise<number[]>;
-}
-export interface EmbedderEvents {
-    onBatchStart?: (batchNum: number, totalBatches: number) => void;
-    onBatchDone?: (batchNum: number, succeeded: number, failed: number) => void;
-    onReload?: (attempt: number, maxAttempts: number) => void;
-}
-/**
- * Thrown when too many consecutive embedding batches fail.
- * The caller should persist any successful chunks before surfacing this error.
- */
-export declare class EmbeddingAbortError extends Error {
-    readonly vectors: (number[] | null)[];
-    readonly failedIndices: Set<number>;
-    readonly total: number;
-    constructor(vectors: (number[] | null)[], failedIndices: Set<number>, total: number);
-}
-export declare class AISDKEmbedder implements Embedder {
-    private cachedModel;
-    private config;
-    private validated;
-    constructor(config: LmgrepConfig);
-    private validateDimension;
-    private getModel;
-    embed(texts: string[]): Promise<number[][]>;
-    embedQuery(query: string): Promise<number[]>;
-}
-/**
- * Resilient embedder that retries failed batches individually.
- * On too many consecutive failures, throws EmbeddingAbortError so the caller
- * can persist partial progress and exit cleanly.
- *
- * For local LM Studio setups with baseURL set to localhost, it will attempt
- * to reload the model before aborting.
- */
-export declare class ResilientEmbedder {
-    private embedder;
-    private config;
-    private events?;
-    private consecutiveFailures;
-    private reloads;
-    private readonly isLocal;
-    constructor(embedder: Embedder, config: LmgrepConfig, events?: EmbedderEvents | undefined);
-    private static readonly BATCH_DELAY_MS;
-    private static readonly MAX_CONSECUTIVE_FAILURES;
-    private static readonly MAX_RELOADS;
-    /**
-     * Embed `texts` batch by batch. After each batch, `onBatch` (if given) is
-     * awaited with the items that just embedded successfully, so the caller can
-     * persist incrementally rather than buffering every vector until the end —
-     * embeddings then survive a hard crash and resume skips completed work.
-     * Each successful index is reported through `onBatch` exactly once.
-     */
-    embedBatched(texts: string[], onBatch?: (items: Array<{
-        index: number;
-        vector: number[];
-    }>) => Promise<void>): Promise<{
-        vectors: (number[] | null)[];
-        failedIndices: Set<number>;
-    }>;
-    private reloadModel;
-}

package/dist/lib/embedder.js

@@ -1,236 +0,0 @@
-import { embed, embedMany } from "ai";
-import { createProviderRegistry } from "ai";
-import { importProvider } from "./providers.js";
-/**
- * Thrown when too many consecutive embedding batches fail.
- * The caller should persist any successful chunks before surfacing this error.
- */
-export class EmbeddingAbortError extends Error {
-    vectors;
-    failedIndices;
-    total;
-    constructor(vectors, failedIndices, total) {
-        const succeeded = vectors.filter((v) => v !== null).length;
-        super(`Embedding aborted after ${failedIndices.size} failures. ` +
-            `${succeeded}/${total} chunks embedded successfully. ` +
-            `Check your embedding provider and run \`lmgrep index\` to resume.`);
-        this.vectors = vectors;
-        this.failedIndices = failedIndices;
-        this.total = total;
-        this.name = "EmbeddingAbortError";
-    }
-}
-export class AISDKEmbedder {
-    cachedModel;
-    config;
-    validated = false;
-    constructor(config) {
-        this.config = config;
-    }
-    validateDimension(vector) {
-        if (this.validated)
-            return;
-        this.validated = true;
-        if (this.config.dimensions != null &&
-            vector.length !== this.config.dimensions) {
-            throw new Error(`Embedding dimension mismatch: expected ${this.config.dimensions}, got ${vector.length}. Check your model and config.dimensions.`);
-        }
-    }
-    async getModel() {
-        if (this.cachedModel)
-            return this.cachedModel;
-        const colonIdx = this.config.model.indexOf(":");
-        if (colonIdx === -1) {
-            throw new Error(`Model must be in "provider:model" format. Got: "${this.config.model}"`);
-        }
-        const providerName = this.config.model.slice(0, colonIdx);
-        const pkg = this.config.provider ?? `@ai-sdk/${providerName}`;
-        const providerModule = await importProvider(pkg);
-        let providerInstance = providerModule[providerName];
-        if (!providerInstance) {
-            const factoryKey = Object.keys(providerModule).find((k) => k.startsWith("create"));
-            if (factoryKey) {
-                const factory = providerModule[factoryKey];
-                providerInstance = factory({
-                    name: providerName,
-                    ...(this.config.baseURL ? { baseURL: this.config.baseURL } : {}),
-                });
-            }
-        }
-        if (!providerInstance) {
-            throw new Error(`Package "${pkg}" has no usable provider export. Available: ${Object.keys(providerModule).join(", ")}`);
-        }
-        const registry = createProviderRegistry({
-            [providerName]: providerInstance,
-        });
-        this.cachedModel = registry.embeddingModel(this.config.model);
-        return this.cachedModel;
-    }
-    async embed(texts) {
-        const model = await this.getModel();
-        const prefix = this.config.documentPrefix ?? "";
-        const allEmbeddings = [];
-        for (let i = 0; i < texts.length; i += this.config.batchSize) {
-            const batch = texts.slice(i, i + this.config.batchSize);
-            const values = prefix ? batch.map((t) => prefix + t) : batch;
-            const { embeddings } = await embedMany({ model, values });
-            if (embeddings.length > 0)
-                this.validateDimension(embeddings[0]);
-            allEmbeddings.push(...embeddings);
-        }
-        return allEmbeddings;
-    }
-    async embedQuery(query) {
-        const model = await this.getModel();
-        const prefix = this.config.queryPrefix ?? "";
-        const { embedding } = await embed({ model, value: prefix + query });
-        this.validateDimension(embedding);
-        return embedding;
-    }
-}
-/**
- * Resilient embedder that retries failed batches individually.
- * On too many consecutive failures, throws EmbeddingAbortError so the caller
- * can persist partial progress and exit cleanly.
- *
- * For local LM Studio setups with baseURL set to localhost, it will attempt
- * to reload the model before aborting.
- */
-export class ResilientEmbedder {
-    embedder;
-    config;
-    events;
-    consecutiveFailures = 0;
-    reloads = 0;
-    isLocal;
-    constructor(embedder, config, events) {
-        this.embedder = embedder;
-        this.config = config;
-        this.events = events;
-        this.isLocal = isLocalProvider(config.baseURL);
-    }
-    static BATCH_DELAY_MS = 200;
-    static MAX_CONSECUTIVE_FAILURES = 3;
-    static MAX_RELOADS = 3;
-    /**
-     * Embed `texts` batch by batch. After each batch, `onBatch` (if given) is
-     * awaited with the items that just embedded successfully, so the caller can
-     * persist incrementally rather than buffering every vector until the end —
-     * embeddings then survive a hard crash and resume skips completed work.
-     * Each successful index is reported through `onBatch` exactly once.
-     */
-    async embedBatched(texts, onBatch) {
-        const batchSize = this.config.batchSize;
-        const vectors = new Array(texts.length).fill(null);
-        const failedIndices = new Set();
-        const totalBatches = Math.ceil(texts.length / batchSize);
-        for (let i = 0; i < texts.length; i += batchSize) {
-            const batchNum = Math.floor(i / batchSize) + 1;
-            const end = Math.min(i + batchSize, texts.length);
-            const batch = texts.slice(i, end);
-            this.events?.onBatchStart?.(batchNum, totalBatches);
-            const persisted = [];
-            try {
-                const batchVectors = await this.embedder.embed(batch);
-                for (let j = 0; j < batch.length; j++) {
-                    vectors[i + j] = batchVectors[j];
-                    persisted.push({ index: i + j, vector: batchVectors[j] });
-                }
-                this.consecutiveFailures = 0;
-            }
-            catch {
-                // Batch failed — retry individually
-                let batchFailed = 0;
-                for (let j = 0; j < batch.length; j++) {
-                    try {
-                        const [vec] = await this.embedder.embed([batch[j]]);
-                        vectors[i + j] = vec;
-                        persisted.push({ index: i + j, vector: vec });
-                    }
-                    catch (err) {
-                        failedIndices.add(i + j);
-                        batchFailed++;
-                        console.error(`  ! Failed (~${Math.ceil(batch[j].length / 4)} tok): ${err instanceof Error ? err.message : err}`);
-                    }
-                }
-                if (batchFailed === batch.length) {
-                    this.consecutiveFailures++;
-                }
-                else {
-                    this.consecutiveFailures = 0;
-                }
-            }
-            // Persist this batch's successes before any abort below, so partial
-            // progress is durable even if the next batches force an abort.
-            if (onBatch && persisted.length > 0) {
-                await onBatch(persisted);
-            }
-            const succeeded = vectors.filter((v) => v !== null).length;
-            this.events?.onBatchDone?.(batchNum, succeeded, failedIndices.size);
-            // Handle consecutive failures
-            if (this.consecutiveFailures >=
-                ResilientEmbedder.MAX_CONSECUTIVE_FAILURES) {
-                // For local providers, try reloading the model
-                if (this.isLocal && this.reloads < ResilientEmbedder.MAX_RELOADS) {
-                    this.reloads++;
-                    this.events?.onReload?.(this.reloads, ResilientEmbedder.MAX_RELOADS);
-                    const ok = await this.reloadModel();
-                    if (ok) {
-                        this.consecutiveFailures = 0;
-                        i -= ResilientEmbedder.MAX_CONSECUTIVE_FAILURES * batchSize;
-                        continue;
-                    }
-                }
-                // Mark remaining as failed
-                for (let k = end; k < texts.length; k++) {
-                    failedIndices.add(k);
-                }
-                // Abort — carry partial results so the caller can persist them
-                throw new EmbeddingAbortError(vectors, failedIndices, texts.length);
-            }
-            // Cooldown between batches
-            if (end < texts.length) {
-                await new Promise((r) => setTimeout(r, ResilientEmbedder.BATCH_DELAY_MS));
-            }
-        }
-        return { vectors, failedIndices };
-    }
-    async reloadModel() {
-        const modelId = this.config.model.split(":").slice(1).join(":");
-        const baseURL = this.config.baseURL ?? "http://localhost:1234";
-        const apiBase = baseURL.replace(/\/v1\/?$/, "");
-        try {
-            await fetch(`${apiBase}/api/v1/models/unload`, {
-                method: "POST",
-                headers: { "Content-Type": "application/json" },
-                body: JSON.stringify({ instance_id: modelId }),
-            });
-            await new Promise((r) => setTimeout(r, 2000));
-            const res = await fetch(`${apiBase}/api/v1/models/load`, {
-                method: "POST",
-                headers: { "Content-Type": "application/json" },
-                body: JSON.stringify({
-                    model: modelId,
-                    context_length: this.config.maxTokens ?? 8192,
-                }),
-            });
-            await res.json();
-            return true;
-        }
-        catch {
-            return false;
-        }
-    }
-}
-function isLocalProvider(baseURL) {
-    if (!baseURL)
-        return false;
-    try {
-        const host = new URL(baseURL).hostname;
-        return host === "localhost" || host === "127.0.0.1" || host === "0.0.0.0";
-    }
-    catch {
-        return false;
-    }
-}
-//# sourceMappingURL=embedder.js.map