@liendev/core 0.19.5

package/dist/indexer/liquid-chunker.js

@@ -0,0 +1,272 @@
+/**
+ * Extract schema name from JSON content
+ *
+ * Extracts the "name" field from Shopify schema JSON.
+ * Uses JSON.parse to properly handle escaped quotes and other JSON edge cases.
+ *
+ * Example:
+ * {% schema %}
+ * {
+ *   "name": "My \"Special\" Section",
+ *   "settings": []
+ * }
+ * {% endschema %}
+ *
+ * Returns: 'My "Special" Section' (with literal quotes, unescaped)
+ */
+function extractSchemaName(schemaContent) {
+    try {
+        // Remove Liquid tags to isolate JSON content
+        // Replace {% schema %} and {% endschema %} (with optional whitespace control)
+        let jsonContent = schemaContent
+            .replace(/\{%-?\s*schema\s*-?%\}/g, '')
+            .replace(/\{%-?\s*endschema\s*-?%\}/g, '')
+            .trim();
+        // Parse the JSON
+        const schema = JSON.parse(jsonContent);
+        // Ensure name is a string before returning
+        return typeof schema.name === 'string' ? schema.name : undefined;
+    }
+    catch (error) {
+        // Invalid JSON - return undefined
+        // This is acceptable: schema blocks with invalid JSON won't have names extracted
+    }
+    return undefined;
+}
+/**
+ * Remove Liquid comment blocks from content to avoid extracting tags from comments
+ *
+ * Example:
+ * {% comment %}Don't use {% render 'old-snippet' %}{% endcomment %}
+ * → (removed)
+ */
+function removeComments(content) {
+    // Remove {% comment %}...{% endcomment %} blocks (with optional whitespace control)
+    return content.replace(/\{%-?\s*comment\s*-?%\}[\s\S]*?\{%-?\s*endcomment\s*-?%\}/g, '');
+}
+/**
+ * Extract dependencies from {% render %}, {% include %}, and {% section %} tags
+ *
+ * Examples:
+ * - {% render 'product-card' %} → 'product-card'
+ * - {% render "cart-item", product: product %} → 'cart-item'
+ * - {% include 'snippets/header' %} → 'snippets/header'
+ * - {% section 'announcement-bar' %} → 'announcement-bar'
+ *
+ * Limitations:
+ * - Does not handle escaped quotes in snippet names (e.g., {% render 'name\'s' %})
+ * - This is acceptable because Shopify snippet names map to filenames, and
+ *   filesystem restrictions prevent quotes in filenames (snippets/name's.liquid is invalid)
+ * - In practice, Shopify snippet names use only alphanumeric, dash, and underscore
+ *
+ * Note: Expects content with comments already removed for performance
+ *
+ * @param contentWithoutComments - Content with Liquid comments already removed
+ */
+function extractRenderTags(contentWithoutComments) {
+    const dependencies = new Set();
+    // Match {% render 'snippet-name' %} or {% render "snippet-name" %}
+    // Note: Does not handle escaped quotes - see function docs for rationale
+    const renderPattern = /\{%-?\s*render\s+['"]([^'"]+)['"]/g;
+    let match;
+    while ((match = renderPattern.exec(contentWithoutComments)) !== null) {
+        dependencies.add(match[1]);
+    }
+    // Match {% include 'snippet-name' %} or {% include "snippet-name" %}
+    const includePattern = /\{%-?\s*include\s+['"]([^'"]+)['"]/g;
+    while ((match = includePattern.exec(contentWithoutComments)) !== null) {
+        dependencies.add(match[1]);
+    }
+    // Match {% section 'section-name' %} or {% section "section-name" %}
+    const sectionPattern = /\{%-?\s*section\s+['"]([^'"]+)['"]/g;
+    while ((match = sectionPattern.exec(contentWithoutComments)) !== null) {
+        dependencies.add(match[1]);
+    }
+    return Array.from(dependencies);
+}
+/**
+ * Find all special Liquid blocks in the template
+ *
+ * Limitation: Does not support nested blocks of the same type.
+ * - Matches first start tag with first end tag
+ * - This is acceptable because Shopify Liquid does not allow nested blocks
+ * - Example invalid: {% schema %}...{% schema %}...{% endschema %} (Shopify rejects this)
+ * - If malformed input contains nested blocks, only outermost block is extracted
+ */
+function findLiquidBlocks(content) {
+    const lines = content.split('\n');
+    const blocks = [];
+    // Regex patterns for Liquid blocks
+    // Note: Matches first start → first end (no nesting support, which is correct for Shopify)
+    const blockPatterns = [
+        { type: 'schema', start: /\{%-?\s*schema\s*-?%\}/, end: /\{%-?\s*endschema\s*-?%\}/ },
+        { type: 'style', start: /\{%-?\s*style\s*-?%\}/, end: /\{%-?\s*endstyle\s*-?%\}/ },
+        { type: 'javascript', start: /\{%-?\s*javascript\s*-?%\}/, end: /\{%-?\s*endjavascript\s*-?%\}/ },
+    ];
+    for (const pattern of blockPatterns) {
+        let searchStart = 0;
+        while (searchStart < lines.length) {
+            // Find start tag
+            const startIdx = lines.findIndex((line, idx) => idx >= searchStart && pattern.start.test(line));
+            if (startIdx === -1)
+                break;
+            // Find end tag (allow same line for single-line blocks)
+            const endIdx = lines.findIndex((line, idx) => idx >= startIdx && pattern.end.test(line));
+            if (endIdx === -1) {
+                // No end tag found, treat rest as template
+                break;
+            }
+            // Extract block content
+            const blockContent = lines.slice(startIdx, endIdx + 1).join('\n');
+            blocks.push({
+                type: pattern.type,
+                startLine: startIdx,
+                endLine: endIdx,
+                content: blockContent,
+            });
+            searchStart = endIdx + 1;
+        }
+    }
+    return blocks.sort((a, b) => a.startLine - b.startLine);
+}
+/**
+ * Create a CodeChunk with consistent structure
+ */
+function createCodeChunk(content, startLine, endLine, filepath, type, options = {}) {
+    return {
+        content,
+        metadata: {
+            file: filepath,
+            startLine,
+            endLine,
+            language: 'liquid',
+            type,
+            symbolName: options.symbolName,
+            symbolType: options.symbolType,
+            imports: options.imports?.length ? options.imports : undefined,
+        },
+    };
+}
+/**
+ * Split a large block into multiple chunks with overlap
+ */
+function splitLargeBlock(block, ctx, symbolName, imports) {
+    const chunks = [];
+    const blockLines = block.content.split('\n');
+    const { chunkSize, chunkOverlap, filepath } = ctx.params;
+    for (let offset = 0; offset < blockLines.length; offset += chunkSize - chunkOverlap) {
+        const endOffset = Math.min(offset + chunkSize, blockLines.length);
+        const chunkContent = blockLines.slice(offset, endOffset).join('\n');
+        if (chunkContent.trim().length > 0) {
+            chunks.push(createCodeChunk(chunkContent, block.startLine + offset + 1, block.startLine + endOffset, filepath, 'block', { symbolName, symbolType: block.type, imports }));
+        }
+        if (endOffset >= blockLines.length)
+            break;
+    }
+    return chunks;
+}
+/**
+ * Create chunks from a special Liquid block (schema, style, javascript)
+ * Returns the chunks and marks covered lines
+ */
+function processSpecialBlock(block, ctx, coveredLines) {
+    // Mark lines as covered
+    for (let i = block.startLine; i <= block.endLine; i++) {
+        coveredLines.add(i);
+    }
+    // Extract metadata
+    const symbolName = block.type === 'schema' ? extractSchemaName(block.content) : undefined;
+    // Extract imports from cleaned content
+    const blockContentWithoutComments = ctx.linesWithoutComments
+        .slice(block.startLine, block.endLine + 1)
+        .join('\n');
+    const imports = extractRenderTags(blockContentWithoutComments);
+    const blockLineCount = block.endLine - block.startLine + 1;
+    const maxBlockSize = ctx.params.chunkSize * 3;
+    // Keep small blocks as single chunk, split large ones
+    if (blockLineCount <= maxBlockSize) {
+        return [createCodeChunk(block.content, block.startLine + 1, block.endLine + 1, ctx.params.filepath, 'block', { symbolName, symbolType: block.type, imports })];
+    }
+    return splitLargeBlock(block, ctx, symbolName, imports);
+}
+/**
+ * Create a template chunk from accumulated lines
+ */
+function flushTemplateChunk(currentChunk, chunkStartLine, endLine, ctx) {
+    if (currentChunk.length === 0)
+        return null;
+    const chunkContent = currentChunk.join('\n');
+    if (chunkContent.trim().length === 0)
+        return null;
+    const cleanedChunk = ctx.linesWithoutComments.slice(chunkStartLine, endLine).join('\n');
+    const imports = extractRenderTags(cleanedChunk);
+    return createCodeChunk(chunkContent, chunkStartLine + 1, endLine, ctx.params.filepath, 'template', { imports });
+}
+/**
+ * Process uncovered template content into chunks
+ */
+function processTemplateContent(ctx, coveredLines) {
+    const chunks = [];
+    const { lines, params } = ctx;
+    const { chunkSize, chunkOverlap } = params;
+    let currentChunk = [];
+    let chunkStartLine = 0;
+    for (let i = 0; i < lines.length; i++) {
+        // Skip lines covered by special blocks
+        if (coveredLines.has(i)) {
+            const chunk = flushTemplateChunk(currentChunk, chunkStartLine, i, ctx);
+            if (chunk)
+                chunks.push(chunk);
+            currentChunk = [];
+            continue;
+        }
+        // Start new chunk if needed
+        if (currentChunk.length === 0) {
+            chunkStartLine = i;
+        }
+        currentChunk.push(lines[i]);
+        // Flush if chunk is full
+        if (currentChunk.length >= chunkSize) {
+            const chunk = flushTemplateChunk(currentChunk, chunkStartLine, i + 1, ctx);
+            if (chunk)
+                chunks.push(chunk);
+            // Add overlap for next chunk
+            currentChunk = currentChunk.slice(-chunkOverlap);
+            chunkStartLine = Math.max(0, i + 1 - chunkOverlap);
+        }
+    }
+    // Flush remaining chunk
+    const finalChunk = flushTemplateChunk(currentChunk, chunkStartLine, lines.length, ctx);
+    if (finalChunk)
+        chunks.push(finalChunk);
+    return chunks;
+}
+/**
+ * Chunk a Liquid template file
+ *
+ * Special handling for:
+ * - {% schema %} blocks (kept together, extract section name)
+ * - {% style %} blocks (kept together)
+ * - {% javascript %} blocks (kept together)
+ * - {% render %}, {% include %}, and {% section %} tags (tracked as imports)
+ * - Regular template content (chunked by lines)
+ */
+export function chunkLiquidFile(filepath, content, chunkSize = 75, chunkOverlap = 10) {
+    // Build context once for reuse across helpers
+    const contentWithoutComments = removeComments(content);
+    const ctx = {
+        lines: content.split('\n'),
+        linesWithoutComments: contentWithoutComments.split('\n'),
+        params: { filepath, chunkSize, chunkOverlap },
+    };
+    // Find special blocks and track covered lines
+    const blocks = findLiquidBlocks(content);
+    const coveredLines = new Set();
+    // Process special blocks
+    const blockChunks = blocks.flatMap(block => processSpecialBlock(block, ctx, coveredLines));
+    // Process uncovered template content
+    const templateChunks = processTemplateContent(ctx, coveredLines);
+    // Combine and sort by line number
+    return [...blockChunks, ...templateChunks].sort((a, b) => a.metadata.startLine - b.metadata.startLine);
+}
+//# sourceMappingURL=liquid-chunker.js.map

package/dist/indexer/liquid-chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"liquid-chunker.js","sourceRoot":"","sources":["../../src/indexer/liquid-chunker.ts"],"names":[],"mappings":"AAgBA;;;;;;;;;;;;;;;GAeG;AACH,SAAS,iBAAiB,CAAC,aAAqB;IAC9C,IAAI,CAAC;QACH,6CAA6C;QAC7C,8EAA8E;QAC9E,IAAI,WAAW,GAAG,aAAa;aAC5B,OAAO,CAAC,yBAAyB,EAAE,EAAE,CAAC;aACtC,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;aACzC,IAAI,EAAE,CAAC;QAEV,iBAAiB;QACjB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACvC,2CAA2C;QAC3C,OAAO,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,kCAAkC;QAClC,iFAAiF;IACnF,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,OAAe;IACrC,oFAAoF;IACpF,OAAO,OAAO,CAAC,OAAO,CAAC,4DAA4D,EAAE,EAAE,CAAC,CAAC;AAC3F,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAS,iBAAiB,CAAC,sBAA8B;IACvD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IAEvC,mEAAmE;IACnE,yEAAyE;IACzE,MAAM,aAAa,GAAG,oCAAoC,CAAC;IAC3D,IAAI,KAAK,CAAC;IAEV,OAAO,CAAC,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACrE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,qEAAqE;IACrE,MAAM,cAAc,GAAG,qCAAqC,CAAC;IAE7D,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACtE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,qEAAqE;IACrE,MAAM,cAAc,GAAG,qCAAqC,CAAC;IAE7D,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACtE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AAClC,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,mCAAmC;IACnC,2FAA2F;IAC3F,MAAM,aAAa,GAAG;QACpB,EAAE,IAAI,EAAE,QAAiB,EAAE,KAAK,EAAE,wBAAwB,EAAE,GAAG,EAAE,2BAA2B,EAAE;QAC9F,EAAE,IAAI,EAAE,OAAgB,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,0BAA0B,EAAE;QAC3F,EAAE,IAAI,EAAE,YAAqB,EAAE,KAAK,EAAE,4BAA4B,EAAE,GAAG,EAAE,+BAA+B,EAAE;KAC3G,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;QACpC,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,OAAO,WAAW,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAClC,iBAAiB;YACjB,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAC7C,GAAG,IAAI,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAC/C,CAAC;YAEF,IAAI,QAAQ,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE3B,wDAAwD;YACxD,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAC3C,GAAG,IAAI,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAC1C,CAAC;YAEF,IAAI,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;gBAClB,2CAA2C;gBAC3C,MAAM;YACR,CAAC;YAED,wBAAwB;YACxB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAElE,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,SAAS,EAAE,QAAQ;gBACnB,OAAO,EAAE,MAAM;gBACf,OAAO,EAAE,YAAY;aACtB,CAAC,CAAC;YAEH,WAAW,GAAG,MAAM,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;AAC1D,CAAC;AAgBD;;GAEG;AACH,SAAS,eAAe,CACtB,OAAe,EACf,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAA0B,EAC1B,UAII,EAAE;IAEN,OAAO;QACL,OAAO;QACP,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,SAAS;YACT,OAAO;YACP,QAAQ,EAAE,QAAQ;YAClB,IAAI;YACJ,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;SAC/D;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CACtB,KAAkB,EAClB,GAAiB,EACjB,UAA8B,EAC9B,OAAiB;IAEjB,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IAEzD,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,MAAM,IAAI,SAAS,GAAG,YAAY,EAAE,CAAC;QACpF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAClE,MAAM,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpE,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,CAAC,IAAI,CAAC,eAAe,CACzB,YAAY,EACZ,KAAK,CAAC,SAAS,GAAG,MAAM,GAAG,CAAC,EAC5B,KAAK,CAAC,SAAS,GAAG,SAAS,EAC3B,QAAQ,EACR,OAAO,EACP,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,CAChD,CAAC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,IAAI,UAAU,CAAC,MAAM;YAAE,MAAM;IAC5C,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAC1B,KAAkB,EAClB,GAAiB,EACjB,YAAyB;IAEzB,wBAAwB;IACxB,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IAED,mBAAmB;IACnB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE1F,uCAAuC;IACvC,MAAM,2BAA2B,GAAG,GAAG,CAAC,oBAAoB;SACzD,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;SACzC,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,MAAM,OAAO,GAAG,iBAAiB,CAAC,2BAA2B,CAAC,CAAC;IAE/D,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;IAC3D,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;IAE9C,sDAAsD;IACtD,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;QACnC,OAAO,CAAC,eAAe,CACrB,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,SAAS,GAAG,CAAC,EACnB,KAAK,CAAC,OAAO,GAAG,CAAC,EACjB,GAAG,CAAC,MAAM,CAAC,QAAQ,EACnB,OAAO,EACP,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,CAChD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,eAAe,CAAC,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;AAC1D,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,YAAsB,EACtB,cAAsB,EACtB,OAAe,EACf,GAAiB;IAEjB,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAE3C,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAElD,MAAM,YAAY,GAAG,GAAG,CAAC,oBAAoB,CAAC,KAAK,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAEhD,OAAO,eAAe,CACpB,YAAY,EACZ,cAAc,GAAG,CAAC,EAClB,OAAO,EACP,GAAG,CAAC,MAAM,CAAC,QAAQ,EACnB,UAAU,EACV,EAAE,OAAO,EAAE,CACZ,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAC7B,GAAiB,EACjB,YAAyB;IAEzB,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC;IAC9B,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC;IAE3C,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,uCAAuC;QACvC,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,kBAAkB,CAAC,YAAY,EAAE,cAAc,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YACvE,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,YAAY,GAAG,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QAED,4BAA4B;QAC5B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,cAAc,GAAG,CAAC,CAAC;QACrB,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5B,yBAAyB;QACzB,IAAI,YAAY,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,kBAAkB,CAAC,YAAY,EAAE,cAAc,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;YAC3E,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE9B,6BAA6B;YAC7B,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;YACjD,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,UAAU,GAAG,kBAAkB,CAAC,YAAY,EAAE,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACvF,IAAI,UAAU;QAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAExC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAgB,EAChB,OAAe,EACf,YAAoB,EAAE,EACtB,eAAuB,EAAE;IAEzB,8CAA8C;IAC9C,MAAM,sBAAsB,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACvD,MAAM,GAAG,GAAiB;QACxB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;QAC1B,oBAAoB,EAAE,sBAAsB,CAAC,KAAK,CAAC,IAAI,CAAC;QACxD,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE;KAC9C,CAAC;IAEF,8CAA8C;IAC9C,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IAEvC,yBAAyB;IACzB,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,mBAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,YAAY,CAAC,CAAC,CAAC;IAE3F,qCAAqC;IACrC,MAAM,cAAc,GAAG,sBAAsB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAEjE,kCAAkC;IAClC,OAAO,CAAC,GAAG,WAAW,EAAE,GAAG,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;AACzG,CAAC"}

package/dist/indexer/manifest.d.ts

@@ -0,0 +1,122 @@
+import { GitState } from '../git/tracker.js';
+/**
+ * Represents a single file in the index manifest
+ */
+export interface FileEntry {
+    filepath: string;
+    lastModified: number;
+    chunkCount: number;
+}
+/**
+ * Index manifest tracking all indexed files and version information
+ */
+export interface IndexManifest {
+    formatVersion: number;
+    lienVersion: string;
+    lastIndexed: number;
+    gitState?: GitState;
+    files: Record<string, FileEntry>;
+}
+/**
+ * Manages the index manifest file, tracking which files are indexed
+ * and their metadata for incremental indexing support.
+ *
+ * The manifest includes version checking to invalidate indices when
+ * Lien's indexing format changes (e.g., new chunking algorithm,
+ * different embedding model, schema changes).
+ */
+export declare class ManifestManager {
+    private manifestPath;
+    private indexPath;
+    /**
+     * Promise-based lock to prevent race conditions during concurrent updates.
+     * Ensures read-modify-write operations are atomic.
+     */
+    private updateLock;
+    /**
+     * Creates a new ManifestManager
+     * @param indexPath - Path to the index directory (same as VectorDB path)
+     */
+    constructor(indexPath: string);
+    /**
+     * Loads the manifest from disk.
+     * Returns null if:
+     * - Manifest doesn't exist (first run)
+     * - Manifest is corrupt
+     * - Format version is incompatible (triggers full reindex)
+     *
+     * @returns Loaded manifest or null
+     */
+    load(): Promise<IndexManifest | null>;
+    /**
+     * Saves the manifest to disk.
+     * Always saves with current format and package versions.
+     *
+     * @param manifest - Manifest to save
+     */
+    save(manifest: IndexManifest): Promise<void>;
+    /**
+     * Adds or updates a file entry in the manifest.
+     * Protected by lock to prevent race conditions during concurrent updates.
+     *
+     * @param filepath - Path to the file
+     * @param entry - File entry metadata
+     */
+    updateFile(filepath: string, entry: FileEntry): Promise<void>;
+    /**
+     * Removes a file entry from the manifest.
+     * Protected by lock to prevent race conditions during concurrent updates.
+     *
+     * Note: If the manifest doesn't exist, this is a no-op (not an error).
+     * This can happen legitimately after clearing the index or on fresh installs.
+     *
+     * @param filepath - Path to the file to remove
+     */
+    removeFile(filepath: string): Promise<void>;
+    /**
+     * Updates multiple files at once (more efficient than individual updates).
+     * Protected by lock to prevent race conditions during concurrent updates.
+     *
+     * @param entries - Array of file entries to update
+     */
+    updateFiles(entries: FileEntry[]): Promise<void>;
+    /**
+     * Updates the git state in the manifest.
+     * Protected by lock to prevent race conditions during concurrent updates.
+     *
+     * @param gitState - Current git state
+     */
+    updateGitState(gitState: GitState): Promise<void>;
+    /**
+     * Gets the list of files currently in the manifest
+     *
+     * @returns Array of filepaths
+     */
+    getIndexedFiles(): Promise<string[]>;
+    /**
+     * Detects which files have changed based on mtime comparison
+     *
+     * @param currentFiles - Map of current files with their mtimes
+     * @returns Array of filepaths that have changed
+     */
+    getChangedFiles(currentFiles: Map<string, number>): Promise<string[]>;
+    /**
+     * Gets files that are in the manifest but not in the current file list
+     * (i.e., deleted files)
+     *
+     * @param currentFiles - Set of current file paths
+     * @returns Array of deleted file paths
+     */
+    getDeletedFiles(currentFiles: Set<string>): Promise<string[]>;
+    /**
+     * Clears the manifest file
+     */
+    clear(): Promise<void>;
+    /**
+     * Creates an empty manifest with current version information
+     *
+     * @returns Empty manifest
+     */
+    private createEmpty;
+}
+//# sourceMappingURL=manifest.d.ts.map

package/dist/indexer/manifest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/indexer/manifest.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAK7C;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CAClC;AAED;;;;;;;GAOG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,SAAS,CAAS;IAE1B;;;OAGG;IACH,OAAO,CAAC,UAAU,CAAqB;IAEvC;;;OAGG;gBACS,SAAS,EAAE,MAAM;IAK7B;;;;;;;;OAQG;IACG,IAAI,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA8B3C;;;;;OAKG;IACG,IAAI,CAAC,QAAQ,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBlD;;;;;;OAMG;IACG,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBnE;;;;;;;;OAQG;IACG,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBjD;;;;;OAKG;IACG,WAAW,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBtD;;;;;OAKG;IACG,cAAc,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBvD;;;;OAIG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAO1C;;;;;OAKG;IACG,eAAe,CAAC,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAwB3E;;;;;;OAMG;IACG,eAAe,CAAC,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAenE;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B;;;;OAIG;IACH,OAAO,CAAC,WAAW;CAQpB"}

package/dist/indexer/manifest.js

@@ -0,0 +1,262 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { INDEX_FORMAT_VERSION } from '../constants.js';
+import { getPackageVersion } from '../utils/version.js';
+const MANIFEST_FILE = 'manifest.json';
+/**
+ * Manages the index manifest file, tracking which files are indexed
+ * and their metadata for incremental indexing support.
+ *
+ * The manifest includes version checking to invalidate indices when
+ * Lien's indexing format changes (e.g., new chunking algorithm,
+ * different embedding model, schema changes).
+ */
+export class ManifestManager {
+    manifestPath;
+    indexPath;
+    /**
+     * Promise-based lock to prevent race conditions during concurrent updates.
+     * Ensures read-modify-write operations are atomic.
+     */
+    updateLock = Promise.resolve();
+    /**
+     * Creates a new ManifestManager
+     * @param indexPath - Path to the index directory (same as VectorDB path)
+     */
+    constructor(indexPath) {
+        this.indexPath = indexPath;
+        this.manifestPath = path.join(indexPath, MANIFEST_FILE);
+    }
+    /**
+     * Loads the manifest from disk.
+     * Returns null if:
+     * - Manifest doesn't exist (first run)
+     * - Manifest is corrupt
+     * - Format version is incompatible (triggers full reindex)
+     *
+     * @returns Loaded manifest or null
+     */
+    async load() {
+        try {
+            const content = await fs.readFile(this.manifestPath, 'utf-8');
+            const manifest = JSON.parse(content);
+            // VERSION CHECK: Invalidate if format version doesn't match
+            if (manifest.formatVersion !== INDEX_FORMAT_VERSION) {
+                console.error(`[Lien] Index format v${manifest.formatVersion} is incompatible with current v${INDEX_FORMAT_VERSION}`);
+                console.error(`[Lien] Full reindex required after Lien upgrade`);
+                // Clear old manifest and return null (triggers full reindex)
+                await this.clear();
+                return null;
+            }
+            return manifest;
+        }
+        catch (error) {
+            // File doesn't exist or is invalid - return null for first run
+            if (error.code === 'ENOENT') {
+                return null;
+            }
+            // Corrupt manifest - log warning and return null
+            console.error(`[Lien] Warning: Failed to load manifest: ${error}`);
+            return null;
+        }
+    }
+    /**
+     * Saves the manifest to disk.
+     * Always saves with current format and package versions.
+     *
+     * @param manifest - Manifest to save
+     */
+    async save(manifest) {
+        try {
+            // Ensure index directory exists
+            await fs.mkdir(this.indexPath, { recursive: true });
+            // Always save with current versions
+            const manifestToSave = {
+                ...manifest,
+                formatVersion: INDEX_FORMAT_VERSION,
+                lienVersion: getPackageVersion(),
+                lastIndexed: Date.now(),
+            };
+            const content = JSON.stringify(manifestToSave, null, 2);
+            await fs.writeFile(this.manifestPath, content, 'utf-8');
+        }
+        catch (error) {
+            // Don't throw - manifest is best-effort
+            console.error(`[Lien] Warning: Failed to save manifest: ${error}`);
+        }
+    }
+    /**
+     * Adds or updates a file entry in the manifest.
+     * Protected by lock to prevent race conditions during concurrent updates.
+     *
+     * @param filepath - Path to the file
+     * @param entry - File entry metadata
+     */
+    async updateFile(filepath, entry) {
+        // Chain this operation to the lock to ensure atomicity
+        this.updateLock = this.updateLock.then(async () => {
+            const manifest = await this.load() || this.createEmpty();
+            manifest.files[filepath] = entry;
+            await this.save(manifest);
+        }).catch(error => {
+            console.error(`[Lien] Failed to update manifest for ${filepath}: ${error}`);
+            // Return to reset lock - don't let errors block future operations
+            return undefined;
+        });
+        // Wait for this operation to complete
+        await this.updateLock;
+    }
+    /**
+     * Removes a file entry from the manifest.
+     * Protected by lock to prevent race conditions during concurrent updates.
+     *
+     * Note: If the manifest doesn't exist, this is a no-op (not an error).
+     * This can happen legitimately after clearing the index or on fresh installs.
+     *
+     * @param filepath - Path to the file to remove
+     */
+    async removeFile(filepath) {
+        // Chain this operation to the lock to ensure atomicity
+        this.updateLock = this.updateLock.then(async () => {
+            const manifest = await this.load();
+            if (!manifest) {
+                // No manifest exists - nothing to remove from (expected in some scenarios)
+                return;
+            }
+            delete manifest.files[filepath];
+            await this.save(manifest);
+        }).catch(error => {
+            console.error(`[Lien] Failed to remove manifest entry for ${filepath}: ${error}`);
+            // Return to reset lock - don't let errors block future operations
+            return undefined;
+        });
+        // Wait for this operation to complete
+        await this.updateLock;
+    }
+    /**
+     * Updates multiple files at once (more efficient than individual updates).
+     * Protected by lock to prevent race conditions during concurrent updates.
+     *
+     * @param entries - Array of file entries to update
+     */
+    async updateFiles(entries) {
+        // Chain this operation to the lock to ensure atomicity
+        this.updateLock = this.updateLock.then(async () => {
+            const manifest = await this.load() || this.createEmpty();
+            for (const entry of entries) {
+                manifest.files[entry.filepath] = entry;
+            }
+            await this.save(manifest);
+        }).catch(error => {
+            console.error(`[Lien] Failed to update manifest for ${entries.length} files: ${error}`);
+            // Return to reset lock - don't let errors block future operations
+            return undefined;
+        });
+        // Wait for this operation to complete
+        await this.updateLock;
+    }
+    /**
+     * Updates the git state in the manifest.
+     * Protected by lock to prevent race conditions during concurrent updates.
+     *
+     * @param gitState - Current git state
+     */
+    async updateGitState(gitState) {
+        // Chain this operation to the lock to ensure atomicity
+        this.updateLock = this.updateLock.then(async () => {
+            const manifest = await this.load() || this.createEmpty();
+            manifest.gitState = gitState;
+            await this.save(manifest);
+        }).catch(error => {
+            console.error(`[Lien] Failed to update git state in manifest: ${error}`);
+            // Return to reset lock - don't let errors block future operations
+            return undefined;
+        });
+        // Wait for this operation to complete
+        await this.updateLock;
+    }
+    /**
+     * Gets the list of files currently in the manifest
+     *
+     * @returns Array of filepaths
+     */
+    async getIndexedFiles() {
+        const manifest = await this.load();
+        if (!manifest)
+            return [];
+        return Object.keys(manifest.files);
+    }
+    /**
+     * Detects which files have changed based on mtime comparison
+     *
+     * @param currentFiles - Map of current files with their mtimes
+     * @returns Array of filepaths that have changed
+     */
+    async getChangedFiles(currentFiles) {
+        const manifest = await this.load();
+        if (!manifest) {
+            // No manifest = all files are "changed" (need full index)
+            return Array.from(currentFiles.keys());
+        }
+        const changedFiles = [];
+        for (const [filepath, mtime] of currentFiles) {
+            const entry = manifest.files[filepath];
+            if (!entry) {
+                // New file
+                changedFiles.push(filepath);
+            }
+            else if (entry.lastModified < mtime) {
+                // File modified since last index
+                changedFiles.push(filepath);
+            }
+        }
+        return changedFiles;
+    }
+    /**
+     * Gets files that are in the manifest but not in the current file list
+     * (i.e., deleted files)
+     *
+     * @param currentFiles - Set of current file paths
+     * @returns Array of deleted file paths
+     */
+    async getDeletedFiles(currentFiles) {
+        const manifest = await this.load();
+        if (!manifest)
+            return [];
+        const deletedFiles = [];
+        for (const filepath of Object.keys(manifest.files)) {
+            if (!currentFiles.has(filepath)) {
+                deletedFiles.push(filepath);
+            }
+        }
+        return deletedFiles;
+    }
+    /**
+     * Clears the manifest file
+     */
+    async clear() {
+        try {
+            await fs.unlink(this.manifestPath);
+        }
+        catch (error) {
+            // Ignore error if file doesn't exist
+            if (error.code !== 'ENOENT') {
+                console.error(`[Lien] Warning: Failed to clear manifest: ${error}`);
+            }
+        }
+    }
+    /**
+     * Creates an empty manifest with current version information
+     *
+     * @returns Empty manifest
+     */
+    createEmpty() {
+        return {
+            formatVersion: INDEX_FORMAT_VERSION,
+            lienVersion: getPackageVersion(),
+            lastIndexed: Date.now(),
+            files: {},
+        };
+    }
+}
+//# sourceMappingURL=manifest.js.map