@copilotkit/pathfinder 1.1.0

package/dist/indexing/chunking/code.js

@@ -0,0 +1,277 @@
+// Line-based code splitter
+const DEFAULT_TARGET_LINES = 80;
+const DEFAULT_OVERLAP_LINES = 10;
+/**
+ * Map file extension to language name.
+ */
+function detectLanguage(filePath) {
+    const ext = filePath.split('.').pop()?.toLowerCase() || '';
+    const languageMap = {
+        ts: 'typescript',
+        tsx: 'typescript',
+        js: 'javascript',
+        jsx: 'javascript',
+        mjs: 'javascript',
+        cjs: 'javascript',
+        py: 'python',
+        rb: 'ruby',
+        go: 'go',
+        rs: 'rust',
+        java: 'java',
+        kt: 'kotlin',
+        swift: 'swift',
+        c: 'c',
+        cpp: 'cpp',
+        h: 'c',
+        hpp: 'cpp',
+        cs: 'csharp',
+        md: 'markdown',
+        mdx: 'markdown',
+        json: 'json',
+        yaml: 'yaml',
+        yml: 'yaml',
+        toml: 'toml',
+        sql: 'sql',
+        sh: 'shell',
+        bash: 'shell',
+        zsh: 'shell',
+        css: 'css',
+        scss: 'scss',
+        html: 'html',
+        xml: 'xml',
+    };
+    return languageMap[ext] || ext || 'text';
+}
+/**
+ * Check whether the character at `pos` is escaped by counting preceding
+ * backslashes.  An odd number means the character is escaped.
+ */
+function isEscaped(line, pos) {
+    let backslashes = 0;
+    for (let j = pos - 1; j >= 0 && line[j] === '\\'; j--) {
+        backslashes++;
+    }
+    return backslashes % 2 === 1;
+}
+/**
+ * Strip string literals and single-line comments from a line so that
+ * block-comment and template-string detection only fires on real syntax.
+ */
+function stripStringsAndLineComments(line) {
+    let result = '';
+    let inSingle = false;
+    let inDouble = false;
+    for (let i = 0; i < line.length; i++) {
+        const ch = line[i];
+        if (!inSingle && !inDouble && ch === '/' && line[i + 1] === '/') {
+            break; // rest of line is a single-line comment
+        }
+        if (!inDouble && ch === "'" && !isEscaped(line, i)) {
+            inSingle = !inSingle;
+        }
+        else if (!inSingle && ch === '"' && !isEscaped(line, i)) {
+            inDouble = !inDouble;
+        }
+        if (!inSingle && !inDouble) {
+            result += ch;
+        }
+    }
+    return result;
+}
+function trackBlockState(line, state) {
+    const newState = { ...state };
+    const stripped = stripStringsAndLineComments(line);
+    if (newState.inBlockComment) {
+        if (stripped.includes('*/')) {
+            newState.inBlockComment = false;
+        }
+        return newState;
+    }
+    if (newState.inTemplateString) {
+        // Count unescaped backticks
+        const backticks = (stripped.match(/(?<!\\)`/g) || []).length;
+        if (backticks % 2 === 1) {
+            newState.inTemplateString = false;
+        }
+        return newState;
+    }
+    // Check for block comment start (not on same line as end)
+    if (stripped.includes('/*') && !stripped.includes('*/')) {
+        newState.inBlockComment = true;
+    }
+    else if (!newState.inBlockComment) {
+        // Only check template strings if we didn't just enter a block comment
+        const backticks = (stripped.match(/(?<!\\)`/g) || []).length;
+        if (backticks % 2 === 1) {
+            newState.inTemplateString = true;
+        }
+    }
+    // Python triple-quote strings — run on original line since stripping is JS-oriented
+    if (!newState.inBlockComment && !newState.inTemplateString) {
+        if (line.includes('"""') || line.includes("'''")) {
+            const tripleDouble = (line.match(/"""/g) || []).length;
+            const tripleSingle = (line.match(/'''/g) || []).length;
+            if ((tripleDouble % 2 === 1) || (tripleSingle % 2 === 1)) {
+                newState.inBlockComment = true; // reuse flag for python docstrings
+            }
+        }
+    }
+    return newState;
+}
+/**
+ * Determine safe split points: lines where we're not inside a block comment
+ * or string literal, and that represent logical boundaries.
+ */
+function findSplitPoints(lines) {
+    const safePoints = new Set();
+    let state = { inBlockComment: false, inTemplateString: false };
+    for (let i = 0; i < lines.length; i++) {
+        const prevState = { ...state };
+        state = trackBlockState(lines[i], state);
+        // A double-newline boundary is a safe split point
+        if (i > 0 && lines[i].trim() === '' && lines[i - 1].trim() === '') {
+            if (!state.inBlockComment && !state.inTemplateString &&
+                !prevState.inBlockComment && !prevState.inTemplateString) {
+                safePoints.add(i);
+            }
+        }
+        // A single blank line is a secondary split point
+        if (lines[i].trim() === '' && !state.inBlockComment && !state.inTemplateString) {
+            safePoints.add(i);
+        }
+    }
+    return safePoints;
+}
+/**
+ * Format a range of lines with line numbers and a file breadcrumb.
+ */
+function formatChunk(lines, startLine, filePath) {
+    const breadcrumb = `// File: ${filePath}`;
+    const maxLineNum = startLine + lines.length - 1;
+    const padWidth = String(maxLineNum).length;
+    const numbered = lines.map((line, i) => {
+        const lineNum = String(startLine + i).padStart(padWidth, ' ');
+        return `${lineNum} | ${line}`;
+    });
+    return breadcrumb + '\n' + numbered.join('\n');
+}
+/**
+ * Split lines into groups at double-newline boundaries, respecting block state.
+ */
+function splitAtBoundaries(lines, targetLines) {
+    if (lines.length <= targetLines) {
+        return [{ start: 0, end: lines.length - 1 }];
+    }
+    const safePoints = findSplitPoints(lines);
+    const ranges = [];
+    let rangeStart = 0;
+    // Prefer double-newline boundaries first
+    const doubleNewlines = [];
+    for (let i = 1; i < lines.length; i++) {
+        if (lines[i].trim() === '' && lines[i - 1].trim() === '' && safePoints.has(i)) {
+            doubleNewlines.push(i);
+        }
+    }
+    // Try splitting on double-newline boundaries
+    if (doubleNewlines.length > 0) {
+        const splitPoints = selectSplitPoints(doubleNewlines, lines.length, targetLines);
+        for (const point of splitPoints) {
+            if (point > rangeStart) {
+                ranges.push({ start: rangeStart, end: point - 1 });
+                rangeStart = point;
+            }
+        }
+        ranges.push({ start: rangeStart, end: lines.length - 1 });
+        // Check if any range is still too large
+        const needsRefinement = ranges.some(r => (r.end - r.start + 1) > targetLines * 1.5);
+        if (!needsRefinement)
+            return ranges;
+    }
+    // Fall back to single blank line boundaries
+    const blankLines = Array.from(safePoints).sort((a, b) => a - b);
+    if (blankLines.length > 0) {
+        const refinedRanges = [];
+        rangeStart = 0;
+        const splitPoints = selectSplitPoints(blankLines, lines.length, targetLines);
+        for (const point of splitPoints) {
+            if (point > rangeStart) {
+                refinedRanges.push({ start: rangeStart, end: point - 1 });
+                rangeStart = point;
+            }
+        }
+        refinedRanges.push({ start: rangeStart, end: lines.length - 1 });
+        return refinedRanges;
+    }
+    // No good split points; split mechanically on line boundaries
+    const mechanicalRanges = [];
+    for (let i = 0; i < lines.length; i += targetLines) {
+        mechanicalRanges.push({
+            start: i,
+            end: Math.min(i + targetLines - 1, lines.length - 1),
+        });
+    }
+    return mechanicalRanges;
+}
+/**
+ * Select split points from candidates that best partition the content
+ * into chunks near the target size.
+ */
+function selectSplitPoints(candidates, _totalLines, targetLines) {
+    const selected = [];
+    let lastSplit = 0;
+    for (const candidate of candidates) {
+        const distance = candidate - lastSplit;
+        if (distance >= targetLines) {
+            selected.push(candidate);
+            lastSplit = candidate;
+        }
+    }
+    return selected;
+}
+/**
+ * Split code content into embedding-friendly chunks with line numbers.
+ *
+ * @param content - The full source file content
+ * @param filePath - Path to the source file
+ * @returns Array of CodeChunk objects
+ */
+export function chunkCode(content, filePath, config) {
+    if (!content || !content.trim()) {
+        return [];
+    }
+    const targetLines = config.chunk?.target_lines ?? DEFAULT_TARGET_LINES;
+    const overlapLines = config.chunk?.overlap_lines ?? DEFAULT_OVERLAP_LINES;
+    const language = detectLanguage(filePath);
+    const lines = content.split('\n');
+    // Remove trailing empty line if file ends with newline
+    if (lines.length > 0 && lines[lines.length - 1] === '') {
+        lines.pop();
+    }
+    if (lines.length === 0) {
+        return [];
+    }
+    // Split into ranges
+    const ranges = splitAtBoundaries(lines, targetLines);
+    // Apply overlap and build chunks
+    const chunks = [];
+    for (let i = 0; i < ranges.length; i++) {
+        let { start, end } = ranges[i];
+        // Apply overlap from previous chunk
+        if (i > 0 && overlapLines > 0) {
+            const overlapStart = Math.max(ranges[i - 1].end - overlapLines + 1, ranges[i - 1].start);
+            start = Math.min(start, overlapStart);
+        }
+        const chunkLines = lines.slice(start, end + 1);
+        const startLine = start + 1; // 1-indexed
+        const endLine = end + 1; // 1-indexed
+        chunks.push({
+            content: formatChunk(chunkLines, startLine, filePath),
+            startLine,
+            endLine,
+            language,
+            chunkIndex: chunks.length,
+        });
+    }
+    return chunks;
+}
+//# sourceMappingURL=code.js.map

package/dist/indexing/chunking/code.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"code.js","sourceRoot":"","sources":["../../../src/indexing/chunking/code.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAY3B,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAChC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAEjC;;GAEG;AACH,SAAS,cAAc,CAAC,QAAgB;IACpC,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC3D,MAAM,WAAW,GAA2B;QACxC,EAAE,EAAE,YAAY;QAChB,GAAG,EAAE,YAAY;QACjB,EAAE,EAAE,YAAY;QAChB,GAAG,EAAE,YAAY;QACjB,GAAG,EAAE,YAAY;QACjB,GAAG,EAAE,YAAY;QACjB,EAAE,EAAE,QAAQ;QACZ,EAAE,EAAE,MAAM;QACV,EAAE,EAAE,IAAI;QACR,EAAE,EAAE,MAAM;QACV,IAAI,EAAE,MAAM;QACZ,EAAE,EAAE,QAAQ;QACZ,KAAK,EAAE,OAAO;QACd,CAAC,EAAE,GAAG;QACN,GAAG,EAAE,KAAK;QACV,CAAC,EAAE,GAAG;QACN,GAAG,EAAE,KAAK;QACV,EAAE,EAAE,QAAQ;QACZ,EAAE,EAAE,UAAU;QACd,GAAG,EAAE,UAAU;QACf,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,MAAM;QACZ,GAAG,EAAE,MAAM;QACX,IAAI,EAAE,MAAM;QACZ,GAAG,EAAE,KAAK;QACV,EAAE,EAAE,OAAO;QACX,IAAI,EAAE,OAAO;QACb,GAAG,EAAE,OAAO;QACZ,GAAG,EAAE,KAAK;QACV,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,MAAM;QACZ,GAAG,EAAE,KAAK;KACb,CAAC;IAEF,OAAO,WAAW,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,MAAM,CAAC;AAC7C,CAAC;AAWD;;;GAGG;AACH,SAAS,SAAS,CAAC,IAAY,EAAE,GAAW;IACxC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,WAAW,EAAE,CAAC;IAClB,CAAC;IACD,OAAO,WAAW,GAAG,CAAC,KAAK,CAAC,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,SAAS,2BAA2B,CAAC,IAAY;IAC7C,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAEnB,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,IAAI,EAAE,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC9D,MAAM,CAAC,wCAAwC;QACnD,CAAC;QAED,IAAI,CAAC,QAAQ,IAAI,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;YACjD,QAAQ,GAAG,CAAC,QAAQ,CAAC;QACzB,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;YACxD,QAAQ,GAAG,CAAC,QAAQ,CAAC;QACzB,CAAC;QAED,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzB,MAAM,IAAI,EAAE,CAAC;QACjB,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,eAAe,CAAC,IAAY,EAAE,KAAiB;IACpD,MAAM,QAAQ,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAC;IAEnD,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;QAC1B,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,cAAc,GAAG,KAAK,CAAC;QACpC,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;QAC5B,4BAA4B;QAC5B,MAAM,SAAS,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAC7D,IAAI,SAAS,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,QAAQ,CAAC,gBAAgB,GAAG,KAAK,CAAC;QACtC,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,0DAA0D;IAC1D,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACtD,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC;IACnC,CAAC;SAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;QAClC,sEAAsE;QACtE,MAAM,SAAS,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAC7D,IAAI,SAAS,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,QAAQ,CAAC,gBAAgB,GAAG,IAAI,CAAC;QACrC,CAAC;IACL,CAAC;IAED,oFAAoF;IACpF,IAAI,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;QACzD,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/C,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YACvD,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YACvD,IAAI,CAAC,YAAY,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACvD,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC,mCAAmC;YACvE,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,KAAe;IACpC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,IAAI,KAAK,GAAe,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC;IAE3E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAC/B,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAEzC,kDAAkD;QAClD,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAChE,IAAI,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,gBAAgB;gBAChD,CAAC,SAAS,CAAC,cAAc,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,CAAC;gBAC3D,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACL,CAAC;QAED,iDAAiD;QACjD,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC7E,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACL,CAAC;IAED,OAAO,UAAU,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,KAAe,EAAE,SAAiB,EAAE,QAAgB;IACrE,MAAM,UAAU,GAAG,YAAY,QAAQ,EAAE,CAAC;IAC1C,MAAM,UAAU,GAAG,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;IAE3C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACnC,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC9D,OAAO,GAAG,OAAO,MAAM,IAAI,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,OAAO,UAAU,GAAG,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,KAAe,EAAE,WAAmB;IAC3D,IAAI,KAAK,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;QAC9B,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAC1C,MAAM,MAAM,GAA0C,EAAE,CAAC;IACzD,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,yCAAyC;IACzC,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,CAAC;IACL,CAAC;IAED,6CAA6C;IAC7C,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,iBAAiB,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACjF,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YAC9B,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACrB,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;gBACnD,UAAU,GAAG,KAAK,CAAC;YACvB,CAAC;QACL,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC;QAE1D,wCAAwC;QACxC,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,WAAW,GAAG,GAAG,CAAC,CAAC;QACpF,IAAI,CAAC,eAAe;YAAE,OAAO,MAAM,CAAC;IACxC,CAAC;IAED,4CAA4C;IAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,aAAa,GAA0C,EAAE,CAAC;QAChE,UAAU,GAAG,CAAC,CAAC;QAEf,MAAM,WAAW,GAAG,iBAAiB,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAC7E,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YAC9B,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACrB,aAAa,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC1D,UAAU,GAAG,KAAK,CAAC;YACvB,CAAC;QACL,CAAC;QACD,aAAa,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC;QACjE,OAAO,aAAa,CAAC;IACzB,CAAC;IAED,8DAA8D;IAC9D,MAAM,gBAAgB,GAA0C,EAAE,CAAC;IACnE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACjD,gBAAgB,CAAC,IAAI,CAAC;YAClB,KAAK,EAAE,CAAC;YACR,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,WAAW,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;SACvD,CAAC,CAAC;IACP,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,UAAoB,EAAE,WAAmB,EAAE,WAAmB;IACrF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;QACvC,IAAI,QAAQ,IAAI,WAAW,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzB,SAAS,GAAG,SAAS,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,OAAe,EAAE,QAAgB,EAAE,MAAoB;IAC7E,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,EAAE,YAAY,IAAI,oBAAoB,CAAC;IACvE,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,EAAE,aAAa,IAAI,qBAAqB,CAAC;IAE1E,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAElC,uDAAuD;IACvD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;QACrD,KAAK,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC;IACd,CAAC;IAED,oBAAoB;IACpB,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAErD,iCAAiC;IACjC,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE/B,oCAAoC;QACpC,IAAI,CAAC,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,YAAY,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACzF,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,YAAY;QACzC,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,CAAC,CAAK,YAAY;QAEzC,MAAM,CAAC,IAAI,CAAC;YACR,OAAO,EAAE,WAAW,CAAC,UAAU,EAAE,SAAS,EAAE,QAAQ,CAAC;YACrD,SAAS;YACT,OAAO;YACP,QAAQ;YACR,UAAU,EAAE,MAAM,CAAC,MAAM;SAC5B,CAAC,CAAC;IACP,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/dist/indexing/chunking/index.d.ts

@@ -0,0 +1,6 @@
+import { type ChunkOutput, type SourceConfig } from '../../types.js';
+type ChunkerFn = (content: string, filePath: string, config: SourceConfig) => ChunkOutput[];
+export declare function registerChunker(type: string, fn: ChunkerFn): void;
+export declare function getChunker(type: string): ChunkerFn;
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/indexing/chunking/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/indexing/chunking/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAErE,KAAK,SAAS,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,KAAK,WAAW,EAAE,CAAC;AAI5F,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,GAAG,IAAI,CAEjE;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAIlD"}

package/dist/indexing/chunking/index.js

@@ -0,0 +1,19 @@
+// Chunker registry — maps source.type to a chunking function.
+const registry = new Map();
+export function registerChunker(type, fn) {
+    registry.set(type, fn);
+}
+export function getChunker(type) {
+    const fn = registry.get(type);
+    if (!fn)
+        throw new Error(`Unknown chunker type: "${type}". Available: ${[...registry.keys()].join(', ')}`);
+    return fn;
+}
+// Register built-ins on import
+import { chunkMarkdown } from './markdown.js';
+import { chunkCode } from './code.js';
+import { chunkRawText } from './raw-text.js';
+registerChunker('markdown', chunkMarkdown);
+registerChunker('code', chunkCode);
+registerChunker('raw-text', chunkRawText);
+//# sourceMappingURL=index.js.map

package/dist/indexing/chunking/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/indexing/chunking/index.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAM9D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;AAE9C,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,EAAa;IACvD,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACnC,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,iBAAiB,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3G,OAAO,EAAE,CAAC;AACd,CAAC;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,eAAe,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAC3C,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AACnC,eAAe,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC"}

package/dist/indexing/chunking/markdown.d.ts

@@ -0,0 +1,16 @@
+import { type ChunkOutput, type SourceConfig } from '../../types.js';
+export interface MarkdownChunk {
+    content: string;
+    title: string;
+    headingPath: string[];
+    chunkIndex: number;
+}
+/**
+ * Split markdown/MDX content into embedding-friendly chunks.
+ *
+ * @param content - The full markdown/MDX file content
+ * @param filePath - Path to the source file (used for metadata)
+ * @returns Array of MarkdownChunk objects
+ */
+export declare function chunkMarkdown(content: string, filePath: string, config: SourceConfig): ChunkOutput[];
+//# sourceMappingURL=markdown.d.ts.map

package/dist/indexing/chunking/markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../../src/indexing/chunking/markdown.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAErE,MAAM,WAAW,aAAa;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACtB;AAsRD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,WAAW,EAAE,CAuDpG"}

package/dist/indexing/chunking/markdown.js

@@ -0,0 +1,283 @@
+// Recursive markdown/MDX splitter
+const DEFAULT_TARGET_TOKENS = 600;
+const DEFAULT_OVERLAP_TOKENS = 50;
+/**
+ * Parse YAML frontmatter from markdown content.
+ * Returns the title (if found) and the content with frontmatter stripped.
+ */
+function parseFrontmatter(content) {
+    const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
+    if (!match)
+        return { title: null, body: content };
+    const frontmatter = match[1];
+    const body = content.slice(match[0].length);
+    const titleMatch = frontmatter.match(/^title:\s*["']?(.+?)["']?\s*$/m);
+    return {
+        title: titleMatch ? titleMatch[1].trim() : null,
+        body,
+    };
+}
+/**
+ * Extract the first heading from content to use as fallback title.
+ */
+function extractFirstHeading(content) {
+    const match = content.match(/^#{1,6}\s+(.+)$/m);
+    return match ? match[1].trim() : null;
+}
+/**
+ * Strip MDX-specific syntax: import statements and JSX component tags.
+ * Preserves text content inside JSX tags.
+ */
+function stripMdx(content) {
+    // Strip import statements (single and multi-line)
+    let result = content.replace(/^import\s+[\s\S]*?from\s+['"][^'"]+['"];?\s*$/gm, '');
+    // Strip self-closing JSX tags: <Component ... />
+    result = result.replace(/<[A-Z][A-Za-z0-9]*(?:\s+[^>]*)?\s*\/>/g, '');
+    // Strip JSX component open/close tags but keep inner content
+    // Handles nested tags by repeatedly stripping innermost pairs
+    let prev = '';
+    while (prev !== result) {
+        prev = result;
+        result = result.replace(/<([A-Z][A-Za-z0-9]*)(?:\s+[^>]*)?>([^]*?)<\/\1>/g, '$2');
+    }
+    // Clean up excessive blank lines left by stripping
+    result = result.replace(/\n{3,}/g, '\n\n');
+    return result.trim();
+}
+function segmentCodeBlocks(content) {
+    const segments = [];
+    const codeBlockRegex = /^(`{3,})[^\n]*\n(?:[\s\S]*?\n)?\1\s*$/gm;
+    let lastIndex = 0;
+    let match;
+    while ((match = codeBlockRegex.exec(content)) !== null) {
+        if (match.index > lastIndex) {
+            segments.push({ text: content.slice(lastIndex, match.index), isCodeBlock: false });
+        }
+        segments.push({ text: match[0], isCodeBlock: true });
+        lastIndex = match.index + match[0].length;
+    }
+    if (lastIndex < content.length) {
+        segments.push({ text: content.slice(lastIndex), isCodeBlock: false });
+    }
+    return segments;
+}
+/**
+ * Split text on a delimiter, but never split inside code blocks.
+ */
+function splitPreservingCodeBlocks(content, delimiter) {
+    const segments = segmentCodeBlocks(content);
+    const parts = [];
+    let current = '';
+    for (const segment of segments) {
+        if (segment.isCodeBlock) {
+            current += segment.text;
+        }
+        else {
+            const subParts = typeof delimiter === 'string'
+                ? segment.text.split(delimiter)
+                : segment.text.split(delimiter);
+            if (subParts.length === 1) {
+                current += subParts[0];
+            }
+            else {
+                // First sub-part continues the current accumulator
+                current += subParts[0];
+                for (let i = 1; i < subParts.length; i++) {
+                    parts.push(current);
+                    // Re-attach the delimiter for heading-based splits
+                    if (typeof delimiter === 'string' && delimiter.startsWith('#')) {
+                        current = delimiter + subParts[i];
+                    }
+                    else {
+                        current = subParts[i];
+                    }
+                }
+            }
+        }
+    }
+    if (current) {
+        parts.push(current);
+    }
+    return parts.filter(p => p.trim().length > 0);
+}
+/**
+ * Track heading hierarchy up to a given position in the original content.
+ */
+function getHeadingPathAtPosition(fullContent, position) {
+    const contentBefore = fullContent.slice(0, position);
+    const headingRegex = /^(#{1,6})\s+(.+)$/gm;
+    const headings = [];
+    let match;
+    while ((match = headingRegex.exec(contentBefore)) !== null) {
+        const level = match[1].length;
+        const text = match[2].trim();
+        // Remove headings at same or deeper level (new section at this level)
+        while (headings.length > 0 && headings[headings.length - 1].level >= level) {
+            headings.pop();
+        }
+        headings.push({ level, text });
+    }
+    return headings.map(h => h.text);
+}
+/**
+ * Split text on heading boundaries at a specific level.
+ * Re-attaches the heading marker to each section.
+ */
+function splitOnHeading(content, level) {
+    const prefix = '#'.repeat(level) + ' ';
+    const regex = new RegExp(`(?=^${prefix.replace(/ $/, ' ')})`, 'gm');
+    const segments = segmentCodeBlocks(content);
+    const parts = [];
+    let current = '';
+    for (const segment of segments) {
+        if (segment.isCodeBlock) {
+            current += segment.text;
+        }
+        else {
+            const subParts = segment.text.split(regex);
+            if (subParts.length === 1) {
+                current += subParts[0];
+            }
+            else {
+                current += subParts[0];
+                for (let i = 1; i < subParts.length; i++) {
+                    if (current.trim())
+                        parts.push(current);
+                    current = subParts[i];
+                }
+            }
+        }
+    }
+    if (current.trim())
+        parts.push(current);
+    return parts;
+}
+/**
+ * Recursively split content to fit within target chunk size.
+ * Priority: h2 -> h3 -> paragraph -> line
+ */
+function recursiveSplit(content, targetChars, depth = 0) {
+    if (content.length <= targetChars) {
+        return [content];
+    }
+    let parts;
+    if (depth === 0) {
+        // Split on ## headings
+        parts = splitOnHeading(content, 2);
+        if (parts.length > 1) {
+            return parts.flatMap(p => recursiveSplit(p, targetChars, 1));
+        }
+    }
+    if (depth <= 1) {
+        // Split on ### headings
+        parts = splitOnHeading(content, 3);
+        if (parts.length > 1) {
+            return parts.flatMap(p => recursiveSplit(p, targetChars, 2));
+        }
+    }
+    if (depth <= 2) {
+        // Split on paragraph boundaries
+        parts = splitPreservingCodeBlocks(content, /\n\n+/);
+        if (parts.length > 1) {
+            return mergeSmallParts(parts, targetChars).flatMap(p => recursiveSplit(p, targetChars, 3));
+        }
+    }
+    // Split on line boundaries
+    const lines = content.split('\n');
+    if (lines.length > 1) {
+        return mergeSmallParts(lines, targetChars);
+    }
+    // Content is a single very long line; return as-is
+    return [content];
+}
+/**
+ * Merge adjacent small parts until they approach the target size.
+ */
+function mergeSmallParts(parts, targetSize) {
+    const merged = [];
+    let current = '';
+    for (const part of parts) {
+        const separator = current && !current.endsWith('\n') ? '\n\n' : '';
+        if (current && (current.length + separator.length + part.length) > targetSize) {
+            merged.push(current);
+            current = part;
+        }
+        else {
+            current = current ? current + separator + part : part;
+        }
+    }
+    if (current.trim()) {
+        merged.push(current);
+    }
+    return merged;
+}
+/**
+ * Apply overlap between consecutive chunks.
+ */
+function applyOverlap(chunks, overlapChars) {
+    if (chunks.length <= 1 || overlapChars <= 0)
+        return chunks;
+    const result = [chunks[0]];
+    for (let i = 1; i < chunks.length; i++) {
+        const prevChunk = chunks[i - 1];
+        const overlapText = prevChunk.slice(-overlapChars);
+        // Find a clean break point (newline or space) in the overlap
+        const breakPoint = overlapText.lastIndexOf('\n');
+        const cleanOverlap = breakPoint > 0 ? overlapText.slice(breakPoint) : overlapText;
+        result.push(cleanOverlap + chunks[i]);
+    }
+    return result;
+}
+/**
+ * Split markdown/MDX content into embedding-friendly chunks.
+ *
+ * @param content - The full markdown/MDX file content
+ * @param filePath - Path to the source file (used for metadata)
+ * @returns Array of MarkdownChunk objects
+ */
+export function chunkMarkdown(content, filePath, config) {
+    if (!content || !content.trim()) {
+        return [];
+    }
+    const targetChars = (config.chunk?.target_tokens ?? DEFAULT_TARGET_TOKENS) * 4;
+    const overlapChars = (config.chunk?.overlap_tokens ?? DEFAULT_OVERLAP_TOKENS) * 4;
+    // Parse frontmatter
+    const { title: fmTitle, body } = parseFrontmatter(content);
+    // Strip MDX syntax
+    const cleanBody = stripMdx(body);
+    if (!cleanBody.trim()) {
+        return [];
+    }
+    // Determine title
+    const title = fmTitle || extractFirstHeading(cleanBody) || filePath.split('/').pop() || filePath;
+    // Recursively split the content
+    const rawChunks = recursiveSplit(cleanBody, targetChars);
+    // Apply overlap
+    const overlappedChunks = applyOverlap(rawChunks, overlapChars);
+    // Build heading paths by finding where each raw chunk starts in the original
+    const chunks = [];
+    let searchFrom = 0;
+    for (let i = 0; i < overlappedChunks.length; i++) {
+        const chunkText = overlappedChunks[i].trim();
+        if (!chunkText)
+            continue;
+        // Find the position of this chunk's primary content in the clean body
+        // Use the raw (non-overlapped) chunk to find position
+        const rawText = rawChunks[i]?.trim() || chunkText;
+        const pos = cleanBody.indexOf(rawText, searchFrom);
+        const headingPath = pos >= 0
+            ? getHeadingPathAtPosition(cleanBody, pos)
+            : [];
+        if (pos >= 0) {
+            searchFrom = pos;
+        }
+        chunks.push({
+            content: chunkText,
+            title,
+            headingPath,
+            chunkIndex: chunks.length,
+        });
+    }
+    return chunks;
+}
+//# sourceMappingURL=markdown.js.map

package/dist/indexing/chunking/markdown.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../../src/indexing/chunking/markdown.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAWlC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAElC;;;GAGG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACjE,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAElD,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAE5C,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACvE,OAAO;QACH,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;QAC/C,IAAI;KACP,CAAC;AACN,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,OAAe;IACxC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1C,CAAC;AAED;;;GAGG;AACH,SAAS,QAAQ,CAAC,OAAe;IAC7B,kDAAkD;IAClD,IAAI,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,iDAAiD,EAAE,EAAE,CAAC,CAAC;IAEpF,iDAAiD;IACjD,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,wCAAwC,EAAE,EAAE,CAAC,CAAC;IAEtE,6DAA6D;IAC7D,8DAA8D;IAC9D,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,OAAO,IAAI,KAAK,MAAM,EAAE,CAAC;QACrB,IAAI,GAAG,MAAM,CAAC;QACd,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,kDAAkD,EAAE,IAAI,CAAC,CAAC;IACtF,CAAC;IAED,mDAAmD;IACnD,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE3C,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAWD,SAAS,iBAAiB,CAAC,OAAe;IACtC,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,MAAM,cAAc,GAAG,yCAAyC,CAAC;IAEjE,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,KAA6B,CAAC;IAElC,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACrD,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QACvF,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,SAAS,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC9C,CAAC;IAED,IAAI,SAAS,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAAC,OAAe,EAAE,SAA0B;IAC1E,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;QAC5B,CAAC;aAAM,CAAC;YACJ,MAAM,QAAQ,GAAG,OAAO,SAAS,KAAK,QAAQ;gBAC1C,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;gBAC/B,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAEpC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACJ,mDAAmD;gBACnD,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACpB,mDAAmD;oBACnD,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC7D,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;oBACtC,CAAC;yBAAM,CAAC;wBACJ,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;oBAC1B,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IACD,IAAI,OAAO,EAAE,CAAC;QACV,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAClD,CAAC;AAQD;;GAEG;AACH,SAAS,wBAAwB,CAAC,WAAmB,EAAE,QAAgB;IACnE,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrD,MAAM,YAAY,GAAG,qBAAqB,CAAC;IAC3C,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,IAAI,KAA6B,CAAC;IAElC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7B,sEAAsE;QACtE,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;YACzE,QAAQ,CAAC,GAAG,EAAE,CAAC;QACnB,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACrC,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,OAAe,EAAE,KAAa;IAClD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;IACvC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAEpE,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;QAC5B,CAAC;aAAM,CAAC;YACJ,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACJ,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvC,IAAI,OAAO,CAAC,IAAI,EAAE;wBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACxC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC1B,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAExC,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,OAAe,EAAE,WAAmB,EAAE,QAAgB,CAAC;IAC3E,IAAI,OAAO,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;QAChC,OAAO,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,IAAI,KAAe,CAAC;IAEpB,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QACd,uBAAuB;QACvB,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC;IACL,CAAC;IAED,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QACb,wBAAwB;QACxB,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC;IACL,CAAC;IAED,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QACb,gCAAgC;QAChC,KAAK,GAAG,yBAAyB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACpD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,OAAO,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/F,CAAC;IACL,CAAC;IAED,2BAA2B;IAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAC/C,CAAC;IAED,mDAAmD;IACnD,OAAO,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,KAAe,EAAE,UAAkB;IACxD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,UAAU,EAAE,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,GAAG,IAAI,CAAC;QACnB,CAAC;aAAM,CAAC;YACJ,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1D,CAAC;IACL,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,MAAgB,EAAE,YAAoB;IACxD,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,IAAI,YAAY,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAE3D,MAAM,MAAM,GAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAChC,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;QAEnD,6DAA6D;QAC7D,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;QAElF,MAAM,CAAC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe,EAAE,QAAgB,EAAE,MAAoB;IACjF,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,aAAa,IAAI,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC/E,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,cAAc,IAAI,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAElF,oBAAoB;IACpB,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE3D,mBAAmB;IACnB,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEjC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC;IACd,CAAC;IAED,kBAAkB;IAClB,MAAM,KAAK,GAAG,OAAO,IAAI,mBAAmB,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC;IAEjG,gCAAgC;IAChC,MAAM,SAAS,GAAG,cAAc,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAEzD,gBAAgB;IAChB,MAAM,gBAAgB,GAAG,YAAY,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAE/D,6EAA6E;IAC7E,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,SAAS;YAAE,SAAS;QAEzB,sEAAsE;QACtE,sDAAsD;QACtD,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;QAClD,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACnD,MAAM,WAAW,GAAG,GAAG,IAAI,CAAC;YACxB,CAAC,CAAC,wBAAwB,CAAC,SAAS,EAAE,GAAG,CAAC;YAC1C,CAAC,CAAC,EAAE,CAAC;QACT,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;YACX,UAAU,GAAG,GAAG,CAAC;QACrB,CAAC;QAED,MAAM,CAAC,IAAI,CAAC;YACR,OAAO,EAAE,SAAS;YAClB,KAAK;YACL,WAAW;YACX,UAAU,EAAE,MAAM,CAAC,MAAM;SAC5B,CAAC,CAAC;IACP,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/dist/indexing/chunking/raw-text.d.ts

@@ -0,0 +1,11 @@
+import { type ChunkOutput, type SourceConfig } from '../../types.js';
+/**
+ * Split plain text into embedding-friendly chunks on paragraph boundaries.
+ *
+ * @param content - The full file content
+ * @param _filePath - Path to the source file (unused, kept for registry signature)
+ * @param config - Source configuration with chunk size parameters
+ * @returns Array of ChunkOutput objects
+ */
+export declare function chunkRawText(content: string, _filePath: string, config: SourceConfig): ChunkOutput[];
+//# sourceMappingURL=raw-text.d.ts.map

package/dist/indexing/chunking/raw-text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"raw-text.d.ts","sourceRoot":"","sources":["../../../src/indexing/chunking/raw-text.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAKrE;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,WAAW,EAAE,CA0DpG"}