@copilotkit/pathfinder 1.1.0 → 1.5.0

package/dist/indexing/chunking/html.js

@@ -0,0 +1,356 @@
+// HTML chunker — extracts semantic text content from HTML documents
+// using cheerio for DOM traversal, splits on heading boundaries.
+import * as cheerio from 'cheerio';
+const DEFAULT_TARGET_TOKENS = 600;
+const DEFAULT_OVERLAP_TOKENS = 50;
+/** Elements to remove entirely before content extraction. */
+const STRIP_SELECTORS = 'script, style, nav, footer, header, svg, noscript';
+/** Selectors to try for the main content container, in priority order. */
+const CONTENT_SELECTORS = ['main', 'article', '[role="main"]', '.content', '#content'];
+/**
+ * Extract text from a cheerio element, converting block elements to newlines
+ * and preserving code blocks.
+ */
+function extractText($, el) {
+    const lines = [];
+    el.contents().each((_, node) => {
+        if (node.type === 'text') {
+            const text = node.data?.trim();
+            if (text)
+                lines.push(text);
+            return;
+        }
+        if (node.type !== 'tag')
+            return;
+        const tagNode = node;
+        const tag = tagNode.tagName?.toLowerCase();
+        const child = $(tagNode);
+        if (tag === 'pre') {
+            // Preserve code blocks with whitespace intact
+            lines.push('\n```\n' + child.text() + '\n```\n');
+        }
+        else if (tag === 'ul') {
+            child.children('li').each((_, li) => {
+                lines.push('- ' + extractText($, $(li)));
+            });
+        }
+        else if (tag === 'ol') {
+            child.children('li').each((i, li) => {
+                lines.push(`${i + 1}. ` + extractText($, $(li)));
+            });
+        }
+        else if (tag === 'table') {
+            child.find('tr').each((_, tr) => {
+                const cells = [];
+                $(tr).find('th, td').each((_, cell) => {
+                    cells.push($(cell).text().trim());
+                });
+                if (cells.length > 0)
+                    lines.push(cells.join(' | '));
+            });
+        }
+        else if (tag === 'img') {
+            const alt = child.attr('alt');
+            if (alt)
+                lines.push(`[image: ${alt}]`);
+        }
+        else if (['p', 'div', 'blockquote', 'dd', 'section', 'figcaption'].includes(tag)) {
+            const text = extractText($, child);
+            if (text)
+                lines.push(text);
+        }
+        else if (['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(tag)) {
+            // Headings are split by the section splitter at the top level; here we just extract text content
+            lines.push(child.text().trim());
+        }
+        else {
+            // Recurse into other elements (spans, links, etc.)
+            const text = extractText($, child);
+            if (text)
+                lines.push(text);
+        }
+    });
+    return lines.join('\n').replace(/\n{3,}/g, '\n\n').trim();
+}
+function splitOnHeadings($, container) {
+    const sections = [];
+    let currentHeading = null;
+    let currentLevel = 0;
+    let currentContent = [];
+    function flush() {
+        const text = currentContent.join('\n').replace(/\n{3,}/g, '\n\n').trim();
+        if (text) {
+            sections.push({ heading: currentHeading, level: currentLevel, content: text });
+        }
+        currentContent = [];
+    }
+    // Walk all descendant nodes in document order, splitting on h1/h2/h3
+    function walk(el) {
+        el.contents().each((_, node) => {
+            if (node.type === 'text') {
+                const text = node.data?.trim();
+                if (text)
+                    currentContent.push(text);
+                return;
+            }
+            if (node.type !== 'tag')
+                return;
+            const tagNode = node;
+            const tag = tagNode.tagName?.toLowerCase();
+            if (tag && /^h[123]$/.test(tag)) {
+                flush();
+                currentHeading = $(tagNode).text().trim();
+                currentLevel = parseInt(tag[1]);
+                return; // Don't recurse into the heading
+            }
+            // h4-h6 are not section boundaries but should stand out in text
+            if (tag && /^h[456]$/.test(tag)) {
+                const text = $(tagNode).text().trim();
+                if (text)
+                    currentContent.push('\n' + text);
+                return;
+            }
+            // Block elements with special formatting — keep in sync with extractText
+            if (tag === 'pre') {
+                currentContent.push('\n```\n' + $(tagNode).text() + '\n```\n');
+                return;
+            }
+            if (tag === 'ul') {
+                $(tagNode).children('li').each((_, li) => {
+                    currentContent.push('- ' + extractText($, $(li)));
+                });
+                return;
+            }
+            if (tag === 'ol') {
+                $(tagNode).children('li').each((i, li) => {
+                    currentContent.push(`${i + 1}. ` + extractText($, $(li)));
+                });
+                return;
+            }
+            if (tag === 'table') {
+                $(tagNode).find('tr').each((_, tr) => {
+                    const cells = [];
+                    $(tr).find('th, td').each((_, cell) => {
+                        cells.push($(cell).text().trim());
+                    });
+                    if (cells.length > 0)
+                        currentContent.push(cells.join(' | '));
+                });
+                return;
+            }
+            if (tag === 'img') {
+                const alt = $(tagNode).attr('alt');
+                if (alt)
+                    currentContent.push(`[image: ${alt}]`);
+                return;
+            }
+            // Other block content elements — delegate to extractText
+            if (['p', 'blockquote', 'dd', 'figcaption', 'dl', 'figure'].includes(tag)) {
+                const text = extractText($, $(tagNode));
+                if (text)
+                    currentContent.push(text);
+                return;
+            }
+            // Recurse into container elements (section, div, article, etc.)
+            walk($(tagNode));
+        });
+    }
+    walk(container);
+    flush();
+    return sections;
+}
+/**
+ * Build heading path from sections up to a given index.
+ * Tracks h1 > h2 > h3 hierarchy (same concept as markdown chunker).
+ */
+function buildHeadingPath(sections, upToIndex) {
+    const stack = [];
+    for (let i = 0; i <= upToIndex; i++) {
+        const section = sections[i];
+        if (!section.heading)
+            continue;
+        // Pop headings at same or deeper level
+        while (stack.length > 0 && stack[stack.length - 1].level >= section.level) {
+            stack.pop();
+        }
+        stack.push({ level: section.level, text: section.heading });
+    }
+    return stack.map(h => h.text);
+}
+/**
+ * Recursively split oversized text on paragraph then line boundaries.
+ * Ensures no chunk exceeds targetChars (best-effort — a single very long
+ * line will be returned as-is).
+ */
+function splitLargeText(text, targetChars) {
+    if (text.length <= targetChars)
+        return [text];
+    // Try paragraph boundaries first
+    const paragraphs = text.split(/\n\n+/);
+    if (paragraphs.length > 1) {
+        return mergeSmallParts(paragraphs, targetChars).flatMap(p => splitLargeText(p, targetChars));
+    }
+    // Fall back to line boundaries
+    const lines = text.split('\n');
+    if (lines.length > 1) {
+        return mergeSmallParts(lines, targetChars, '\n');
+    }
+    // Single long line — return as-is
+    return [text];
+}
+/** Merge adjacent small parts until they approach target size. */
+function mergeSmallParts(parts, targetSize, separator = '\n\n') {
+    const merged = [];
+    let current = '';
+    for (const part of parts) {
+        const sep = current ? separator : '';
+        if (current && (current.length + sep.length + part.length) > targetSize) {
+            merged.push(current);
+            current = part;
+        }
+        else {
+            current = current ? current + sep + part : part;
+        }
+    }
+    if (current)
+        merged.push(current);
+    return merged;
+}
+/**
+ * Merge small consecutive sections that share the same heading context,
+ * then apply overlap between chunks.
+ *
+ * Sections with distinct headings are never merged — each heading-bearing
+ * section becomes its own chunk so that headingPath stays accurate.
+ * Only headingless content following the same heading gets merged.
+ */
+function mergeAndOverlap(sections, targetChars, overlapChars) {
+    const merged = [];
+    let current = '';
+    let currentIdx = 0;
+    for (let i = 0; i < sections.length; i++) {
+        const section = sections[i];
+        const text = section.content;
+        const separator = current ? '\n\n' : '';
+        // Start a new chunk when the section has its own heading
+        // (preserves headingPath per-section) or when size exceeds target
+        const sizeExceeded = current && (current.length + separator.length + text.length) > targetChars;
+        const hasNewHeading = section.heading !== null && current.length > 0;
+        if (sizeExceeded || hasNewHeading) {
+            if (current.trim()) {
+                merged.push({ content: current, sectionIndex: currentIdx });
+            }
+            current = text;
+            currentIdx = i;
+        }
+        else {
+            if (!current)
+                currentIdx = i;
+            current = current ? current + separator + text : text;
+        }
+    }
+    if (current.trim()) {
+        merged.push({ content: current, sectionIndex: currentIdx });
+    }
+    // Apply overlap
+    let result;
+    if (merged.length <= 1 || overlapChars <= 0) {
+        result = merged;
+    }
+    else {
+        result = [merged[0]];
+        for (let i = 1; i < merged.length; i++) {
+            const prev = merged[i - 1].content;
+            const overlapText = prev.slice(-overlapChars);
+            const breakPoint = overlapText.lastIndexOf('\n');
+            const cleanOverlap = breakPoint > 0 ? overlapText.slice(breakPoint) : overlapText;
+            result.push({
+                content: cleanOverlap + '\n\n' + merged[i].content,
+                sectionIndex: merged[i].sectionIndex,
+            });
+        }
+    }
+    // Post-pass: split any chunks that still exceed target
+    const final = [];
+    for (const chunk of result) {
+        if (chunk.content.length > targetChars) {
+            const parts = splitLargeText(chunk.content, targetChars);
+            for (const part of parts) {
+                final.push({ content: part, sectionIndex: chunk.sectionIndex });
+            }
+        }
+        else {
+            final.push(chunk);
+        }
+    }
+    return final;
+}
+/**
+ * Extract title from HTML: <title> tag, then first <h1>, then filename.
+ */
+function extractTitle($, filePath) {
+    const titleTag = $('title').first().text().trim();
+    if (titleTag) {
+        // Strip " — SiteName", " - SiteName", " | SiteName" suffixes common in doc sites
+        const match = titleTag.match(/^(.+)(?:\s+[—\-|]\s+.+)$/);
+        return match ? match[1] : titleTag;
+    }
+    const h1 = $('h1').first().text().trim();
+    if (h1)
+        return h1;
+    return filePath.split('/').pop() ?? filePath;
+}
+/**
+ * Chunk HTML content into embedding-friendly chunks.
+ * Follows the ChunkerFn signature: (content, filePath, config) => ChunkOutput[]
+ */
+export function chunkHtml(content, filePath, config) {
+    if (!content || !content.trim()) {
+        return [];
+    }
+    const $ = cheerio.load(content);
+    // Strip non-content elements
+    $(STRIP_SELECTORS).remove();
+    // Find the main content container
+    let container = null;
+    for (const selector of CONTENT_SELECTORS) {
+        const found = $(selector).first();
+        if (found.length > 0) {
+            container = found;
+            break;
+        }
+    }
+    if (!container) {
+        container = $('body');
+    }
+    if (!container || container.length === 0) {
+        return [];
+    }
+    const title = extractTitle($, filePath);
+    const targetChars = (config.chunk?.target_tokens ?? DEFAULT_TARGET_TOKENS) * 4;
+    const overlapChars = (config.chunk?.overlap_tokens ?? DEFAULT_OVERLAP_TOKENS) * 4;
+    // Split content on heading boundaries
+    const sections = splitOnHeadings($, container);
+    if (sections.length === 0) {
+        // No headings — treat entire content as one or more chunks
+        const text = extractText($, container);
+        if (!text.trim())
+            return [];
+        const parts = splitLargeText(text.trim(), targetChars);
+        return parts.map((content, i) => ({
+            content,
+            title,
+            headingPath: [],
+            chunkIndex: i,
+        }));
+    }
+    // Merge small sections and apply overlap
+    const merged = mergeAndOverlap(sections, targetChars, overlapChars);
+    return merged.map((chunk, i) => ({
+        content: chunk.content,
+        title,
+        headingPath: buildHeadingPath(sections, chunk.sectionIndex),
+        chunkIndex: i,
+    }));
+}
+//# sourceMappingURL=html.js.map

package/dist/indexing/chunking/html.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.js","sourceRoot":"","sources":["../../../src/indexing/chunking/html.ts"],"names":[],"mappings":"AAAA,oEAAoE;AACpE,iEAAiE;AAEjE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAKnC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAElC,6DAA6D;AAC7D,MAAM,eAAe,GAAG,mDAAmD,CAAC;AAE5E,0EAA0E;AAC1E,MAAM,iBAAiB,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;AAEvF;;;GAGG;AACH,SAAS,WAAW,CAAC,CAAa,EAAE,EAAoB;IACpD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,EAAE,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QAC3B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,GAAI,IAAa,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;YACzC,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,OAAO;QACX,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK;YAAE,OAAO;QAChC,MAAM,OAAO,GAAG,IAAe,CAAC;QAChC,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAEzB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YAChB,8CAA8C;YAC9C,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,GAAG,SAAS,CAAC,CAAC;QACrD,CAAC;aAAM,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACtB,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBAChC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7C,CAAC,CAAC,CAAC;QACP,CAAC;aAAM,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACtB,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBAChC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACrD,CAAC,CAAC,CAAC;QACP,CAAC;aAAM,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBAC5B,MAAM,KAAK,GAAa,EAAE,CAAC;gBAC3B,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;oBAClC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC;gBACH,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;QACP,CAAC;aAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,GAAG;gBAAE,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC;QAC3C,CAAC;aAAM,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACjF,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACnC,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;aAAM,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5D,iGAAiG;YACjG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,mDAAmD;YACnD,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACnC,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;AAC9D,CAAC;AAYD,SAAS,eAAe,CAAC,CAAa,EAAE,SAA2B;IAC/D,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,IAAI,cAAc,GAAkB,IAAI,CAAC;IACzC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAa,EAAE,CAAC;IAElC,SAAS,KAAK;QACV,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACzE,IAAI,IAAI,EAAE,CAAC;YACP,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,cAAc,GAAG,EAAE,CAAC;IACxB,CAAC;IAED,qEAAqE;IACrE,SAAS,IAAI,CAAC,EAAoB;QAC9B,EAAE,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAC3B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,MAAM,IAAI,GAAI,IAAa,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;gBACzC,IAAI,IAAI;oBAAE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpC,OAAO;YACX,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK;gBAAE,OAAO;YAChC,MAAM,OAAO,GAAG,IAAe,CAAC;YAChC,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,CAAC;YAE3C,IAAI,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9B,KAAK,EAAE,CAAC;gBACR,cAAc,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAC1C,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChC,OAAO,CAAC,iCAAiC;YAC7C,CAAC;YAED,gEAAgE;YAChE,IAAI,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACtC,IAAI,IAAI;oBAAE,cAAc,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;gBAC3C,OAAO;YACX,CAAC;YAED,yEAAyE;YACzE,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;gBAChB,cAAc,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,SAAS,CAAC,CAAC;gBAC/D,OAAO;YACX,CAAC;YACD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACf,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;oBACrC,cAAc,CAAC,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACtD,CAAC,CAAC,CAAC;gBACH,OAAO;YACX,CAAC;YACD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACf,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;oBACrC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAC9D,CAAC,CAAC,CAAC;gBACH,OAAO;YACX,CAAC;YACD,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;gBAClB,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;oBACjC,MAAM,KAAK,GAAa,EAAE,CAAC;oBAC3B,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;wBAClC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;oBACtC,CAAC,CAAC,CAAC;oBACH,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;wBAAE,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjE,CAAC,CAAC,CAAC;gBACH,OAAO;YACX,CAAC;YACD,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;gBAChB,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnC,IAAI,GAAG;oBAAE,cAAc,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC;gBAChD,OAAO;YACX,CAAC;YAED,yDAAyD;YACzD,IAAI,CAAC,GAAG,EAAE,YAAY,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxE,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;gBACxC,IAAI,IAAI;oBAAE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpC,OAAO;YACX,CAAC;YAED,gEAAgE;YAChE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;IACP,CAAC;IAED,IAAI,CAAC,SAAS,CAAC,CAAC;IAChB,KAAK,EAAE,CAAC;IACR,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,QAAuB,EAAE,SAAiB;IAChE,MAAM,KAAK,GAAsC,EAAE,CAAC;IAEpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,OAAO;YAAE,SAAS;QAE/B,uCAAuC;QACvC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACxE,KAAK,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAClC,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,IAAY,EAAE,WAAmB;IACrD,IAAI,IAAI,CAAC,MAAM,IAAI,WAAW;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9C,iCAAiC;IACjC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,eAAe,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC;IACjG,CAAC;IAED,+BAA+B;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,eAAe,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,kCAAkC;IAClC,OAAO,CAAC,IAAI,CAAC,CAAC;AAClB,CAAC;AAED,kEAAkE;AAClE,SAAS,eAAe,CAAC,KAAe,EAAE,UAAkB,EAAE,YAAoB,MAAM;IACpF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QACrC,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,UAAU,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,GAAG,IAAI,CAAC;QACnB,CAAC;aAAM,CAAC;YACJ,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QACpD,CAAC;IACL,CAAC;IACD,IAAI,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,eAAe,CAAC,QAAuB,EAAE,WAAmB,EAAE,YAAoB;IACvF,MAAM,MAAM,GAAgD,EAAE,CAAC;IAC/D,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAExC,yDAAyD;QACzD,kEAAkE;QAClE,MAAM,YAAY,GAAG,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC;QAChG,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QAErE,IAAI,YAAY,IAAI,aAAa,EAAE,CAAC;YAChC,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC;YAChE,CAAC;YACD,OAAO,GAAG,IAAI,CAAC;YACf,UAAU,GAAG,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,OAAO;gBAAE,UAAU,GAAG,CAAC,CAAC;YAC7B,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1D,CAAC;IACL,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,gBAAgB;IAChB,IAAI,MAAmD,CAAC;IACxD,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1C,MAAM,GAAG,MAAM,CAAC;IACpB,CAAC;SAAM,CAAC;QACJ,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;YACnC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;YAC9C,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,YAAY,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;YAClF,MAAM,CAAC,IAAI,CAAC;gBACR,OAAO,EAAE,YAAY,GAAG,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO;gBAClD,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY;aACvC,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,uDAAuD;IACvD,MAAM,KAAK,GAAgD,EAAE,CAAC;IAC9D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YACzD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC;YACpE,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,CAAa,EAAE,QAAgB;IACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAClD,IAAI,QAAQ,EAAE,CAAC;QACX,iFAAiF;QACjF,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACzD,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IACvC,CAAC;IAED,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAElB,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC;AACjD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,OAAe,EAAE,QAAgB,EAAE,MAAoB;IAC7E,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAC;IACd,CAAC;IAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEhC,6BAA6B;IAC7B,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC;IAE5B,kCAAkC;IAClC,IAAI,SAAS,GAA4B,IAAI,CAAC;IAC9C,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;QAClC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,SAAS,GAAG,KAAK,CAAC;YAClB,MAAM;QACV,CAAC;IACL,CAAC;IACD,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAED,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvC,OAAO,EAAE,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACxC,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,aAAa,IAAI,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC/E,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,cAAc,IAAI,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAElF,sCAAsC;IACtC,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAE/C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,2DAA2D;QAC3D,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;QACvD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC9B,OAAO;YACP,KAAK;YACL,WAAW,EAAE,EAAE;YACf,UAAU,EAAE,CAAC;SAChB,CAAC,CAAC,CAAC;IACR,CAAC;IAED,yCAAyC;IACzC,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;IAEpE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7B,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,KAAK;QACL,WAAW,EAAE,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,YAAY,CAAC;QAC3D,UAAU,EAAE,CAAC;KAChB,CAAC,CAAC,CAAC;AACR,CAAC"}

package/dist/indexing/chunking/index.js

@@ -13,7 +13,12 @@ export function getChunker(type) {
 import { chunkMarkdown } from './markdown.js';
 import { chunkCode } from './code.js';
 import { chunkRawText } from './raw-text.js';
+import { chunkHtml } from './html.js';
 registerChunker('markdown', chunkMarkdown);
 registerChunker('code', chunkCode);
 registerChunker('raw-text', chunkRawText);
+registerChunker('html', chunkHtml);
+import { chunkQa } from './qa.js';
+registerChunker('slack', chunkQa);
+registerChunker('discord', chunkQa);
 //# sourceMappingURL=index.js.map

package/dist/indexing/chunking/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/indexing/chunking/index.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAM9D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;AAE9C,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,EAAa;IACvD,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACnC,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,iBAAiB,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3G,OAAO,EAAE,CAAC;AACd,CAAC;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,eAAe,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAC3C,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AACnC,eAAe,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/indexing/chunking/index.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAM9D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;AAE9C,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,EAAa;IACvD,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACnC,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,iBAAiB,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3G,OAAO,EAAE,CAAC;AACd,CAAC;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,eAAe,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAC3C,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AACnC,eAAe,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;AAC1C,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEnC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAClC,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC"}

package/dist/indexing/chunking/qa.d.ts

@@ -0,0 +1,8 @@
+import type { ChunkOutput, SourceConfig } from '../../types.js';
+/**
+ * Chunk Q&A content. Each content item from a FAQ-category provider
+ * is a single Q&A pair, already sized appropriately.
+ * The chunker formats it and returns a single ChunkOutput.
+ */
+export declare function chunkQa(content: string, filePath: string, config: SourceConfig): ChunkOutput[];
+//# sourceMappingURL=qa.d.ts.map

package/dist/indexing/chunking/qa.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"qa.d.ts","sourceRoot":"","sources":["../../../src/indexing/chunking/qa.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEhE;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,WAAW,EAAE,CAe9F"}

package/dist/indexing/chunking/qa.js

@@ -0,0 +1,22 @@
+// Q&A chunker — formats distilled Q&A pairs for embedding.
+// Source-agnostic: used by any source that produces Q&A-formatted content.
+/**
+ * Chunk Q&A content. Each content item from a FAQ-category provider
+ * is a single Q&A pair, already sized appropriately.
+ * The chunker formats it and returns a single ChunkOutput.
+ */
+export function chunkQa(content, filePath, config) {
+    if (!content || !content.trim()) {
+        return [];
+    }
+    // The content is already formatted as "Q: ...\n\nA: ..." by the provider.
+    // Extract the question for use as title.
+    const questionMatch = content.match(/^Q:\s*(.+?)(?:\n|$)/);
+    const title = questionMatch ? questionMatch[1].trim() : undefined;
+    return [{
+            content: content.trim(),
+            title,
+            chunkIndex: 0,
+        }];
+}
+//# sourceMappingURL=qa.js.map

package/dist/indexing/chunking/qa.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"qa.js","sourceRoot":"","sources":["../../../src/indexing/chunking/qa.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,2EAA2E;AAI3E;;;;GAIG;AACH,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,QAAgB,EAAE,MAAoB;IAC3E,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAC;IACd,CAAC;IAED,0EAA0E;IAC1E,yCAAyC;IACzC,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAElE,OAAO,CAAC;YACJ,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;YACvB,KAAK;YACL,UAAU,EAAE,CAAC;SAChB,CAAC,CAAC;AACP,CAAC"}

package/dist/indexing/distiller.d.ts

@@ -0,0 +1,29 @@
+import OpenAI from 'openai';
+export interface ThreadMessage {
+    author: string;
+    content: string;
+    timestamp: string;
+    reactions?: Array<{
+        name: string;
+        count: number;
+    }>;
+}
+export interface DistilledPair {
+    question: string;
+    answer: string;
+    confidence: number;
+}
+export interface DistillerResult {
+    pairs: DistilledPair[];
+}
+export interface DistillerOptions {
+    model?: string;
+    maxMessages?: number;
+    apiKey?: string;
+    client?: OpenAI;
+}
+/**
+ * Distill a conversation thread into Q&A pairs using an LLM.
+ */
+export declare function distillThread(messages: ThreadMessage[], options?: DistillerOptions): Promise<DistillerResult>;
+//# sourceMappingURL=distiller.d.ts.map

package/dist/indexing/distiller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"distiller.d.ts","sourceRoot":"","sources":["../../src/indexing/distiller.ts"],"names":[],"mappings":"AAGA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,MAAM,WAAW,aAAa;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,aAAa;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC5B,KAAK,EAAE,aAAa,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAuCD;;GAEG;AACH,wBAAsB,aAAa,CAC/B,QAAQ,EAAE,aAAa,EAAE,EACzB,OAAO,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,eAAe,CAAC,CAyE1B"}

package/dist/indexing/distiller.js

@@ -0,0 +1,104 @@
+// LLM thread distiller — extracts Q&A pairs from conversation threads.
+// Source-agnostic: takes structured messages, returns structured Q&A pairs.
+import OpenAI from 'openai';
+// ── Constants ────────────────────────────────────────────────────────────────
+const DEFAULT_MODEL = 'gpt-4o-mini';
+const DEFAULT_MAX_MESSAGES = 100;
+const SYSTEM_PROMPT = `You are a Q&A extraction engine. Given a conversation thread, identify distinct question-answer pairs.
+
+For each pair:
+1. Extract the core question (rephrase if needed for clarity)
+2. Extract the best answer (synthesize from multiple replies if needed)
+3. Score confidence from 0.0 to 1.0 based on:
+   - Answer completeness (does it fully address the question?)
+   - Questioner satisfaction signals ("thanks", "that worked", etc.)
+   - Community validation (reactions like thumbsup, check marks)
+   - Answer specificity (concrete steps vs vague suggestions)
+
+Return JSON with this exact structure:
+{
+  "pairs": [
+    {
+      "question": "How do I configure X?",
+      "answer": "You can configure X by...",
+      "confidence": 0.85
+    }
+  ]
+}
+
+Rules:
+- A thread may contain multiple Q&A pairs (follow-up questions)
+- Skip greetings, pleasantries, and off-topic tangents
+- If no clear Q&A exists, return {"pairs": []}
+- Keep answers concise but complete (aim for 1-3 paragraphs)
+- Preserve code blocks, URLs, and technical details from answers
+- Confidence below 0.3 means the answer is likely incomplete or wrong`;
+// ── Distiller ────────────────────────────────────────────────────────────────
+/**
+ * Distill a conversation thread into Q&A pairs using an LLM.
+ */
+export async function distillThread(messages, options) {
+    const model = options?.model ?? DEFAULT_MODEL;
+    const maxMessages = options?.maxMessages ?? DEFAULT_MAX_MESSAGES;
+    if (messages.length === 0) {
+        return { pairs: [] };
+    }
+    // Truncate to max messages
+    const truncated = messages.slice(0, maxMessages);
+    // Format as conversation transcript
+    const transcript = truncated.map(msg => {
+        const reactions = msg.reactions && msg.reactions.length > 0
+            ? ` [reactions: ${msg.reactions.map(r => `:${r.name}: x${r.count}`).join(', ')}]`
+            : '';
+        return `[${msg.timestamp}] ${msg.author}: ${msg.content}${reactions}`;
+    }).join('\n\n');
+    const client = options?.client ?? new OpenAI({ apiKey: options?.apiKey });
+    try {
+        const response = await client.chat.completions.create({
+            model,
+            messages: [
+                { role: 'system', content: SYSTEM_PROMPT },
+                { role: 'user', content: transcript },
+            ],
+            response_format: { type: 'json_object' },
+            temperature: 0.1,
+        });
+        const content = response.choices[0]?.message?.content;
+        if (!content) {
+            console.warn('[distiller] Empty response from LLM');
+            return { pairs: [] };
+        }
+        const parsed = JSON.parse(content);
+        // Validate structure
+        if (!Array.isArray(parsed.pairs)) {
+            console.warn('[distiller] Invalid response structure — missing pairs array');
+            return { pairs: [] };
+        }
+        // Validate and filter each pair
+        const validPairs = [];
+        for (const pair of parsed.pairs) {
+            if (typeof pair.question === 'string' && pair.question.trim() &&
+                typeof pair.answer === 'string' && pair.answer.trim() &&
+                typeof pair.confidence === 'number' &&
+                pair.confidence >= 0 && pair.confidence <= 1) {
+                validPairs.push({
+                    question: pair.question.trim(),
+                    answer: pair.answer.trim(),
+                    confidence: pair.confidence,
+                });
+            }
+            else {
+                console.warn('[distiller] Skipping malformed pair:', JSON.stringify(pair).slice(0, 200));
+            }
+        }
+        return { pairs: validPairs };
+    }
+    catch (error) {
+        if (error instanceof SyntaxError) {
+            console.error('[distiller] Failed to parse LLM JSON response:', error.message);
+            return { pairs: [] };
+        }
+        throw error; // Re-throw API errors for caller to handle
+    }
+}
+//# sourceMappingURL=distiller.js.map

package/dist/indexing/distiller.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"distiller.js","sourceRoot":"","sources":["../../src/indexing/distiller.ts"],"names":[],"mappings":"AAAA,uEAAuE;AACvE,4EAA4E;AAE5E,OAAO,MAAM,MAAM,QAAQ,CAAC;AA4B5B,gFAAgF;AAEhF,MAAM,aAAa,GAAG,aAAa,CAAC;AACpC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAEjC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;sEA4BgD,CAAC;AAEvE,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAC/B,QAAyB,EACzB,OAA0B;IAE1B,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,aAAa,CAAC;IAC9C,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,oBAAoB,CAAC;IAEjE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACzB,CAAC;IAED,2BAA2B;IAC3B,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IAEjD,oCAAoC;IACpC,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QACnC,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;YACvD,CAAC,CAAC,gBAAgB,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACjF,CAAC,CAAC,EAAE,CAAC;QACT,OAAO,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,OAAO,GAAG,SAAS,EAAE,CAAC;IAC1E,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAE1E,IAAI,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YAClD,KAAK;YACL,QAAQ,EAAE;gBACN,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;gBAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE;aACxC;YACD,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;YACxC,WAAW,EAAE,GAAG;SACnB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACtD,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACpD,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACzB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;YAC7E,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACzB,CAAC;QAED,gCAAgC;QAChC,MAAM,UAAU,GAAoB,EAAE,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC9B,IACI,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;gBACzD,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;gBACrD,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;gBACnC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,EAC9C,CAAC;gBACC,UAAU,CAAC,IAAI,CAAC;oBACZ,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;oBAC9B,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;oBAC1B,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC9B,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7F,CAAC;QACL,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,gDAAgD,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/E,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,CAAC,CAAC,2CAA2C;IAC5D,CAAC;AACL,CAAC"}

package/dist/indexing/orchestrator.d.ts

@@ -4,6 +4,7 @@ export declare class IndexingOrchestrator {
     private processing;
     private activeSources;
     private lastReindexDate;
+    onReindexComplete?: (sourceNames: string[]) => void;
     constructor();
     /**
      * Smart startup check: compare DB commit SHAs against remote HEAD.
@@ -12,10 +13,10 @@ export declare class IndexingOrchestrator {
      */
     checkAndIndex(): Promise<void>;
     /**
-     * Get the HEAD SHA of a remote repo without cloning.
-     * Uses `git ls-remote` which only fetches refs.
+     * Get the current state token for a source without acquiring items.
+     * Returns null if the source is unavailable.
      */
-    private getRemoteHead;
+    private getSourceStateToken;
     /**
      * Queue a full re-index of all sources. Returns immediately.
      */
@@ -24,6 +25,11 @@ export declare class IndexingOrchestrator {
      * Queue an incremental re-index for a specific repo. Returns immediately.
      */
     queueIncrementalReindex(repoUrl: string): void;
+    /**
+     * Queue a reindex for a single named source. Returns immediately.
+     * Used by webhook handlers to trigger reindexing of specific sources.
+     */
+    queueSourceReindex(sourceName: string): void;
     /**
      * Returns true if any indexing job is currently running.
      */

package/dist/indexing/orchestrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AA+BA,qBAAa,oBAAoB;IAC7B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAAS;IAG3B,OAAO,CAAC,aAAa,CAAqB;IAG1C,OAAO,CAAC,eAAe,CAAuB;;IAM9C;;;;OAIG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAqGpC;;;OAGG;YACW,aAAa;IAa3B;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAQxB;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAU9C;;OAEG;IACH,UAAU,IAAI,OAAO;IAIrB;;;OAGG;IACH,mBAAmB,IAAI,IAAI;IA4B3B;;OAEG;IACH,OAAO,CAAC,OAAO;IASf;;OAEG;YACW,KAAK;IAwBnB;;OAEG;YACW,UAAU;IAoCxB;;OAEG;YACW,cAAc;IAqB5B;;OAEG;YACW,qBAAqB;IAoDnC;;OAEG;YACW,oBAAoB;IAgDlC;;OAEG;YACW,cAAc;IAmB5B;;OAEG;YACW,cAAc;CAgB/B"}
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAqCA,qBAAa,oBAAoB;IAC7B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAAS;IAG3B,OAAO,CAAC,aAAa,CAAqB;IAG1C,OAAO,CAAC,eAAe,CAAuB;IAG9C,iBAAiB,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC;;IAMpD;;;;OAIG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IA+GpC;;;OAGG;YACW,mBAAmB;IAYjC;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAQxB;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAU9C;;;OAGG;IACH,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAQ5C;;OAEG;IACH,UAAU,IAAI,OAAO;IAIrB;;;OAGG;IACH,mBAAmB,IAAI,IAAI;IA4B3B;;OAEG;IACH,OAAO,CAAC,OAAO;IASf;;OAEG;YACW,KAAK;IAwBnB;;OAEG;YACW,UAAU;IA8DxB;;OAEG;YACW,cAAc;IAqB5B;;OAEG;YACW,qBAAqB;IAiBnC;;OAEG;YACW,oBAAoB;IAsDlC;;OAEG;YACW,cAAc;IAmB5B;;OAEG;YACW,cAAc;CAgB/B"}