doc2vec 1.3.0 → 2.2.0

package/dist/content-processor.js

@@ -47,6 +47,7 @@ const turndown_1 = __importDefault(require("turndown"));
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const utils_1 = require("./utils");
+const code_chunker_1 = require("./code-chunker");
 class ContentProcessor {
     constructor(logger) {
         this.logger = logger;
@@ -54,6 +55,9 @@ class ContentProcessor {
             codeBlockStyle: 'fenced',
             headingStyle: 'atx'
         });
+        this.tokenChunkerCache = new Map();
+        this.codeChunkerCache = new Map();
+        this.tokenizerCache = null;
         this.setupTurndownRules();
     }
     setupTurndownRules() {
@@ -200,15 +204,38 @@ class ContentProcessor {
     async crawlWebsite(baseUrl, sourceConfig, processPageContent, parentLogger, visitedUrls) {
         const logger = parentLogger.child('crawler');
         const queue = [baseUrl];
+        const brokenLinks = [];
+        const brokenLinkKeys = new Set();
+        const referrers = new Map();
+        const addReferrer = (targetUrl, sourceUrl) => {
+            const existing = referrers.get(targetUrl);
+            if (existing) {
+                existing.add(sourceUrl);
+            }
+            else {
+                referrers.set(targetUrl, new Set([sourceUrl]));
+            }
+        };
+        const addBrokenLink = (sourceUrl, targetUrl) => {
+            const key = `${sourceUrl} -> ${targetUrl}`;
+            if (brokenLinkKeys.has(key))
+                return;
+            brokenLinkKeys.add(key);
+            brokenLinks.push({ source: sourceUrl, target: targetUrl });
+        };
+        addReferrer(baseUrl, baseUrl);
         // Process sitemap if provided
         if (sourceConfig.sitemap_url) {
             logger.section('SITEMAP PROCESSING');
             const sitemapUrls = await this.parseSitemap(sourceConfig.sitemap_url, logger);
             // Add sitemap URLs to the queue if they're within the website scope
             for (const url of sitemapUrls) {
-                if (url.startsWith(sourceConfig.url) && !queue.includes(url)) {
-                    logger.debug(`Adding URL from sitemap to queue: ${url}`);
-                    queue.push(url);
+                if (url.startsWith(sourceConfig.url)) {
+                    addReferrer(url, sourceConfig.sitemap_url);
+                    if (!queue.includes(url)) {
+                        logger.debug(`Adding URL from sitemap to queue: ${url}`);
+                        queue.push(url);
+                    }
                 }
             }
             logger.info(`Added ${queue.length - 1} URLs from sitemap to the crawl queue`);
@@ -235,7 +262,14 @@ class ContentProcessor {
             }
             try {
                 logger.info(`Crawling: ${url}`);
-                const content = await this.processPage(url, sourceConfig);
+                const sources = referrers.get(url) ?? new Set([baseUrl]);
+                const content = await this.processPage(url, sourceConfig, (reportedUrl, status) => {
+                    if (status === 404) {
+                        for (const source of sources) {
+                            addBrokenLink(source, reportedUrl);
+                        }
+                    }
+                });
                 if (content !== null) {
                     await processPageContent(url, content);
                     if (utils_1.Utils.isPdfUrl(url)) {
@@ -252,17 +286,21 @@ class ContentProcessor {
                 if (!utils_1.Utils.isPdfUrl(url)) {
                     const response = await axios_1.default.get(url);
                     const $ = (0, cheerio_1.load)(response.data);
+                    const pageUrlForLinks = response?.request?.res?.responseUrl || url;
                     logger.debug(`Finding links on page ${url}`);
                     let newLinksFound = 0;
                     $('a[href]').each((_, element) => {
                         const href = $(element).attr('href');
                         if (!href || href.startsWith('#') || href.startsWith('mailto:'))
                             return;
-                        const fullUrl = utils_1.Utils.buildUrl(href, url);
-                        if (fullUrl.startsWith(sourceConfig.url) && !visitedUrls.has(utils_1.Utils.normalizeUrl(fullUrl))) {
-                            if (!queue.includes(fullUrl)) {
-                                queue.push(fullUrl);
-                                newLinksFound++;
+                        const fullUrl = utils_1.Utils.buildUrl(href, pageUrlForLinks);
+                        if (fullUrl.startsWith(sourceConfig.url)) {
+                            addReferrer(fullUrl, pageUrlForLinks);
+                            if (!visitedUrls.has(utils_1.Utils.normalizeUrl(fullUrl))) {
+                                if (!queue.includes(fullUrl)) {
+                                    queue.push(fullUrl);
+                                    newLinksFound++;
+                                }
                             }
                         }
                     });
@@ -272,6 +310,13 @@ class ContentProcessor {
             catch (error) {
                 logger.error(`Failed during processing or link discovery for ${url}:`, error);
                 errorCount++;
+                const status = this.getHttpStatus(error);
+                if (status === 404) {
+                    const sources = referrers.get(url) ?? new Set([baseUrl]);
+                    for (const source of sources) {
+                        addBrokenLink(source, url);
+                    }
+                }
                 // Check if this is a network error (DNS resolution, connection issues, etc.)
                 if (this.isNetworkError(error)) {
                     hasNetworkErrors = true;
@@ -283,7 +328,7 @@ class ContentProcessor {
         if (hasNetworkErrors) {
             logger.warn('Network errors were encountered during crawling. Cleanup may be skipped to avoid removing valid chunks.');
         }
-        return { hasNetworkErrors };
+        return { hasNetworkErrors, brokenLinks };
     }
     isNetworkError(error) {
         // Check for common network error patterns
@@ -324,7 +369,7 @@ class ContentProcessor {
         }
         return false;
     }
-    async processPage(url, sourceConfig) {
+    async processPage(url, sourceConfig, onHttpStatus) {
         const logger = this.logger.child('page-processor');
         logger.debug(`Processing content from ${url}`);
         // Check if this is a PDF URL
@@ -340,6 +385,13 @@ class ContentProcessor {
                 return markdown;
             }
             catch (error) {
+                const status = this.getHttpStatus(error);
+                if (status !== undefined && status >= 400) {
+                    if (onHttpStatus) {
+                        onHttpStatus(url, status);
+                    }
+                    throw error;
+                }
                 logger.error(`Failed to process PDF ${url}:`, error);
                 return null;
             }
@@ -347,14 +399,38 @@ class ContentProcessor {
         // Original HTML page processing logic
         let browser = null;
         try {
+            // Use system Chromium if available (for Docker environments)
+            let executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
+            if (!executablePath) {
+                if (fs.existsSync('/usr/bin/chromium')) {
+                    executablePath = '/usr/bin/chromium';
+                }
+                else if (fs.existsSync('/usr/bin/chromium-browser')) {
+                    executablePath = '/usr/bin/chromium-browser';
+                }
+            }
             browser = await puppeteer_1.default.launch({
+                executablePath,
                 args: ['--no-sandbox', '--disable-setuid-sandbox'],
             });
             const page = await browser.newPage();
             logger.debug(`Navigating to ${url}`);
-            await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
+            const response = await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
+            const status = response?.status();
+            if (status !== undefined && status >= 400) {
+                const error = new Error(`Failed to load page: HTTP ${status}`);
+                error.status = status;
+                throw error;
+            }
             const htmlContent = await page.evaluate(() => {
-                const mainContentElement = document.querySelector('div[role="main"].document') || document.querySelector('main') || document.body;
+                // 💡 Try specific content selectors first, then fall back to broader ones
+                const mainContentElement = document.querySelector('.docs-content') || // Common docs pattern
+                    document.querySelector('.doc-content') || // Alternative docs pattern
+                    document.querySelector('.markdown-body') || // GitHub-style
+                    document.querySelector('article') || // Semantic article
+                    document.querySelector('div[role="main"].document') ||
+                    document.querySelector('main') ||
+                    document.body;
                 return mainContentElement.innerHTML;
             });
             if (htmlContent.length > sourceConfig.max_size) {
@@ -370,10 +446,24 @@ class ContentProcessor {
                 pre.setAttribute('data-readable-content-score', '100');
                 this.markCodeParents(pre.parentElement);
             });
+            // 💡 Extract H1s BEFORE Readability - it often strips them as "chrome"
+            // We'll inject them back after Readability processing
+            const h1Elements = document.querySelectorAll('h1');
+            const extractedH1s = [];
+            logger.debug(`[Readability Debug] Found ${h1Elements.length} H1 elements before Readability`);
+            h1Elements.forEach((h1, index) => {
+                const h1Text = h1.textContent?.trim() || '';
+                // Skip empty H1s or icon-only H1s (like "link" anchors)
+                if (h1Text && h1Text.length > 3 && !h1Text.match(/^(link|#|menu|close)$/i)) {
+                    extractedH1s.push(h1Text);
+                    logger.debug(`[Readability Debug] Extracted H1[${index}]: "${h1Text.substring(0, 50)}..."`);
+                }
+                h1.classList.add('original-h1');
+            });
             logger.debug(`Applying Readability to extract main content`);
             const reader = new readability_1.Readability(document, {
                 charThreshold: 20,
-                classesToPreserve: ['article-content'],
+                classesToPreserve: ['article-content', 'original-h1'],
             });
             const article = reader.parse();
             if (!article) {
@@ -381,8 +471,35 @@ class ContentProcessor {
                 await browser.close();
                 return null;
             }
-            logger.debug(`Sanitizing HTML (${article.content.length} chars)`);
-            const cleanHtml = (0, sanitize_html_1.default)(article.content, {
+            // Debug: Log what Readability extracted
+            logger.debug(`[Readability Debug] article.title: "${article.title}"`);
+            logger.debug(`[Readability Debug] article.content length: ${article.content?.length}`);
+            logger.debug(`[Readability Debug] article.content starts with: "${article.content?.substring(0, 200)}..."`);
+            logger.debug(`[Readability Debug] Contains H1 tag: ${article.content?.includes('<h1')}`);
+            logger.debug(`[Readability Debug] Contains H2 tag: ${article.content?.includes('<h2')}`);
+            logger.debug(`[Readability Debug] Contains original-h1 class: ${article.content?.includes('original-h1')}`);
+            // 💡 Restore H1s: find elements with our marker class and convert back from H2
+            const articleDom = new jsdom_1.JSDOM(article.content);
+            const articleDoc = articleDom.window.document;
+            const originalH1Elements = articleDoc.querySelectorAll('.original-h1');
+            logger.debug(`[Readability Debug] Found ${originalH1Elements.length} elements with .original-h1 class to restore`);
+            originalH1Elements.forEach((heading, index) => {
+                logger.debug(`[Readability Debug] Restoring[${index}]: tagName=${heading.tagName}, text="${heading.textContent?.trim().substring(0, 50)}..."`);
+                // Create a new H1 element with the same content
+                const h1 = articleDoc.createElement('h1');
+                h1.innerHTML = heading.innerHTML;
+                // Copy other attributes except class
+                Array.from(heading.attributes).forEach(attr => {
+                    if (attr.name !== 'class') {
+                        h1.setAttribute(attr.name, attr.value);
+                    }
+                });
+                heading.replaceWith(h1);
+            });
+            const restoredContent = articleDoc.body.innerHTML;
+            logger.debug(`[Readability Debug] Restored content contains H1: ${restoredContent.includes('<h1')}`);
+            logger.debug(`Sanitizing HTML (${restoredContent.length} chars)`);
+            const cleanHtml = (0, sanitize_html_1.default)(restoredContent, {
                 allowedTags: [
                     'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
                     'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
@@ -397,11 +514,37 @@ class ContentProcessor {
                 }
             });
             logger.debug(`Converting HTML to Markdown`);
-            const markdown = this.turndownService.turndown(cleanHtml);
+            let markdown = this.turndownService.turndown(cleanHtml);
+            // 💡 Inject extracted H1s back if they're not in the markdown
+            // Readability often strips them as "page chrome"
+            // Use article.title as fallback if no H1 was extracted
+            const pageTitle = extractedH1s.length > 0 ? extractedH1s[0] : (article.title?.trim() || '');
+            if (pageTitle) {
+                // Check if markdown already starts with this exact H1 (allowing for leading whitespace)
+                const normalizedTitle = pageTitle.replace(/\s+/g, ' ');
+                const markdownFirstLine = markdown.trimStart().split('\n')[0] || '';
+                const existingH1Match = markdownFirstLine.match(/^#\s+(.+)$/);
+                const existingH1Text = existingH1Match ? existingH1Match[1].replace(/\s+/g, ' ').trim() : '';
+                // Only inject if markdown doesn't already start with this H1
+                if (!existingH1Match || existingH1Text !== normalizedTitle) {
+                    markdown = `# ${pageTitle}\n\n${markdown}`;
+                    logger.debug(`[Readability Debug] Injected page title as H1: "${pageTitle}"`);
+                }
+                else {
+                    logger.debug(`[Readability Debug] H1 "${pageTitle}" already present in markdown`);
+                }
+            }
             logger.debug(`Markdown conversion complete (${markdown.length} chars)`);
             return markdown;
         }
         catch (error) {
+            const status = this.getHttpStatus(error);
+            if (status !== undefined && status >= 400) {
+                if (onHttpStatus) {
+                    onHttpStatus(url, status);
+                }
+                throw error;
+            }
             logger.error(`Error processing page ${url}:`, error);
             return null;
         }
@@ -412,6 +555,15 @@ class ContentProcessor {
             }
         }
     }
+    getHttpStatus(error) {
+        if (typeof error?.status === 'number') {
+            return error.status;
+        }
+        if (typeof error?.response?.status === 'number') {
+            return error.response.status;
+        }
+        return undefined;
+    }
     markCodeParents(node) {
         if (!node)
             return;
@@ -421,6 +573,68 @@ class ContentProcessor {
         }
         this.markCodeParents(node.parentElement);
     }
+    async convertDocToMarkdown(filePath, logger) {
+        logger.debug(`Converting DOC to markdown: ${filePath}`);
+        try {
+            // Dynamic import for word-extractor
+            const WordExtractor = (await Promise.resolve().then(() => __importStar(require('word-extractor')))).default;
+            const extractor = new WordExtractor();
+            const extracted = await extractor.extract(filePath);
+            const text = extracted.getBody();
+            // Create markdown with filename as title
+            let markdown = `# ${path.basename(filePath, '.doc')}\n\n`;
+            // Clean up the text and add to markdown
+            const cleanedText = text
+                .replace(/\r\n/g, '\n') // Normalize line endings
+                .replace(/\n{3,}/g, '\n\n') // Remove excessive line breaks
+                .trim();
+            markdown += cleanedText;
+            logger.debug(`Converted DOC to ${markdown.length} characters of markdown`);
+            return markdown;
+        }
+        catch (error) {
+            logger.error(`Failed to convert DOC ${filePath}:`, error);
+            throw error;
+        }
+    }
+    async convertDocxToMarkdown(filePath, logger) {
+        logger.debug(`Converting DOCX to markdown: ${filePath}`);
+        try {
+            // Dynamic import for mammoth
+            const mammoth = await Promise.resolve().then(() => __importStar(require('mammoth')));
+            const result = await mammoth.convertToHtml({ path: filePath });
+            const html = result.value;
+            // Log any warnings from mammoth
+            if (result.messages.length > 0) {
+                logger.debug(`Mammoth warnings: ${result.messages.map(m => m.message).join(', ')}`);
+            }
+            // Create markdown with filename as title
+            let markdown = `# ${path.basename(filePath, '.docx')}\n\n`;
+            // Convert HTML to Markdown using turndown
+            const cleanHtml = (0, sanitize_html_1.default)(html, {
+                allowedTags: [
+                    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
+                    'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
+                    'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td', 'br'
+                ],
+                allowedAttributes: {
+                    'a': ['href'],
+                    'pre': ['class'],
+                    'code': ['class']
+                }
+            });
+            const convertedContent = this.turndownService.turndown(cleanHtml);
+            markdown += convertedContent;
+            // Clean up excessive line breaks
+            markdown = markdown.replace(/\n{3,}/g, '\n\n').trim();
+            logger.debug(`Converted DOCX to ${markdown.length} characters of markdown`);
+            return markdown;
+        }
+        catch (error) {
+            logger.error(`Failed to convert DOCX ${filePath}:`, error);
+            throw error;
+        }
+    }
     async convertPdfToMarkdown(filePath, logger) {
         logger.debug(`Converting PDF to markdown: ${filePath}`);
         try {
@@ -611,6 +825,16 @@ class ContentProcessor {
                             logger.debug(`Processing PDF file: ${filePath}`);
                             processedContent = await this.convertPdfToMarkdown(filePath, logger);
                         }
+                        else if (extension === '.doc') {
+                            // Handle legacy Word DOC files
+                            logger.debug(`Processing DOC file: ${filePath}`);
+                            processedContent = await this.convertDocToMarkdown(filePath, logger);
+                        }
+                        else if (extension === '.docx') {
+                            // Handle modern Word DOCX files
+                            logger.debug(`Processing DOCX file: ${filePath}`);
+                            processedContent = await this.convertDocxToMarkdown(filePath, logger);
+                        }
                         else {
                             // Handle text-based files
                             content = fs.readFileSync(filePath, { encoding: encoding });
@@ -662,86 +886,410 @@ class ContentProcessor {
             logger.error(`Error reading directory ${dirPath}:`, error);
         }
     }
+    async processCodeDirectory(dirPath, config, processFileContent, parentLogger, visitedPaths = new Set(), options) {
+        const logger = parentLogger.child('code-directory-processor');
+        logger.info(`Processing code directory: ${dirPath}`);
+        const recursive = config.recursive !== undefined ? config.recursive : true;
+        const includeExtensions = config.include_extensions || [
+            '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
+            '.py', '.go', '.rs', '.java', '.kt', '.kts', '.swift',
+            '.c', '.cc', '.cpp', '.h', '.hpp', '.cs',
+            '.rb', '.php', '.scala', '.sql', '.sh', '.bash', '.zsh',
+            '.html', '.css', '.scss', '.sass', '.less',
+            '.json', '.yaml', '.yml', '.md'
+        ];
+        const excludeExtensions = config.exclude_extensions || [];
+        const encoding = config.encoding || 'utf8';
+        let maxMtime = 0;
+        try {
+            const files = fs.readdirSync(dirPath);
+            let processedFiles = 0;
+            let skippedFiles = 0;
+            const allowedFiles = options?.allowedFiles;
+            const mtimeCutoff = options?.mtimeCutoff;
+            const trackFiles = options?.trackFiles;
+            for (const file of files) {
+                const filePath = path.join(dirPath, file);
+                const stat = fs.statSync(filePath);
+                if (visitedPaths.has(filePath)) {
+                    logger.debug(`Skipping already visited path: ${filePath}`);
+                    continue;
+                }
+                visitedPaths.add(filePath);
+                if (stat.isDirectory()) {
+                    if (recursive) {
+                        const childResult = await this.processCodeDirectory(filePath, config, processFileContent, logger, visitedPaths, options);
+                        processedFiles += childResult.processedFiles;
+                        skippedFiles += childResult.skippedFiles;
+                        maxMtime = Math.max(maxMtime, childResult.maxMtime);
+                    }
+                    else {
+                        logger.debug(`Skipping directory ${filePath} (recursive=false)`);
+                    }
+                }
+                else if (stat.isFile()) {
+                    const extension = path.extname(file).toLowerCase();
+                    if (excludeExtensions.includes(extension)) {
+                        logger.debug(`Skipping file with excluded extension: ${filePath}`);
+                        skippedFiles++;
+                        continue;
+                    }
+                    if (includeExtensions.length > 0 && !includeExtensions.includes(extension)) {
+                        logger.debug(`Skipping file with non-included extension: ${filePath}`);
+                        skippedFiles++;
+                        continue;
+                    }
+                    trackFiles?.add(filePath);
+                    maxMtime = Math.max(maxMtime, stat.mtimeMs);
+                    if (allowedFiles && !allowedFiles.has(filePath)) {
+                        skippedFiles++;
+                        continue;
+                    }
+                    if (mtimeCutoff !== undefined && stat.mtimeMs <= mtimeCutoff) {
+                        skippedFiles++;
+                        continue;
+                    }
+                    try {
+                        logger.info(`Reading file: ${filePath}`);
+                        const content = fs.readFileSync(filePath, { encoding: encoding });
+                        if (content.length > config.max_size) {
+                            logger.warn(`File content (${content.length} chars) exceeds max size (${config.max_size}). Skipping ${filePath}.`);
+                            skippedFiles++;
+                            continue;
+                        }
+                        await processFileContent(filePath, content);
+                        processedFiles++;
+                    }
+                    catch (error) {
+                        logger.error(`Error processing file ${filePath}:`, error);
+                    }
+                }
+            }
+            logger.info(`Code directory processed. Processed: ${processedFiles}, Skipped: ${skippedFiles}`);
+            return { processedFiles, skippedFiles, maxMtime };
+        }
+        catch (error) {
+            logger.error(`Error reading code directory ${dirPath}:`, error);
+            return { processedFiles: 0, skippedFiles: 0, maxMtime };
+        }
+    }
+    async getTokenChunker(chunkSize) {
+        const cacheKey = `${chunkSize || 'default'}`;
+        const cached = this.tokenChunkerCache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const chunkerPromise = (async () => {
+            const { TokenChunker } = await this.importChonkieModule('@chonkiejs/core');
+            return await TokenChunker.create({ chunkSize, tokenizer: 'character' });
+        })();
+        this.tokenChunkerCache.set(cacheKey, chunkerPromise);
+        return chunkerPromise;
+    }
+    async getCodeChunker(lang, chunkSize) {
+        const cacheKey = `${lang}:${chunkSize || 'default'}`;
+        const cached = this.codeChunkerCache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const chunkerPromise = (async () => {
+            const tokenizer = await this.getTokenizer();
+            return await code_chunker_1.CodeChunker.create({
+                lang,
+                chunkSize,
+                tokenCounter: async (text) => tokenizer.countTokens(text)
+            });
+        })();
+        this.codeChunkerCache.set(cacheKey, chunkerPromise);
+        return chunkerPromise;
+    }
+    async getTokenizer() {
+        if (!this.tokenizerCache) {
+            this.tokenizerCache = (async () => {
+                const { Tokenizer } = await this.importChonkieModule('@chonkiejs/core');
+                return await Tokenizer.create('character');
+            })();
+        }
+        return this.tokenizerCache;
+    }
+    detectCodeLanguage(filePath) {
+        const extension = path.extname(filePath).toLowerCase();
+        const languageMap = {
+            '.ts': 'typescript',
+            '.tsx': 'typescript',
+            '.js': 'javascript',
+            '.jsx': 'javascript',
+            '.mjs': 'javascript',
+            '.cjs': 'javascript',
+            '.py': 'python',
+            '.go': 'go',
+            '.rs': 'rust',
+            '.java': 'java',
+            '.kt': 'kotlin',
+            '.kts': 'kotlin',
+            '.swift': 'swift',
+            '.c': 'c',
+            '.cc': 'cpp',
+            '.cpp': 'cpp',
+            '.h': 'cpp',
+            '.hpp': 'cpp',
+            '.cs': 'csharp',
+            '.rb': 'ruby',
+            '.php': 'php',
+            '.scala': 'scala',
+            '.sql': 'sql',
+            '.sh': 'bash',
+            '.bash': 'bash',
+            '.zsh': 'bash',
+            '.html': 'html',
+            '.css': 'css',
+            '.scss': 'scss',
+            '.sass': 'scss',
+            '.less': 'css',
+            '.json': 'json',
+            '.yaml': 'yaml',
+            '.yml': 'yaml',
+            '.md': 'markdown'
+        };
+        return languageMap[extension];
+    }
+    async importChonkieModule(specifier) {
+        // Use dynamic import() directly - preserved by TypeScript with target ES2020+
+        // even in CommonJS mode, as Node.js supports import() in CJS contexts
+        return Promise.resolve(`${specifier}`).then(s => __importStar(require(s)));
+    }
+    async chunkCode(code, sourceConfig, url, filePath, branch, repo) {
+        const logger = this.logger.child('code-chunker');
+        const normalizedPath = filePath.replace(/\\/g, '/');
+        const lang = this.detectCodeLanguage(filePath);
+        let chunks;
+        if (lang === 'markdown') {
+            const markdownChunks = await this.chunkMarkdown(code, sourceConfig, url);
+            for (const chunk of markdownChunks) {
+                if (normalizedPath) {
+                    const filePrefix = `[File: ${normalizedPath}]\n`;
+                    const searchableText = filePrefix + chunk.content;
+                    const chunkId = utils_1.Utils.generateHash(`${url}::${searchableText}`);
+                    chunk.content = searchableText;
+                    chunk.metadata.heading_hierarchy = [normalizedPath, ...chunk.metadata.heading_hierarchy.filter(Boolean)];
+                    chunk.metadata.section = normalizedPath;
+                    chunk.metadata.chunk_id = chunkId;
+                    chunk.metadata.hash = chunkId;
+                }
+                if (branch) {
+                    chunk.metadata.branch = branch;
+                }
+                if (repo) {
+                    chunk.metadata.repo = repo;
+                }
+            }
+            logger.debug(`Chunked ${normalizedPath || url}: ${markdownChunks.length} chunks created.`);
+            return markdownChunks;
+        }
+        if (lang) {
+            try {
+                const codeChunker = await this.getCodeChunker(lang, sourceConfig.chunk_size);
+                chunks = await codeChunker.chunk(code);
+            }
+            catch (error) {
+                logger.warn(`CodeChunker failed for ${normalizedPath || url}, falling back to token chunking:`, error);
+                const chunker = await this.getTokenChunker(sourceConfig.chunk_size);
+                chunks = await chunker.chunk(code);
+            }
+        }
+        else {
+            const chunker = await this.getTokenChunker(sourceConfig.chunk_size);
+            chunks = await chunker.chunk(code);
+        }
+        const documentChunks = [];
+        let chunkCounter = 0;
+        const headingHierarchy = normalizedPath ? [normalizedPath] : [];
+        const contextPrefix = normalizedPath ? `[File: ${normalizedPath}]\n` : '';
+        for (const chunk of chunks) {
+            const content = chunk.text?.trim();
+            if (!content) {
+                continue;
+            }
+            const searchableText = contextPrefix + content;
+            const chunkId = utils_1.Utils.generateHash(`${url}::${searchableText}`);
+            documentChunks.push({
+                content: searchableText,
+                metadata: {
+                    product_name: sourceConfig.product_name,
+                    version: sourceConfig.version,
+                    ...(branch ? { branch } : {}),
+                    ...(repo ? { repo } : {}),
+                    heading_hierarchy: headingHierarchy,
+                    section: normalizedPath || 'Code',
+                    chunk_id: chunkId,
+                    url: url,
+                    hash: chunkId,
+                    chunk_index: chunkCounter,
+                    total_chunks: 0
+                }
+            });
+            chunkCounter++;
+        }
+        const totalChunks = documentChunks.length;
+        for (const chunk of documentChunks) {
+            chunk.metadata.total_chunks = totalChunks;
+        }
+        logger.debug(`Chunked ${normalizedPath || url}: ${documentChunks.length} chunks created.`);
+        return documentChunks;
+    }
     async chunkMarkdown(markdown, sourceConfig, url) {
         const logger = this.logger.child('chunker');
-        logger.debug(`Chunking markdown from ${url} (${markdown.length} chars)`);
+        // --- Configuration ---
         const MAX_TOKENS = 1000;
+        const MIN_TOKENS = 150; // 💡 Merges "OpenAI-compatible" sentence into the next block
+        const OVERLAP_PERCENT = 0.1; // 10% overlap for large splits
         const chunks = [];
         const lines = markdown.split("\n");
-        let currentChunk = "";
+        let buffer = "";
         let headingHierarchy = [];
-        const processChunk = () => {
-            if (currentChunk.trim()) {
-                const tokens = utils_1.Utils.tokenize(currentChunk);
-                if (tokens.length > MAX_TOKENS) {
-                    logger.debug(`Chunk exceeds max token count (${tokens.length}), splitting into smaller chunks`);
-                    let subChunk = "";
-                    let tokenCount = 0;
-                    const overlapSize = Math.floor(MAX_TOKENS * 0.05);
-                    let lastTokens = [];
-                    for (const token of tokens) {
-                        if (tokenCount + 1 > MAX_TOKENS) {
-                            chunks.push(createDocumentChunk(subChunk, headingHierarchy));
-                            subChunk = lastTokens.join("") + token;
-                            tokenCount = lastTokens.length + 1;
-                            lastTokens = [];
-                        }
-                        else {
-                            subChunk += token;
-                            tokenCount++;
-                            lastTokens.push(token);
-                            if (lastTokens.length > overlapSize) {
-                                lastTokens.shift();
-                            }
-                        }
-                    }
-                    if (subChunk) {
-                        chunks.push(createDocumentChunk(subChunk, headingHierarchy));
-                    }
-                }
-                else {
-                    chunks.push(createDocumentChunk(currentChunk, headingHierarchy));
-                }
+        let bufferHeadings = []; // Track headings in current buffer
+        let chunkCounter = 0; // Tracks chunk position within this page for ordering
+        /**
+         * Computes the topic hierarchy for merged content.
+         * When merging sibling sections (same level), uses their parent heading.
+         * Otherwise uses the current hierarchy.
+         */
+        const computeTopicHierarchy = () => {
+            if (bufferHeadings.length === 0) {
+                return headingHierarchy;
+            }
+            // Find the deepest level (most recent headings)
+            const deepestLevel = Math.max(...bufferHeadings.map(h => h.level));
+            // Get all headings at the deepest level
+            const deepestHeadings = bufferHeadings.filter(h => h.level === deepestLevel);
+            // If we have multiple sibling headings at the deepest level, use their parent
+            if (deepestHeadings.length > 1 && deepestLevel > 1) {
+                // Use parent heading (one level up from the sibling headings)
+                // headingHierarchy still contains the parent at index (deepestLevel - 2)
+                // We want everything up to (but not including) the deepest level
+                return headingHierarchy.slice(0, deepestLevel - 1);
             }
-            currentChunk = "";
+            // Single heading or different levels: use the current hierarchy
+            // This reflects the most recent heading which is appropriate
+            return headingHierarchy;
         };
+        /**
+         * Internal helper to create the final chunk object with injected context.
+         */
         const createDocumentChunk = (content, hierarchy) => {
-            const chunkId = utils_1.Utils.generateHash(content);
-            logger.debug(`Created chunk ${chunkId.substring(0, 8)}... with ${content.length} chars`);
-            return {
-                content,
+            // 💡 BREADCRUMB INJECTION
+            // We prepend the hierarchy to the text. This makes the vector highly relevant 
+            // to searches for parent topics even if the body doesn't mention them.
+            const breadcrumbs = hierarchy.filter(h => h).join(" > ");
+            const contextPrefix = breadcrumbs ? `[Topic: ${breadcrumbs}]\n` : "";
+            const searchableText = contextPrefix + content.trim();
+            const chunkId = utils_1.Utils.generateHash(searchableText);
+            const chunk = {
+                content: searchableText,
                 metadata: {
                     product_name: sourceConfig.product_name,
                     version: sourceConfig.version,
-                    heading_hierarchy: [...hierarchy],
+                    heading_hierarchy: hierarchy.filter(h => h),
                     section: hierarchy[hierarchy.length - 1] || "Introduction",
                     chunk_id: chunkId,
                     url: url,
-                    hash: utils_1.Utils.generateHash(content)
+                    hash: chunkId,
+                    chunk_index: Math.floor(chunkCounter),
+                    total_chunks: 0 // Placeholder, will be updated after all chunks are created
                 }
             };
+            chunkCounter++; // Increment for next chunk
+            return chunk;
         };
+        /**
+         * Flushes the current buffer into the chunks array.
+         * Uses sub-splitting logic if the buffer exceeds MAX_TOKENS.
+         */
+        const flushBuffer = (force = false) => {
+            const trimmedBuffer = buffer.trim();
+            if (!trimmedBuffer)
+                return;
+            const tokenCount = utils_1.Utils.tokenize(trimmedBuffer).length;
+            // 💡 SEMANTIC MERGING
+            // If the current section is too short (like just a title or a one-liner),
+            // we don't flush yet unless it's the end of the file (force=true).
+            if (tokenCount < MIN_TOKENS && !force) {
+                return;
+            }
+            // Compute the appropriate topic hierarchy for merged content
+            const topicHierarchy = computeTopicHierarchy();
+            if (tokenCount > MAX_TOKENS) {
+                // 💡 RECURSIVE OVERLAP SPLITTING
+                // If the section is a massive guide, split it but keep headers on every sub-piece.
+                const tokens = utils_1.Utils.tokenize(trimmedBuffer);
+                const overlapSize = Math.floor(MAX_TOKENS * OVERLAP_PERCENT);
+                for (let i = 0; i < tokens.length; i += (MAX_TOKENS - overlapSize)) {
+                    const subTokens = tokens.slice(i, i + MAX_TOKENS);
+                    const subContent = subTokens.join("");
+                    chunks.push(createDocumentChunk(subContent, topicHierarchy));
+                }
+            }
+            else {
+                chunks.push(createDocumentChunk(trimmedBuffer, topicHierarchy));
+            }
+            buffer = ""; // Reset buffer after successful flush
+            bufferHeadings = []; // Reset tracked headings
+        };
+        // --- Main Processing Loop ---
         for (const line of lines) {
-            if (line.startsWith("#")) {
-                processChunk();
+            const isHeading = line.startsWith("#");
+            if (isHeading) {
+                // Update Hierarchy Stack for the new heading
                 const levelMatch = line.match(/^(#+)/);
-                let level = levelMatch ? levelMatch[1].length : 1;
-                const heading = line.replace(/^#+\s*/, "").trim();
-                logger.debug(`Found heading (level ${level}): ${heading}`);
-                while (headingHierarchy.length < level - 1) {
-                    headingHierarchy.push("");
-                }
-                if (level <= headingHierarchy.length) {
-                    headingHierarchy = headingHierarchy.slice(0, level - 1);
+                const level = levelMatch ? levelMatch[1].length : 1;
+                // Clean heading: remove markdown prefix and anchor links like [](#anchor-id)
+                const headingText = line
+                    .replace(/^#+\s*/, "") // Remove ## prefix
+                    .replace(/\[.*?\]\(#[^)]*\)/g, "") // Remove [text](#anchor) patterns
+                    .replace(/\[\]\(#[^)]*\)/g, "") // Remove [](#anchor) patterns  
+                    .trim();
+                // Check if we should merge with previous content
+                const currentTokenCount = utils_1.Utils.tokenize(buffer.trim()).length;
+                const hasBufferContent = currentTokenCount > 0;
+                const bufferIsSmall = currentTokenCount < MIN_TOKENS;
+                // Only merge if:
+                // 1. Buffer has content and is small
+                // 2. Buffer has tracked headings (we're merging sections, not just content)
+                // 3. New heading is at same or deeper level than the deepest heading in buffer (siblings or children)
+                //    If new heading is shallower (e.g., H2 after H3), it's a new section - flush first
+                const deepestBufferLevel = bufferHeadings.length > 0
+                    ? Math.max(...bufferHeadings.map(h => h.level))
+                    : 0;
+                const shouldMerge = hasBufferContent && bufferIsSmall && bufferHeadings.length > 0 &&
+                    level >= deepestBufferLevel;
+                if (!shouldMerge && hasBufferContent) {
+                    // Buffer is large enough OR new heading starts a new section - flush first
+                    flushBuffer();
                 }
-                headingHierarchy[level - 1] = heading;
+                // If shouldMerge is true, we keep the buffer and merge the sections
+                // Reset hierarchy below this level (e.g., H2 reset should clear previous H3s)
+                headingHierarchy = headingHierarchy.slice(0, level - 1);
+                headingHierarchy[level - 1] = headingText;
+                // Track this heading in the buffer
+                bufferHeadings.push({ level, text: headingText });
+                buffer += `${line}\n`;
             }
             else {
-                currentChunk += `${line}\n`;
+                buffer += `${line}\n`;
+                // Safety valve: if a single section is huge, flush it periodically
+                if (utils_1.Utils.tokenize(buffer).length >= MAX_TOKENS) {
+                    flushBuffer();
+                }
             }
         }
-        processChunk();
-        logger.debug(`Chunking complete, created ${chunks.length} chunks`);
+        // Final sweep
+        flushBuffer(true);
+        // Update all chunks with the final total count
+        const totalChunks = Math.floor(chunks.length);
+        for (const chunk of chunks) {
+            chunk.metadata.total_chunks = totalChunks;
+        }
+        logger.debug(`Chunking complete: ${chunks.length} rich context chunks created.`);
         return chunks;
     }
 }