doc2vec 2.0.0 → 2.3.0

package/dist/content-processor.js

@@ -47,6 +47,7 @@ const turndown_1 = __importDefault(require("turndown"));
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const utils_1 = require("./utils");
+const code_chunker_1 = require("./code-chunker");
 class ContentProcessor {
     constructor(logger) {
         this.logger = logger;
@@ -54,6 +55,9 @@ class ContentProcessor {
             codeBlockStyle: 'fenced',
             headingStyle: 'atx'
         });
+        this.tokenChunkerCache = new Map();
+        this.codeChunkerCache = new Map();
+        this.tokenizerCache = null;
         this.setupTurndownRules();
     }
     setupTurndownRules() {
@@ -200,15 +204,38 @@ class ContentProcessor {
     async crawlWebsite(baseUrl, sourceConfig, processPageContent, parentLogger, visitedUrls) {
         const logger = parentLogger.child('crawler');
         const queue = [baseUrl];
+        const brokenLinks = [];
+        const brokenLinkKeys = new Set();
+        const referrers = new Map();
+        const addReferrer = (targetUrl, sourceUrl) => {
+            const existing = referrers.get(targetUrl);
+            if (existing) {
+                existing.add(sourceUrl);
+            }
+            else {
+                referrers.set(targetUrl, new Set([sourceUrl]));
+            }
+        };
+        const addBrokenLink = (sourceUrl, targetUrl) => {
+            const key = `${sourceUrl} -> ${targetUrl}`;
+            if (brokenLinkKeys.has(key))
+                return;
+            brokenLinkKeys.add(key);
+            brokenLinks.push({ source: sourceUrl, target: targetUrl });
+        };
+        addReferrer(baseUrl, baseUrl);
         // Process sitemap if provided
         if (sourceConfig.sitemap_url) {
             logger.section('SITEMAP PROCESSING');
             const sitemapUrls = await this.parseSitemap(sourceConfig.sitemap_url, logger);
             // Add sitemap URLs to the queue if they're within the website scope
             for (const url of sitemapUrls) {
-                if (url.startsWith(sourceConfig.url) && !queue.includes(url)) {
-                    logger.debug(`Adding URL from sitemap to queue: ${url}`);
-                    queue.push(url);
+                if (url.startsWith(sourceConfig.url)) {
+                    addReferrer(url, sourceConfig.sitemap_url);
+                    if (!queue.includes(url)) {
+                        logger.debug(`Adding URL from sitemap to queue: ${url}`);
+                        queue.push(url);
+                    }
                 }
             }
             logger.info(`Added ${queue.length - 1} URLs from sitemap to the crawl queue`);
@@ -220,70 +247,126 @@ class ContentProcessor {
         let pdfProcessedCount = 0;
         let errorCount = 0;
         let hasNetworkErrors = false;
-        while (queue.length > 0) {
-            const url = queue.shift();
-            if (!url)
-                continue;
-            const normalizedUrl = utils_1.Utils.normalizeUrl(url);
-            if (visitedUrls.has(normalizedUrl))
-                continue;
-            visitedUrls.add(normalizedUrl);
-            if (!utils_1.Utils.shouldProcessUrl(url)) {
-                logger.debug(`Skipping URL with unsupported extension: ${url}`);
-                skippedCount++;
-                continue;
+        // Launch a single browser instance and reuse one page (tab) for all URLs
+        let browser = null;
+        let page = null;
+        const launchBrowser = async () => {
+            let executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
+            if (!executablePath) {
+                if (fs.existsSync('/usr/bin/chromium')) {
+                    executablePath = '/usr/bin/chromium';
+                }
+                else if (fs.existsSync('/usr/bin/chromium-browser')) {
+                    executablePath = '/usr/bin/chromium-browser';
+                }
             }
-            try {
-                logger.info(`Crawling: ${url}`);
-                const content = await this.processPage(url, sourceConfig);
-                if (content !== null) {
-                    await processPageContent(url, content);
-                    if (utils_1.Utils.isPdfUrl(url)) {
-                        pdfProcessedCount++;
+            const b = await puppeteer_1.default.launch({
+                executablePath,
+                args: ['--no-sandbox', '--disable-setuid-sandbox'],
+            });
+            const p = await b.newPage();
+            return { browser: b, page: p };
+        };
+        const ensureBrowser = async () => {
+            if (!browser || !browser.isConnected()) {
+                logger.info(browser ? 'Browser disconnected, relaunching...' : 'Launching browser...');
+                const launched = await launchBrowser();
+                browser = launched.browser;
+                page = launched.page;
+            }
+            return page;
+        };
+        try {
+            while (queue.length > 0) {
+                const url = queue.shift();
+                if (!url)
+                    continue;
+                const normalizedUrl = utils_1.Utils.normalizeUrl(url);
+                if (visitedUrls.has(normalizedUrl))
+                    continue;
+                visitedUrls.add(normalizedUrl);
+                if (!utils_1.Utils.shouldProcessUrl(url)) {
+                    logger.debug(`Skipping URL with unsupported extension: ${url}`);
+                    skippedCount++;
+                    continue;
+                }
+                try {
+                    logger.info(`Crawling: ${url}`);
+                    const sources = referrers.get(url) ?? new Set([baseUrl]);
+                    // For HTML pages, ensure the browser is running and pass the shared page
+                    // For PDFs, processPage handles them without Puppeteer
+                    const currentPage = utils_1.Utils.isPdfUrl(url) ? undefined : await ensureBrowser();
+                    const result = await this.processPage(url, sourceConfig, (reportedUrl, status) => {
+                        if (status === 404) {
+                            for (const source of sources) {
+                                addBrokenLink(source, reportedUrl);
+                            }
+                        }
+                    }, currentPage);
+                    if (result.content !== null) {
+                        await processPageContent(url, result.content);
+                        if (utils_1.Utils.isPdfUrl(url)) {
+                            pdfProcessedCount++;
+                        }
+                        else {
+                            processedCount++;
+                        }
                     }
                     else {
-                        processedCount++;
+                        skippedSizeCount++;
                     }
-                }
-                else {
-                    skippedSizeCount++;
-                }
-                // Only try to extract links from HTML pages, not PDFs
-                if (!utils_1.Utils.isPdfUrl(url)) {
-                    const response = await axios_1.default.get(url);
-                    const $ = (0, cheerio_1.load)(response.data);
-                    logger.debug(`Finding links on page ${url}`);
-                    let newLinksFound = 0;
-                    $('a[href]').each((_, element) => {
-                        const href = $(element).attr('href');
-                        if (!href || href.startsWith('#') || href.startsWith('mailto:'))
-                            return;
-                        const fullUrl = utils_1.Utils.buildUrl(href, url);
-                        if (fullUrl.startsWith(sourceConfig.url) && !visitedUrls.has(utils_1.Utils.normalizeUrl(fullUrl))) {
-                            if (!queue.includes(fullUrl)) {
-                                queue.push(fullUrl);
-                                newLinksFound++;
+                    // Use links extracted from the full rendered DOM by processPage
+                    // (no separate axios request needed)
+                    if (result.links.length > 0) {
+                        const pageUrlForLinks = result.finalUrl || url;
+                        logger.debug(`Finding links on page ${url}`);
+                        let newLinksFound = 0;
+                        for (const href of result.links) {
+                            const fullUrl = utils_1.Utils.buildUrl(href, pageUrlForLinks);
+                            if (fullUrl.startsWith(sourceConfig.url)) {
+                                addReferrer(fullUrl, pageUrlForLinks);
+                                if (!visitedUrls.has(utils_1.Utils.normalizeUrl(fullUrl))) {
+                                    if (!queue.includes(fullUrl)) {
+                                        queue.push(fullUrl);
+                                        newLinksFound++;
+                                    }
+                                }
                             }
                         }
-                    });
-                    logger.debug(`Found ${newLinksFound} new links on ${url}`);
+                        logger.debug(`Found ${newLinksFound} new links on ${url}`);
+                    }
                 }
-            }
-            catch (error) {
-                logger.error(`Failed during processing or link discovery for ${url}:`, error);
-                errorCount++;
-                // Check if this is a network error (DNS resolution, connection issues, etc.)
-                if (this.isNetworkError(error)) {
-                    hasNetworkErrors = true;
-                    logger.warn(`Network error detected for ${url}, this may affect cleanup decisions`);
+                catch (error) {
+                    logger.error(`Failed during processing or link discovery for ${url}:`, error);
+                    errorCount++;
+                    const status = this.getHttpStatus(error);
+                    if (status === 404) {
+                        const sources = referrers.get(url) ?? new Set([baseUrl]);
+                        for (const source of sources) {
+                            addBrokenLink(source, url);
+                        }
+                    }
+                    // Check if this is a network error (DNS resolution, connection issues, etc.)
+                    if (this.isNetworkError(error)) {
+                        hasNetworkErrors = true;
+                        logger.warn(`Network error detected for ${url}, this may affect cleanup decisions`);
+                    }
                 }
             }
         }
+        finally {
+            // Close the shared browser instance when the crawl is done
+            const browserToClose = browser;
+            if (browserToClose && browserToClose.isConnected()) {
+                await browserToClose.close();
+                logger.debug('Shared browser closed after crawl completed');
+            }
+        }
         logger.info(`Crawl completed. HTML Pages: ${processedCount}, PDFs: ${pdfProcessedCount}, Skipped (Extension): ${skippedCount}, Skipped (Size): ${skippedSizeCount}, Errors: ${errorCount}`);
         if (hasNetworkErrors) {
             logger.warn('Network errors were encountered during crawling. Cleanup may be skipped to avoid removing valid chunks.');
         }
-        return { hasNetworkErrors };
+        return { hasNetworkErrors, brokenLinks };
     }
     isNetworkError(error) {
         // Check for common network error patterns
@@ -324,7 +407,7 @@ class ContentProcessor {
         }
         return false;
     }
-    async processPage(url, sourceConfig) {
+    async processPage(url, sourceConfig, onHttpStatus, existingPage) {
         const logger = this.logger.child('page-processor');
         logger.debug(`Processing content from ${url}`);
         // Check if this is a PDF URL
@@ -335,37 +418,74 @@ class ContentProcessor {
                 // Check size limit for PDF content
                 if (markdown.length > sourceConfig.max_size) {
                     logger.warn(`PDF content (${markdown.length} chars) exceeds max size (${sourceConfig.max_size}). Skipping ${url}.`);
-                    return null;
+                    return { content: null, links: [], finalUrl: url };
                 }
-                return markdown;
+                return { content: markdown, links: [], finalUrl: url };
             }
             catch (error) {
+                const status = this.getHttpStatus(error);
+                if (status !== undefined && status >= 400) {
+                    if (onHttpStatus) {
+                        onHttpStatus(url, status);
+                    }
+                    throw error;
+                }
                 logger.error(`Failed to process PDF ${url}:`, error);
-                return null;
+                return { content: null, links: [], finalUrl: url };
             }
         }
-        // Original HTML page processing logic
+        // HTML page processing logic
+        // If an existing page (tab) is provided, reuse it; otherwise launch a standalone browser
         let browser = null;
+        let page;
+        const ownsTheBrowser = !existingPage;
         try {
-            // Use system Chromium if available (for Docker environments)
-            let executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
-            if (!executablePath) {
-                if (fs.existsSync('/usr/bin/chromium')) {
-                    executablePath = '/usr/bin/chromium';
-                }
-                else if (fs.existsSync('/usr/bin/chromium-browser')) {
-                    executablePath = '/usr/bin/chromium-browser';
+            if (existingPage) {
+                page = existingPage;
+            }
+            else {
+                // Standalone mode: launch a browser for this single page
+                let executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
+                if (!executablePath) {
+                    if (fs.existsSync('/usr/bin/chromium')) {
+                        executablePath = '/usr/bin/chromium';
+                    }
+                    else if (fs.existsSync('/usr/bin/chromium-browser')) {
+                        executablePath = '/usr/bin/chromium-browser';
+                    }
                 }
+                browser = await puppeteer_1.default.launch({
+                    executablePath,
+                    args: ['--no-sandbox', '--disable-setuid-sandbox'],
+                });
+                page = await browser.newPage();
             }
-            browser = await puppeteer_1.default.launch({
-                executablePath,
-                args: ['--no-sandbox', '--disable-setuid-sandbox'],
-            });
-            const page = await browser.newPage();
             logger.debug(`Navigating to ${url}`);
-            await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
+            const response = await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
+            const status = response?.status();
+            if (status !== undefined && status >= 400) {
+                const error = new Error(`Failed to load page: HTTP ${status}`);
+                error.status = status;
+                throw error;
+            }
+            // Get the final URL after any redirects
+            const finalUrl = page.url();
+            // Extract ALL links from the full rendered DOM before any content filtering
+            // This searches the entire document, not just the main content area
+            const links = await page.evaluate(() => {
+                const anchors = document.querySelectorAll('a[href]');
+                const hrefs = [];
+                anchors.forEach(a => {
+                    const href = a.getAttribute('href');
+                    if (href && !href.startsWith('#') && !href.startsWith('mailto:')) {
+                        hrefs.push(href);
+                    }
+                });
+                return hrefs;
+            });
+            logger.debug(`Extracted ${links.length} links from full DOM of ${url}`);
             const htmlContent = await page.evaluate(() => {
-                // 💡 Try specific content selectors first, then fall back to broader ones
+                // Try specific content selectors first, then fall back to broader ones
                 const mainContentElement = document.querySelector('.docs-content') || // Common docs pattern
                     document.querySelector('.doc-content') || // Alternative docs pattern
                     document.querySelector('.markdown-body') || // GitHub-style
@@ -377,8 +497,7 @@ class ContentProcessor {
             });
             if (htmlContent.length > sourceConfig.max_size) {
                 logger.warn(`Raw HTML content (${htmlContent.length} chars) exceeds max size (${sourceConfig.max_size}). Skipping detailed processing for ${url}.`);
-                await browser.close();
-                return null;
+                return { content: null, links, finalUrl };
             }
             logger.debug(`Got HTML content (${htmlContent.length} chars), creating DOM`);
             const dom = new jsdom_1.JSDOM(htmlContent);
@@ -388,7 +507,7 @@ class ContentProcessor {
                 pre.setAttribute('data-readable-content-score', '100');
                 this.markCodeParents(pre.parentElement);
             });
-            // 💡 Extract H1s BEFORE Readability - it often strips them as "chrome"
+            // Extract H1s BEFORE Readability - it often strips them as "chrome"
             // We'll inject them back after Readability processing
             const h1Elements = document.querySelectorAll('h1');
             const extractedH1s = [];
@@ -410,8 +529,7 @@ class ContentProcessor {
             const article = reader.parse();
             if (!article) {
                 logger.warn(`Failed to parse article content with Readability for ${url}`);
-                await browser.close();
-                return null;
+                return { content: null, links, finalUrl };
             }
             // Debug: Log what Readability extracted
             logger.debug(`[Readability Debug] article.title: "${article.title}"`);
@@ -420,7 +538,7 @@ class ContentProcessor {
             logger.debug(`[Readability Debug] Contains H1 tag: ${article.content?.includes('<h1')}`);
             logger.debug(`[Readability Debug] Contains H2 tag: ${article.content?.includes('<h2')}`);
             logger.debug(`[Readability Debug] Contains original-h1 class: ${article.content?.includes('original-h1')}`);
-            // 💡 Restore H1s: find elements with our marker class and convert back from H2
+            // Restore H1s: find elements with our marker class and convert back from H2
             const articleDom = new jsdom_1.JSDOM(article.content);
             const articleDoc = articleDom.window.document;
             const originalH1Elements = articleDoc.querySelectorAll('.original-h1');
@@ -457,7 +575,7 @@ class ContentProcessor {
             });
             logger.debug(`Converting HTML to Markdown`);
             let markdown = this.turndownService.turndown(cleanHtml);
-            // 💡 Inject extracted H1s back if they're not in the markdown
+            // Inject extracted H1s back if they're not in the markdown
             // Readability often strips them as "page chrome"
             // Use article.title as fallback if no H1 was extracted
             const pageTitle = extractedH1s.length > 0 ? extractedH1s[0] : (article.title?.trim() || '');
@@ -477,19 +595,36 @@ class ContentProcessor {
                 }
             }
             logger.debug(`Markdown conversion complete (${markdown.length} chars)`);
-            return markdown;
+            return { content: markdown, links, finalUrl };
         }
         catch (error) {
+            const status = this.getHttpStatus(error);
+            if (status !== undefined && status >= 400) {
+                if (onHttpStatus) {
+                    onHttpStatus(url, status);
+                }
+                throw error;
+            }
             logger.error(`Error processing page ${url}:`, error);
-            return null;
+            return { content: null, links: [], finalUrl: url };
         }
         finally {
-            if (browser && browser.isConnected()) {
+            // Only close the browser if we launched it ourselves (standalone mode)
+            if (ownsTheBrowser && browser && browser.isConnected()) {
                 await browser.close();
                 logger.debug(`Browser closed for ${url}`);
             }
         }
     }
+    getHttpStatus(error) {
+        if (typeof error?.status === 'number') {
+            return error.status;
+        }
+        if (typeof error?.response?.status === 'number') {
+            return error.response.status;
+        }
+        return undefined;
+    }
     markCodeParents(node) {
         if (!node)
             return;
@@ -812,6 +947,257 @@ class ContentProcessor {
             logger.error(`Error reading directory ${dirPath}:`, error);
         }
     }
+    async processCodeDirectory(dirPath, config, processFileContent, parentLogger, visitedPaths = new Set(), options) {
+        const logger = parentLogger.child('code-directory-processor');
+        logger.info(`Processing code directory: ${dirPath}`);
+        const recursive = config.recursive !== undefined ? config.recursive : true;
+        const includeExtensions = config.include_extensions || [
+            '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
+            '.py', '.go', '.rs', '.java', '.kt', '.kts', '.swift',
+            '.c', '.cc', '.cpp', '.h', '.hpp', '.cs',
+            '.rb', '.php', '.scala', '.sql', '.sh', '.bash', '.zsh',
+            '.html', '.css', '.scss', '.sass', '.less',
+            '.json', '.yaml', '.yml', '.md'
+        ];
+        const excludeExtensions = config.exclude_extensions || [];
+        const encoding = config.encoding || 'utf8';
+        let maxMtime = 0;
+        try {
+            const files = fs.readdirSync(dirPath);
+            let processedFiles = 0;
+            let skippedFiles = 0;
+            const allowedFiles = options?.allowedFiles;
+            const mtimeCutoff = options?.mtimeCutoff;
+            const trackFiles = options?.trackFiles;
+            for (const file of files) {
+                const filePath = path.join(dirPath, file);
+                const stat = fs.statSync(filePath);
+                if (visitedPaths.has(filePath)) {
+                    logger.debug(`Skipping already visited path: ${filePath}`);
+                    continue;
+                }
+                visitedPaths.add(filePath);
+                if (stat.isDirectory()) {
+                    if (recursive) {
+                        const childResult = await this.processCodeDirectory(filePath, config, processFileContent, logger, visitedPaths, options);
+                        processedFiles += childResult.processedFiles;
+                        skippedFiles += childResult.skippedFiles;
+                        maxMtime = Math.max(maxMtime, childResult.maxMtime);
+                    }
+                    else {
+                        logger.debug(`Skipping directory ${filePath} (recursive=false)`);
+                    }
+                }
+                else if (stat.isFile()) {
+                    const extension = path.extname(file).toLowerCase();
+                    if (excludeExtensions.includes(extension)) {
+                        logger.debug(`Skipping file with excluded extension: ${filePath}`);
+                        skippedFiles++;
+                        continue;
+                    }
+                    if (includeExtensions.length > 0 && !includeExtensions.includes(extension)) {
+                        logger.debug(`Skipping file with non-included extension: ${filePath}`);
+                        skippedFiles++;
+                        continue;
+                    }
+                    trackFiles?.add(filePath);
+                    maxMtime = Math.max(maxMtime, stat.mtimeMs);
+                    if (allowedFiles && !allowedFiles.has(filePath)) {
+                        skippedFiles++;
+                        continue;
+                    }
+                    if (mtimeCutoff !== undefined && stat.mtimeMs <= mtimeCutoff) {
+                        skippedFiles++;
+                        continue;
+                    }
+                    try {
+                        logger.info(`Reading file: ${filePath}`);
+                        const content = fs.readFileSync(filePath, { encoding: encoding });
+                        if (content.length > config.max_size) {
+                            logger.warn(`File content (${content.length} chars) exceeds max size (${config.max_size}). Skipping ${filePath}.`);
+                            skippedFiles++;
+                            continue;
+                        }
+                        await processFileContent(filePath, content);
+                        processedFiles++;
+                    }
+                    catch (error) {
+                        logger.error(`Error processing file ${filePath}:`, error);
+                    }
+                }
+            }
+            logger.info(`Code directory processed. Processed: ${processedFiles}, Skipped: ${skippedFiles}`);
+            return { processedFiles, skippedFiles, maxMtime };
+        }
+        catch (error) {
+            logger.error(`Error reading code directory ${dirPath}:`, error);
+            return { processedFiles: 0, skippedFiles: 0, maxMtime };
+        }
+    }
+    async getTokenChunker(chunkSize) {
+        const cacheKey = `${chunkSize || 'default'}`;
+        const cached = this.tokenChunkerCache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const chunkerPromise = (async () => {
+            const { TokenChunker } = await this.importChonkieModule('@chonkiejs/core');
+            return await TokenChunker.create({ chunkSize, tokenizer: 'character' });
+        })();
+        this.tokenChunkerCache.set(cacheKey, chunkerPromise);
+        return chunkerPromise;
+    }
+    async getCodeChunker(lang, chunkSize) {
+        const cacheKey = `${lang}:${chunkSize || 'default'}`;
+        const cached = this.codeChunkerCache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const chunkerPromise = (async () => {
+            const tokenizer = await this.getTokenizer();
+            return await code_chunker_1.CodeChunker.create({
+                lang,
+                chunkSize,
+                tokenCounter: async (text) => tokenizer.countTokens(text)
+            });
+        })();
+        this.codeChunkerCache.set(cacheKey, chunkerPromise);
+        return chunkerPromise;
+    }
+    async getTokenizer() {
+        if (!this.tokenizerCache) {
+            this.tokenizerCache = (async () => {
+                const { Tokenizer } = await this.importChonkieModule('@chonkiejs/core');
+                return await Tokenizer.create('character');
+            })();
+        }
+        return this.tokenizerCache;
+    }
+    detectCodeLanguage(filePath) {
+        const extension = path.extname(filePath).toLowerCase();
+        const languageMap = {
+            '.ts': 'typescript',
+            '.tsx': 'typescript',
+            '.js': 'javascript',
+            '.jsx': 'javascript',
+            '.mjs': 'javascript',
+            '.cjs': 'javascript',
+            '.py': 'python',
+            '.go': 'go',
+            '.rs': 'rust',
+            '.java': 'java',
+            '.kt': 'kotlin',
+            '.kts': 'kotlin',
+            '.swift': 'swift',
+            '.c': 'c',
+            '.cc': 'cpp',
+            '.cpp': 'cpp',
+            '.h': 'cpp',
+            '.hpp': 'cpp',
+            '.cs': 'csharp',
+            '.rb': 'ruby',
+            '.php': 'php',
+            '.scala': 'scala',
+            '.sql': 'sql',
+            '.sh': 'bash',
+            '.bash': 'bash',
+            '.zsh': 'bash',
+            '.html': 'html',
+            '.css': 'css',
+            '.scss': 'scss',
+            '.sass': 'scss',
+            '.less': 'css',
+            '.json': 'json',
+            '.yaml': 'yaml',
+            '.yml': 'yaml',
+            '.md': 'markdown'
+        };
+        return languageMap[extension];
+    }
+    async importChonkieModule(specifier) {
+        // Use dynamic import() directly - preserved by TypeScript with target ES2020+
+        // even in CommonJS mode, as Node.js supports import() in CJS contexts
+        return Promise.resolve(`${specifier}`).then(s => __importStar(require(s)));
+    }
+    async chunkCode(code, sourceConfig, url, filePath, branch, repo) {
+        const logger = this.logger.child('code-chunker');
+        const normalizedPath = filePath.replace(/\\/g, '/');
+        const lang = this.detectCodeLanguage(filePath);
+        let chunks;
+        if (lang === 'markdown') {
+            const markdownChunks = await this.chunkMarkdown(code, sourceConfig, url);
+            for (const chunk of markdownChunks) {
+                if (normalizedPath) {
+                    const filePrefix = `[File: ${normalizedPath}]\n`;
+                    const searchableText = filePrefix + chunk.content;
+                    const chunkId = utils_1.Utils.generateHash(`${url}::${searchableText}`);
+                    chunk.content = searchableText;
+                    chunk.metadata.heading_hierarchy = [normalizedPath, ...chunk.metadata.heading_hierarchy.filter(Boolean)];
+                    chunk.metadata.section = normalizedPath;
+                    chunk.metadata.chunk_id = chunkId;
+                    chunk.metadata.hash = chunkId;
+                }
+                if (branch) {
+                    chunk.metadata.branch = branch;
+                }
+                if (repo) {
+                    chunk.metadata.repo = repo;
+                }
+            }
+            logger.debug(`Chunked ${normalizedPath || url}: ${markdownChunks.length} chunks created.`);
+            return markdownChunks;
+        }
+        if (lang) {
+            try {
+                const codeChunker = await this.getCodeChunker(lang, sourceConfig.chunk_size);
+                chunks = await codeChunker.chunk(code);
+            }
+            catch (error) {
+                logger.warn(`CodeChunker failed for ${normalizedPath || url}, falling back to token chunking:`, error);
+                const chunker = await this.getTokenChunker(sourceConfig.chunk_size);
+                chunks = await chunker.chunk(code);
+            }
+        }
+        else {
+            const chunker = await this.getTokenChunker(sourceConfig.chunk_size);
+            chunks = await chunker.chunk(code);
+        }
+        const documentChunks = [];
+        let chunkCounter = 0;
+        const headingHierarchy = normalizedPath ? [normalizedPath] : [];
+        const contextPrefix = normalizedPath ? `[File: ${normalizedPath}]\n` : '';
+        for (const chunk of chunks) {
+            const content = chunk.text?.trim();
+            if (!content) {
+                continue;
+            }
+            const searchableText = contextPrefix + content;
+            const chunkId = utils_1.Utils.generateHash(`${url}::${searchableText}`);
+            documentChunks.push({
+                content: searchableText,
+                metadata: {
+                    product_name: sourceConfig.product_name,
+                    version: sourceConfig.version,
+                    ...(branch ? { branch } : {}),
+                    ...(repo ? { repo } : {}),
+                    heading_hierarchy: headingHierarchy,
+                    section: normalizedPath || 'Code',
+                    chunk_id: chunkId,
+                    url: url,
+                    hash: chunkId,
+                    chunk_index: chunkCounter,
+                    total_chunks: 0
+                }
+            });
+            chunkCounter++;
+        }
+        const totalChunks = documentChunks.length;
+        for (const chunk of documentChunks) {
+            chunk.metadata.total_chunks = totalChunks;
+        }
+        logger.debug(`Chunked ${normalizedPath || url}: ${documentChunks.length} chunks created.`);
+        return documentChunks;
+    }
     async chunkMarkdown(markdown, sourceConfig, url) {
         const logger = this.logger.child('chunker');
         // --- Configuration ---