doc2vec 1.1.0 → 1.2.0

package/dist/content-processor.js

@@ -0,0 +1,725 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ContentProcessor = void 0;
+const readability_1 = require("@mozilla/readability");
+const axios_1 = __importDefault(require("axios"));
+const cheerio_1 = require("cheerio");
+const jsdom_1 = require("jsdom");
+const puppeteer_1 = __importDefault(require("puppeteer"));
+const sanitize_html_1 = __importDefault(require("sanitize-html"));
+const turndown_1 = __importDefault(require("turndown"));
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const utils_1 = require("./utils");
+class ContentProcessor {
+    constructor(logger) {
+        this.logger = logger;
+        this.turndownService = new turndown_1.default({
+            codeBlockStyle: 'fenced',
+            headingStyle: 'atx'
+        });
+        this.setupTurndownRules();
+    }
+    setupTurndownRules() {
+        const logger = this.logger.child('markdown');
+        logger.debug('Setting up Turndown rules for markdown conversion');
+        this.turndownService.addRule('codeBlocks', {
+            filter: (node) => node.nodeName === 'PRE',
+            replacement: (content, node) => {
+                const htmlNode = node;
+                const code = htmlNode.querySelector('code');
+                let codeContent;
+                if (code) {
+                    codeContent = code.textContent || '';
+                }
+                else {
+                    codeContent = htmlNode.textContent || '';
+                }
+                const lines = codeContent.split('\n');
+                let minIndent = Infinity;
+                for (const line of lines) {
+                    if (line.trim() === '')
+                        continue;
+                    const leadingWhitespace = line.match(/^\s*/)?.[0] || '';
+                    minIndent = Math.min(minIndent, leadingWhitespace.length);
+                }
+                const cleanedLines = lines.map(line => {
+                    return line.substring(minIndent);
+                });
+                let cleanContent = cleanedLines.join('\n');
+                cleanContent = cleanContent.replace(/^\s+|\s+$/g, '');
+                cleanContent = cleanContent.replace(/\n{2,}/g, '\n');
+                return `\n\`\`\`\n${cleanContent}\n\`\`\`\n`;
+            }
+        });
+        this.turndownService.addRule('tableCell', {
+            filter: ['th', 'td'],
+            replacement: (content, node) => {
+                const htmlNode = node;
+                let cellContent = '';
+                if (htmlNode.querySelector('p')) {
+                    cellContent = Array.from(htmlNode.querySelectorAll('p'))
+                        .map(p => p.textContent || '')
+                        .join(' ')
+                        .trim();
+                }
+                else {
+                    cellContent = content.trim();
+                }
+                return ` ${cellContent.replace(/\|/g, '\\|')} |`;
+            }
+        });
+        this.turndownService.addRule('tableRow', {
+            filter: 'tr',
+            replacement: (content, node) => {
+                const htmlNode = node;
+                const cells = Array.from(htmlNode.cells);
+                const isHeader = htmlNode.parentNode?.nodeName === 'THEAD';
+                let output = '|' + content.trimEnd();
+                if (isHeader) {
+                    const separator = cells.map(() => '---').join(' | ');
+                    output += '\n|' + separator + '|';
+                }
+                if (!isHeader || !htmlNode.nextElementSibling) {
+                    output += '\n';
+                }
+                return output;
+            }
+        });
+        this.turndownService.addRule('table', {
+            filter: 'table',
+            replacement: (content) => {
+                return '\n' + content.replace(/\n+/g, '\n').trim() + '\n';
+            }
+        });
+        this.turndownService.addRule('preserveTableWhitespace', {
+            filter: (node) => {
+                return ((node.nodeName === 'TD' || node.nodeName === 'TH') &&
+                    (node.textContent?.trim().length === 0));
+            },
+            replacement: () => {
+                return ' |';
+            }
+        });
+        logger.debug('Turndown rules setup complete');
+    }
+    async parseSitemap(sitemapUrl, logger) {
+        logger.info(`Parsing sitemap from ${sitemapUrl}`);
+        try {
+            const response = await axios_1.default.get(sitemapUrl);
+            const $ = (0, cheerio_1.load)(response.data, { xmlMode: true });
+            const urls = [];
+            // Handle standard sitemaps
+            $('url > loc').each((_, element) => {
+                const url = $(element).text().trim();
+                if (url) {
+                    urls.push(url);
+                }
+            });
+            // Handle sitemap indexes (sitemaps that link to other sitemaps)
+            const sitemapLinks = [];
+            $('sitemap > loc').each((_, element) => {
+                const nestedSitemapUrl = $(element).text().trim();
+                if (nestedSitemapUrl) {
+                    sitemapLinks.push(nestedSitemapUrl);
+                }
+            });
+            // Recursively process nested sitemaps
+            for (const nestedSitemapUrl of sitemapLinks) {
+                logger.debug(`Found nested sitemap: ${nestedSitemapUrl}`);
+                const nestedUrls = await this.parseSitemap(nestedSitemapUrl, logger);
+                urls.push(...nestedUrls);
+            }
+            logger.info(`Found ${urls.length} URLs in sitemap ${sitemapUrl}`);
+            return urls;
+        }
+        catch (error) {
+            logger.error(`Error parsing sitemap at ${sitemapUrl}:`, error);
+            return [];
+        }
+    }
+    async crawlWebsite(baseUrl, sourceConfig, processPageContent, parentLogger, visitedUrls) {
+        const logger = parentLogger.child('crawler');
+        const queue = [baseUrl];
+        // Process sitemap if provided
+        if (sourceConfig.sitemap_url) {
+            logger.section('SITEMAP PROCESSING');
+            const sitemapUrls = await this.parseSitemap(sourceConfig.sitemap_url, logger);
+            // Add sitemap URLs to the queue if they're within the website scope
+            for (const url of sitemapUrls) {
+                if (url.startsWith(sourceConfig.url) && !queue.includes(url)) {
+                    logger.debug(`Adding URL from sitemap to queue: ${url}`);
+                    queue.push(url);
+                }
+            }
+            logger.info(`Added ${queue.length - 1} URLs from sitemap to the crawl queue`);
+        }
+        logger.info(`Starting crawl from ${baseUrl} with ${queue.length} URLs in initial queue`);
+        let processedCount = 0;
+        let skippedCount = 0;
+        let skippedSizeCount = 0;
+        let pdfProcessedCount = 0;
+        let errorCount = 0;
+        let hasNetworkErrors = false;
+        while (queue.length > 0) {
+            const url = queue.shift();
+            if (!url)
+                continue;
+            const normalizedUrl = utils_1.Utils.normalizeUrl(url);
+            if (visitedUrls.has(normalizedUrl))
+                continue;
+            visitedUrls.add(normalizedUrl);
+            if (!utils_1.Utils.shouldProcessUrl(url)) {
+                logger.debug(`Skipping URL with unsupported extension: ${url}`);
+                skippedCount++;
+                continue;
+            }
+            try {
+                logger.info(`Crawling: ${url}`);
+                const content = await this.processPage(url, sourceConfig);
+                if (content !== null) {
+                    await processPageContent(url, content);
+                    if (utils_1.Utils.isPdfUrl(url)) {
+                        pdfProcessedCount++;
+                    }
+                    else {
+                        processedCount++;
+                    }
+                }
+                else {
+                    skippedSizeCount++;
+                }
+                // Only try to extract links from HTML pages, not PDFs
+                if (!utils_1.Utils.isPdfUrl(url)) {
+                    const response = await axios_1.default.get(url);
+                    const $ = (0, cheerio_1.load)(response.data);
+                    logger.debug(`Finding links on page ${url}`);
+                    let newLinksFound = 0;
+                    $('a[href]').each((_, element) => {
+                        const href = $(element).attr('href');
+                        if (!href || href.startsWith('#') || href.startsWith('mailto:'))
+                            return;
+                        const fullUrl = utils_1.Utils.buildUrl(href, url);
+                        if (fullUrl.startsWith(sourceConfig.url) && !visitedUrls.has(utils_1.Utils.normalizeUrl(fullUrl))) {
+                            if (!queue.includes(fullUrl)) {
+                                queue.push(fullUrl);
+                                newLinksFound++;
+                            }
+                        }
+                    });
+                    logger.debug(`Found ${newLinksFound} new links on ${url}`);
+                }
+            }
+            catch (error) {
+                logger.error(`Failed during processing or link discovery for ${url}:`, error);
+                errorCount++;
+                // Check if this is a network error (DNS resolution, connection issues, etc.)
+                if (this.isNetworkError(error)) {
+                    hasNetworkErrors = true;
+                    logger.warn(`Network error detected for ${url}, this may affect cleanup decisions`);
+                }
+            }
+        }
+        logger.info(`Crawl completed. HTML Pages: ${processedCount}, PDFs: ${pdfProcessedCount}, Skipped (Extension): ${skippedCount}, Skipped (Size): ${skippedSizeCount}, Errors: ${errorCount}`);
+        if (hasNetworkErrors) {
+            logger.warn('Network errors were encountered during crawling. Cleanup may be skipped to avoid removing valid chunks.');
+        }
+        return { hasNetworkErrors };
+    }
+    isNetworkError(error) {
+        // Check for common network error patterns
+        if (error?.code) {
+            // DNS resolution errors
+            if (error.code === 'ENOTFOUND')
+                return true;
+            // Connection refused
+            if (error.code === 'ECONNREFUSED')
+                return true;
+            // Connection timeout
+            if (error.code === 'ETIMEDOUT')
+                return true;
+            // Connection reset
+            if (error.code === 'ECONNRESET')
+                return true;
+            // Host unreachable
+            if (error.code === 'EHOSTUNREACH')
+                return true;
+            // Network unreachable
+            if (error.code === 'ENETUNREACH')
+                return true;
+        }
+        // Check for axios-specific network errors
+        if (error?.isAxiosError) {
+            // If there's no response, it's likely a network error
+            if (!error.response)
+                return true;
+        }
+        // Check error message for network-related terms
+        const errorMessage = error?.message?.toLowerCase() || '';
+        if (errorMessage.includes('getaddrinfo') ||
+            errorMessage.includes('network') ||
+            errorMessage.includes('timeout') ||
+            errorMessage.includes('connection') ||
+            errorMessage.includes('dns')) {
+            return true;
+        }
+        return false;
+    }
+    async processPage(url, sourceConfig) {
+        const logger = this.logger.child('page-processor');
+        logger.debug(`Processing content from ${url}`);
+        // Check if this is a PDF URL
+        if (utils_1.Utils.isPdfUrl(url)) {
+            logger.info(`Processing PDF: ${url}`);
+            try {
+                const markdown = await this.downloadAndConvertPdfFromUrl(url, logger);
+                // Check size limit for PDF content
+                if (markdown.length > sourceConfig.max_size) {
+                    logger.warn(`PDF content (${markdown.length} chars) exceeds max size (${sourceConfig.max_size}). Skipping ${url}.`);
+                    return null;
+                }
+                return markdown;
+            }
+            catch (error) {
+                logger.error(`Failed to process PDF ${url}:`, error);
+                return null;
+            }
+        }
+        // Original HTML page processing logic
+        let browser = null;
+        try {
+            browser = await puppeteer_1.default.launch({
+                args: ['--no-sandbox', '--disable-setuid-sandbox'],
+            });
+            const page = await browser.newPage();
+            logger.debug(`Navigating to ${url}`);
+            await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
+            const htmlContent = await page.evaluate(() => {
+                const mainContentElement = document.querySelector('div[role="main"].document') || document.querySelector('main') || document.body;
+                return mainContentElement.innerHTML;
+            });
+            if (htmlContent.length > sourceConfig.max_size) {
+                logger.warn(`Raw HTML content (${htmlContent.length} chars) exceeds max size (${sourceConfig.max_size}). Skipping detailed processing for ${url}.`);
+                await browser.close();
+                return null;
+            }
+            logger.debug(`Got HTML content (${htmlContent.length} chars), creating DOM`);
+            const dom = new jsdom_1.JSDOM(htmlContent);
+            const document = dom.window.document;
+            document.querySelectorAll('pre').forEach((pre) => {
+                pre.classList.add('article-content');
+                pre.setAttribute('data-readable-content-score', '100');
+                this.markCodeParents(pre.parentElement);
+            });
+            logger.debug(`Applying Readability to extract main content`);
+            const reader = new readability_1.Readability(document, {
+                charThreshold: 20,
+                classesToPreserve: ['article-content'],
+            });
+            const article = reader.parse();
+            if (!article) {
+                logger.warn(`Failed to parse article content with Readability for ${url}`);
+                await browser.close();
+                return null;
+            }
+            logger.debug(`Sanitizing HTML (${article.content.length} chars)`);
+            const cleanHtml = (0, sanitize_html_1.default)(article.content, {
+                allowedTags: [
+                    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
+                    'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
+                    'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td'
+                ],
+                allowedAttributes: {
+                    'a': ['href'],
+                    'pre': ['class', 'data-language'],
+                    'code': ['class', 'data-language'],
+                    'div': ['class'],
+                    'span': ['class']
+                }
+            });
+            logger.debug(`Converting HTML to Markdown`);
+            const markdown = this.turndownService.turndown(cleanHtml);
+            logger.debug(`Markdown conversion complete (${markdown.length} chars)`);
+            return markdown;
+        }
+        catch (error) {
+            logger.error(`Error processing page ${url}:`, error);
+            return null;
+        }
+        finally {
+            if (browser && browser.isConnected()) {
+                await browser.close();
+                logger.debug(`Browser closed for ${url}`);
+            }
+        }
+    }
+    markCodeParents(node) {
+        if (!node)
+            return;
+        if (node.querySelector('pre, code')) {
+            node.classList.add('article-content');
+            node.setAttribute('data-readable-content-score', '100');
+        }
+        this.markCodeParents(node.parentElement);
+    }
+    async convertPdfToMarkdown(filePath, logger) {
+        logger.debug(`Converting PDF to markdown: ${filePath}`);
+        try {
+            // Dynamic import for PDF.js to handle ES module compatibility
+            const pdfjsLib = await Promise.resolve().then(() => __importStar(require('pdfjs-dist/legacy/build/pdf.mjs')));
+            // Read the PDF file as a buffer and convert to Uint8Array
+            const pdfBuffer = fs.readFileSync(filePath);
+            const pdfData = new Uint8Array(pdfBuffer);
+            // Load the PDF document
+            const loadingTask = pdfjsLib.getDocument({
+                data: pdfData,
+                // Disable worker to avoid issues in Node.js environment
+                useWorkerFetch: false,
+                isEvalSupported: false,
+                useSystemFonts: true
+            });
+            const pdfDocument = await loadingTask.promise;
+            const numPages = pdfDocument.numPages;
+            logger.debug(`PDF has ${numPages} pages`);
+            let markdown = `# ${path.basename(filePath, '.pdf')}\n\n`;
+            // Extract text from each page
+            for (let pageNum = 1; pageNum <= numPages; pageNum++) {
+                const page = await pdfDocument.getPage(pageNum);
+                const textContent = await page.getTextContent();
+                // Combine text items into a readable format
+                let pageText = '';
+                let currentY = -1;
+                for (const item of textContent.items) {
+                    if ('str' in item) {
+                        // If this is a new line (different Y position), add a line break
+                        if (currentY !== -1 && Math.abs(item.transform[5] - currentY) > 5) {
+                            pageText += '\n';
+                        }
+                        pageText += item.str;
+                        // Add space if the next item doesn't start immediately after this one
+                        if ('width' in item && item.width > 0) {
+                            pageText += ' ';
+                        }
+                        currentY = item.transform[5];
+                    }
+                }
+                // Clean up the text
+                pageText = pageText
+                    .replace(/\s+/g, ' ') // Replace multiple whitespace with single space
+                    .replace(/\n\s+/g, '\n') // Clean up line starts
+                    .trim();
+                if (pageText.length > 0) {
+                    if (numPages > 1) {
+                        markdown += `## Page ${pageNum}\n\n`;
+                    }
+                    markdown += pageText + '\n\n';
+                }
+            }
+            // Clean up the final markdown
+            markdown = markdown.replace(/\n{3,}/g, '\n\n').trim();
+            logger.debug(`Converted PDF to ${markdown.length} characters of markdown`);
+            return markdown;
+        }
+        catch (error) {
+            logger.error(`Failed to convert PDF ${filePath}:`, error);
+            throw error;
+        }
+    }
+    async downloadAndConvertPdfFromUrl(url, logger) {
+        logger.debug(`Downloading and converting PDF from URL: ${url}`);
+        try {
+            // Download the PDF file
+            const response = await axios_1.default.get(url, {
+                responseType: 'arraybuffer',
+                timeout: 60000, // 60 second timeout
+                headers: {
+                    'User-Agent': 'Mozilla/5.0 (compatible; doc2vec PDF processor)'
+                }
+            });
+            if (response.status !== 200) {
+                throw new Error(`Failed to download PDF: HTTP ${response.status}`);
+            }
+            logger.debug(`Downloaded PDF (${response.data.byteLength} bytes)`);
+            // Dynamic import for PDF.js to handle ES module compatibility
+            const pdfjsLib = await Promise.resolve().then(() => __importStar(require('pdfjs-dist/legacy/build/pdf.mjs')));
+            // Convert ArrayBuffer to Uint8Array
+            const pdfData = new Uint8Array(response.data);
+            // Load the PDF document
+            const loadingTask = pdfjsLib.getDocument({
+                data: pdfData,
+                // Disable worker to avoid issues in Node.js environment
+                useWorkerFetch: false,
+                isEvalSupported: false,
+                useSystemFonts: true
+            });
+            const pdfDocument = await loadingTask.promise;
+            const numPages = pdfDocument.numPages;
+            logger.debug(`PDF has ${numPages} pages`);
+            // Get the filename from URL for the title
+            const urlPath = new URL(url).pathname;
+            const filename = path.basename(urlPath, '.pdf') || 'document';
+            let markdown = `# ${filename}\n\n`;
+            // Extract text from each page
+            for (let pageNum = 1; pageNum <= numPages; pageNum++) {
+                const page = await pdfDocument.getPage(pageNum);
+                const textContent = await page.getTextContent();
+                // Combine text items into a readable format
+                let pageText = '';
+                let currentY = -1;
+                for (const item of textContent.items) {
+                    if ('str' in item) {
+                        // If this is a new line (different Y position), add a line break
+                        if (currentY !== -1 && Math.abs(item.transform[5] - currentY) > 5) {
+                            pageText += '\n';
+                        }
+                        pageText += item.str;
+                        // Add space if the next item doesn't start immediately after this one
+                        if ('width' in item && item.width > 0) {
+                            pageText += ' ';
+                        }
+                        currentY = item.transform[5];
+                    }
+                }
+                // Clean up the text
+                pageText = pageText
+                    .replace(/\s+/g, ' ') // Replace multiple whitespace with single space
+                    .replace(/\n\s+/g, '\n') // Clean up line starts
+                    .trim();
+                if (pageText.length > 0) {
+                    if (numPages > 1) {
+                        markdown += `## Page ${pageNum}\n\n`;
+                    }
+                    markdown += pageText + '\n\n';
+                }
+            }
+            // Clean up the final markdown
+            markdown = markdown.replace(/\n{3,}/g, '\n\n').trim();
+            logger.debug(`Converted PDF to ${markdown.length} characters of markdown`);
+            return markdown;
+        }
+        catch (error) {
+            logger.error(`Failed to download and convert PDF from ${url}:`, error);
+            throw error;
+        }
+    }
+    async processDirectory(dirPath, config, processFileContent, parentLogger, visitedPaths = new Set()) {
+        const logger = parentLogger.child('directory-processor');
+        logger.info(`Processing directory: ${dirPath}`);
+        const recursive = config.recursive !== undefined ? config.recursive : true;
+        const includeExtensions = config.include_extensions || ['.md', '.txt', '.html', '.htm', '.pdf'];
+        const excludeExtensions = config.exclude_extensions || [];
+        const encoding = config.encoding || 'utf8';
+        try {
+            const files = fs.readdirSync(dirPath);
+            let processedFiles = 0;
+            let skippedFiles = 0;
+            for (const file of files) {
+                const filePath = path.join(dirPath, file);
+                const stat = fs.statSync(filePath);
+                // Skip already visited paths
+                if (visitedPaths.has(filePath)) {
+                    logger.debug(`Skipping already visited path: ${filePath}`);
+                    continue;
+                }
+                visitedPaths.add(filePath);
+                if (stat.isDirectory()) {
+                    if (recursive) {
+                        await this.processDirectory(filePath, config, processFileContent, logger, visitedPaths);
+                    }
+                    else {
+                        logger.debug(`Skipping directory ${filePath} (recursive=false)`);
+                    }
+                }
+                else if (stat.isFile()) {
+                    const extension = path.extname(file).toLowerCase();
+                    // Apply extension filters
+                    if (excludeExtensions.includes(extension)) {
+                        logger.debug(`Skipping file with excluded extension: ${filePath}`);
+                        skippedFiles++;
+                        continue;
+                    }
+                    if (includeExtensions.length > 0 && !includeExtensions.includes(extension)) {
+                        logger.debug(`Skipping file with non-included extension: ${filePath}`);
+                        skippedFiles++;
+                        continue;
+                    }
+                    try {
+                        logger.info(`Reading file: ${filePath}`);
+                        let content;
+                        let processedContent;
+                        if (extension === '.pdf') {
+                            // Handle PDF files
+                            logger.debug(`Processing PDF file: ${filePath}`);
+                            processedContent = await this.convertPdfToMarkdown(filePath, logger);
+                        }
+                        else {
+                            // Handle text-based files
+                            content = fs.readFileSync(filePath, { encoding: encoding });
+                            if (content.length > config.max_size) {
+                                logger.warn(`File content (${content.length} chars) exceeds max size (${config.max_size}). Skipping ${filePath}.`);
+                                skippedFiles++;
+                                continue;
+                            }
+                            // Convert HTML to Markdown if needed
+                            if (extension === '.html' || extension === '.htm') {
+                                logger.debug(`Converting HTML to Markdown for ${filePath}`);
+                                const cleanHtml = (0, sanitize_html_1.default)(content, {
+                                    allowedTags: [
+                                        'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
+                                        'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
+                                        'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td'
+                                    ],
+                                    allowedAttributes: {
+                                        'a': ['href'],
+                                        'pre': ['class', 'data-language'],
+                                        'code': ['class', 'data-language'],
+                                        'div': ['class'],
+                                        'span': ['class']
+                                    }
+                                });
+                                processedContent = this.turndownService.turndown(cleanHtml);
+                            }
+                            else {
+                                processedContent = content;
+                            }
+                        }
+                        // Check size limit for processed content
+                        if (processedContent.length > config.max_size) {
+                            logger.warn(`Processed content (${processedContent.length} chars) exceeds max size (${config.max_size}). Skipping ${filePath}.`);
+                            skippedFiles++;
+                            continue;
+                        }
+                        await processFileContent(filePath, processedContent);
+                        processedFiles++;
+                    }
+                    catch (error) {
+                        logger.error(`Error processing file ${filePath}:`, error);
+                    }
+                }
+            }
+            logger.info(`Directory processed. Processed: ${processedFiles}, Skipped: ${skippedFiles}`);
+        }
+        catch (error) {
+            logger.error(`Error reading directory ${dirPath}:`, error);
+        }
+    }
+    async chunkMarkdown(markdown, sourceConfig, url) {
+        const logger = this.logger.child('chunker');
+        logger.debug(`Chunking markdown from ${url} (${markdown.length} chars)`);
+        const MAX_TOKENS = 1000;
+        const chunks = [];
+        const lines = markdown.split("\n");
+        let currentChunk = "";
+        let headingHierarchy = [];
+        const processChunk = () => {
+            if (currentChunk.trim()) {
+                const tokens = utils_1.Utils.tokenize(currentChunk);
+                if (tokens.length > MAX_TOKENS) {
+                    logger.debug(`Chunk exceeds max token count (${tokens.length}), splitting into smaller chunks`);
+                    let subChunk = "";
+                    let tokenCount = 0;
+                    const overlapSize = Math.floor(MAX_TOKENS * 0.05);
+                    let lastTokens = [];
+                    for (const token of tokens) {
+                        if (tokenCount + 1 > MAX_TOKENS) {
+                            chunks.push(createDocumentChunk(subChunk, headingHierarchy));
+                            subChunk = lastTokens.join("") + token;
+                            tokenCount = lastTokens.length + 1;
+                            lastTokens = [];
+                        }
+                        else {
+                            subChunk += token;
+                            tokenCount++;
+                            lastTokens.push(token);
+                            if (lastTokens.length > overlapSize) {
+                                lastTokens.shift();
+                            }
+                        }
+                    }
+                    if (subChunk) {
+                        chunks.push(createDocumentChunk(subChunk, headingHierarchy));
+                    }
+                }
+                else {
+                    chunks.push(createDocumentChunk(currentChunk, headingHierarchy));
+                }
+            }
+            currentChunk = "";
+        };
+        const createDocumentChunk = (content, hierarchy) => {
+            const chunkId = utils_1.Utils.generateHash(content);
+            logger.debug(`Created chunk ${chunkId.substring(0, 8)}... with ${content.length} chars`);
+            return {
+                content,
+                metadata: {
+                    product_name: sourceConfig.product_name,
+                    version: sourceConfig.version,
+                    heading_hierarchy: [...hierarchy],
+                    section: hierarchy[hierarchy.length - 1] || "Introduction",
+                    chunk_id: chunkId,
+                    url: url,
+                    hash: utils_1.Utils.generateHash(content)
+                }
+            };
+        };
+        for (const line of lines) {
+            if (line.startsWith("#")) {
+                processChunk();
+                const levelMatch = line.match(/^(#+)/);
+                let level = levelMatch ? levelMatch[1].length : 1;
+                const heading = line.replace(/^#+\s*/, "").trim();
+                logger.debug(`Found heading (level ${level}): ${heading}`);
+                while (headingHierarchy.length < level - 1) {
+                    headingHierarchy.push("");
+                }
+                if (level <= headingHierarchy.length) {
+                    headingHierarchy = headingHierarchy.slice(0, level - 1);
+                }
+                headingHierarchy[level - 1] = heading;
+            }
+            else {
+                currentChunk += `${line}\n`;
+            }
+        }
+        processChunk();
+        logger.debug(`Chunking complete, created ${chunks.length} chunks`);
+        return chunks;
+    }
+}
+exports.ContentProcessor = ContentProcessor;