doc2vec 2.0.0 → 2.2.0

package/dist/content-processor.js

@@ -47,6 +47,7 @@ const turndown_1 = __importDefault(require("turndown"));
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const utils_1 = require("./utils");
+const code_chunker_1 = require("./code-chunker");
 class ContentProcessor {
     constructor(logger) {
         this.logger = logger;
@@ -54,6 +55,9 @@ class ContentProcessor {
             codeBlockStyle: 'fenced',
             headingStyle: 'atx'
         });
+        this.tokenChunkerCache = new Map();
+        this.codeChunkerCache = new Map();
+        this.tokenizerCache = null;
         this.setupTurndownRules();
     }
     setupTurndownRules() {
@@ -200,15 +204,38 @@ class ContentProcessor {
     async crawlWebsite(baseUrl, sourceConfig, processPageContent, parentLogger, visitedUrls) {
         const logger = parentLogger.child('crawler');
         const queue = [baseUrl];
+        const brokenLinks = [];
+        const brokenLinkKeys = new Set();
+        const referrers = new Map();
+        const addReferrer = (targetUrl, sourceUrl) => {
+            const existing = referrers.get(targetUrl);
+            if (existing) {
+                existing.add(sourceUrl);
+            }
+            else {
+                referrers.set(targetUrl, new Set([sourceUrl]));
+            }
+        };
+        const addBrokenLink = (sourceUrl, targetUrl) => {
+            const key = `${sourceUrl} -> ${targetUrl}`;
+            if (brokenLinkKeys.has(key))
+                return;
+            brokenLinkKeys.add(key);
+            brokenLinks.push({ source: sourceUrl, target: targetUrl });
+        };
+        addReferrer(baseUrl, baseUrl);
         // Process sitemap if provided
         if (sourceConfig.sitemap_url) {
             logger.section('SITEMAP PROCESSING');
             const sitemapUrls = await this.parseSitemap(sourceConfig.sitemap_url, logger);
             // Add sitemap URLs to the queue if they're within the website scope
             for (const url of sitemapUrls) {
-                if (url.startsWith(sourceConfig.url) && !queue.includes(url)) {
-                    logger.debug(`Adding URL from sitemap to queue: ${url}`);
-                    queue.push(url);
+                if (url.startsWith(sourceConfig.url)) {
+                    addReferrer(url, sourceConfig.sitemap_url);
+                    if (!queue.includes(url)) {
+                        logger.debug(`Adding URL from sitemap to queue: ${url}`);
+                        queue.push(url);
+                    }
                 }
             }
             logger.info(`Added ${queue.length - 1} URLs from sitemap to the crawl queue`);
@@ -235,7 +262,14 @@ class ContentProcessor {
             }
             try {
                 logger.info(`Crawling: ${url}`);
-                const content = await this.processPage(url, sourceConfig);
+                const sources = referrers.get(url) ?? new Set([baseUrl]);
+                const content = await this.processPage(url, sourceConfig, (reportedUrl, status) => {
+                    if (status === 404) {
+                        for (const source of sources) {
+                            addBrokenLink(source, reportedUrl);
+                        }
+                    }
+                });
                 if (content !== null) {
                     await processPageContent(url, content);
                     if (utils_1.Utils.isPdfUrl(url)) {
@@ -252,17 +286,21 @@ class ContentProcessor {
                 if (!utils_1.Utils.isPdfUrl(url)) {
                     const response = await axios_1.default.get(url);
                     const $ = (0, cheerio_1.load)(response.data);
+                    const pageUrlForLinks = response?.request?.res?.responseUrl || url;
                     logger.debug(`Finding links on page ${url}`);
                     let newLinksFound = 0;
                     $('a[href]').each((_, element) => {
                         const href = $(element).attr('href');
                         if (!href || href.startsWith('#') || href.startsWith('mailto:'))
                             return;
-                        const fullUrl = utils_1.Utils.buildUrl(href, url);
-                        if (fullUrl.startsWith(sourceConfig.url) && !visitedUrls.has(utils_1.Utils.normalizeUrl(fullUrl))) {
-                            if (!queue.includes(fullUrl)) {
-                                queue.push(fullUrl);
-                                newLinksFound++;
+                        const fullUrl = utils_1.Utils.buildUrl(href, pageUrlForLinks);
+                        if (fullUrl.startsWith(sourceConfig.url)) {
+                            addReferrer(fullUrl, pageUrlForLinks);
+                            if (!visitedUrls.has(utils_1.Utils.normalizeUrl(fullUrl))) {
+                                if (!queue.includes(fullUrl)) {
+                                    queue.push(fullUrl);
+                                    newLinksFound++;
+                                }
                             }
                         }
                     });
@@ -272,6 +310,13 @@ class ContentProcessor {
             catch (error) {
                 logger.error(`Failed during processing or link discovery for ${url}:`, error);
                 errorCount++;
+                const status = this.getHttpStatus(error);
+                if (status === 404) {
+                    const sources = referrers.get(url) ?? new Set([baseUrl]);
+                    for (const source of sources) {
+                        addBrokenLink(source, url);
+                    }
+                }
                 // Check if this is a network error (DNS resolution, connection issues, etc.)
                 if (this.isNetworkError(error)) {
                     hasNetworkErrors = true;
@@ -283,7 +328,7 @@ class ContentProcessor {
         if (hasNetworkErrors) {
             logger.warn('Network errors were encountered during crawling. Cleanup may be skipped to avoid removing valid chunks.');
         }
-        return { hasNetworkErrors };
+        return { hasNetworkErrors, brokenLinks };
     }
     isNetworkError(error) {
         // Check for common network error patterns
@@ -324,7 +369,7 @@ class ContentProcessor {
         }
         return false;
     }
-    async processPage(url, sourceConfig) {
+    async processPage(url, sourceConfig, onHttpStatus) {
         const logger = this.logger.child('page-processor');
         logger.debug(`Processing content from ${url}`);
         // Check if this is a PDF URL
@@ -340,6 +385,13 @@ class ContentProcessor {
                 return markdown;
             }
             catch (error) {
+                const status = this.getHttpStatus(error);
+                if (status !== undefined && status >= 400) {
+                    if (onHttpStatus) {
+                        onHttpStatus(url, status);
+                    }
+                    throw error;
+                }
                 logger.error(`Failed to process PDF ${url}:`, error);
                 return null;
             }
@@ -363,7 +415,13 @@ class ContentProcessor {
             });
             const page = await browser.newPage();
             logger.debug(`Navigating to ${url}`);
-            await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
+            const response = await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
+            const status = response?.status();
+            if (status !== undefined && status >= 400) {
+                const error = new Error(`Failed to load page: HTTP ${status}`);
+                error.status = status;
+                throw error;
+            }
             const htmlContent = await page.evaluate(() => {
                 // 💡 Try specific content selectors first, then fall back to broader ones
                 const mainContentElement = document.querySelector('.docs-content') || // Common docs pattern
@@ -480,6 +538,13 @@ class ContentProcessor {
             return markdown;
         }
         catch (error) {
+            const status = this.getHttpStatus(error);
+            if (status !== undefined && status >= 400) {
+                if (onHttpStatus) {
+                    onHttpStatus(url, status);
+                }
+                throw error;
+            }
             logger.error(`Error processing page ${url}:`, error);
             return null;
         }
@@ -490,6 +555,15 @@ class ContentProcessor {
             }
         }
     }
+    getHttpStatus(error) {
+        if (typeof error?.status === 'number') {
+            return error.status;
+        }
+        if (typeof error?.response?.status === 'number') {
+            return error.response.status;
+        }
+        return undefined;
+    }
     markCodeParents(node) {
         if (!node)
             return;
@@ -812,6 +886,257 @@ class ContentProcessor {
             logger.error(`Error reading directory ${dirPath}:`, error);
         }
     }
+    async processCodeDirectory(dirPath, config, processFileContent, parentLogger, visitedPaths = new Set(), options) {
+        const logger = parentLogger.child('code-directory-processor');
+        logger.info(`Processing code directory: ${dirPath}`);
+        const recursive = config.recursive !== undefined ? config.recursive : true;
+        const includeExtensions = config.include_extensions || [
+            '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
+            '.py', '.go', '.rs', '.java', '.kt', '.kts', '.swift',
+            '.c', '.cc', '.cpp', '.h', '.hpp', '.cs',
+            '.rb', '.php', '.scala', '.sql', '.sh', '.bash', '.zsh',
+            '.html', '.css', '.scss', '.sass', '.less',
+            '.json', '.yaml', '.yml', '.md'
+        ];
+        const excludeExtensions = config.exclude_extensions || [];
+        const encoding = config.encoding || 'utf8';
+        let maxMtime = 0;
+        try {
+            const files = fs.readdirSync(dirPath);
+            let processedFiles = 0;
+            let skippedFiles = 0;
+            const allowedFiles = options?.allowedFiles;
+            const mtimeCutoff = options?.mtimeCutoff;
+            const trackFiles = options?.trackFiles;
+            for (const file of files) {
+                const filePath = path.join(dirPath, file);
+                const stat = fs.statSync(filePath);
+                if (visitedPaths.has(filePath)) {
+                    logger.debug(`Skipping already visited path: ${filePath}`);
+                    continue;
+                }
+                visitedPaths.add(filePath);
+                if (stat.isDirectory()) {
+                    if (recursive) {
+                        const childResult = await this.processCodeDirectory(filePath, config, processFileContent, logger, visitedPaths, options);
+                        processedFiles += childResult.processedFiles;
+                        skippedFiles += childResult.skippedFiles;
+                        maxMtime = Math.max(maxMtime, childResult.maxMtime);
+                    }
+                    else {
+                        logger.debug(`Skipping directory ${filePath} (recursive=false)`);
+                    }
+                }
+                else if (stat.isFile()) {
+                    const extension = path.extname(file).toLowerCase();
+                    if (excludeExtensions.includes(extension)) {
+                        logger.debug(`Skipping file with excluded extension: ${filePath}`);
+                        skippedFiles++;
+                        continue;
+                    }
+                    if (includeExtensions.length > 0 && !includeExtensions.includes(extension)) {
+                        logger.debug(`Skipping file with non-included extension: ${filePath}`);
+                        skippedFiles++;
+                        continue;
+                    }
+                    trackFiles?.add(filePath);
+                    maxMtime = Math.max(maxMtime, stat.mtimeMs);
+                    if (allowedFiles && !allowedFiles.has(filePath)) {
+                        skippedFiles++;
+                        continue;
+                    }
+                    if (mtimeCutoff !== undefined && stat.mtimeMs <= mtimeCutoff) {
+                        skippedFiles++;
+                        continue;
+                    }
+                    try {
+                        logger.info(`Reading file: ${filePath}`);
+                        const content = fs.readFileSync(filePath, { encoding: encoding });
+                        if (content.length > config.max_size) {
+                            logger.warn(`File content (${content.length} chars) exceeds max size (${config.max_size}). Skipping ${filePath}.`);
+                            skippedFiles++;
+                            continue;
+                        }
+                        await processFileContent(filePath, content);
+                        processedFiles++;
+                    }
+                    catch (error) {
+                        logger.error(`Error processing file ${filePath}:`, error);
+                    }
+                }
+            }
+            logger.info(`Code directory processed. Processed: ${processedFiles}, Skipped: ${skippedFiles}`);
+            return { processedFiles, skippedFiles, maxMtime };
+        }
+        catch (error) {
+            logger.error(`Error reading code directory ${dirPath}:`, error);
+            return { processedFiles: 0, skippedFiles: 0, maxMtime };
+        }
+    }
+    async getTokenChunker(chunkSize) {
+        const cacheKey = `${chunkSize || 'default'}`;
+        const cached = this.tokenChunkerCache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const chunkerPromise = (async () => {
+            const { TokenChunker } = await this.importChonkieModule('@chonkiejs/core');
+            return await TokenChunker.create({ chunkSize, tokenizer: 'character' });
+        })();
+        this.tokenChunkerCache.set(cacheKey, chunkerPromise);
+        return chunkerPromise;
+    }
+    async getCodeChunker(lang, chunkSize) {
+        const cacheKey = `${lang}:${chunkSize || 'default'}`;
+        const cached = this.codeChunkerCache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const chunkerPromise = (async () => {
+            const tokenizer = await this.getTokenizer();
+            return await code_chunker_1.CodeChunker.create({
+                lang,
+                chunkSize,
+                tokenCounter: async (text) => tokenizer.countTokens(text)
+            });
+        })();
+        this.codeChunkerCache.set(cacheKey, chunkerPromise);
+        return chunkerPromise;
+    }
+    async getTokenizer() {
+        if (!this.tokenizerCache) {
+            this.tokenizerCache = (async () => {
+                const { Tokenizer } = await this.importChonkieModule('@chonkiejs/core');
+                return await Tokenizer.create('character');
+            })();
+        }
+        return this.tokenizerCache;
+    }
+    detectCodeLanguage(filePath) {
+        const extension = path.extname(filePath).toLowerCase();
+        const languageMap = {
+            '.ts': 'typescript',
+            '.tsx': 'typescript',
+            '.js': 'javascript',
+            '.jsx': 'javascript',
+            '.mjs': 'javascript',
+            '.cjs': 'javascript',
+            '.py': 'python',
+            '.go': 'go',
+            '.rs': 'rust',
+            '.java': 'java',
+            '.kt': 'kotlin',
+            '.kts': 'kotlin',
+            '.swift': 'swift',
+            '.c': 'c',
+            '.cc': 'cpp',
+            '.cpp': 'cpp',
+            '.h': 'cpp',
+            '.hpp': 'cpp',
+            '.cs': 'csharp',
+            '.rb': 'ruby',
+            '.php': 'php',
+            '.scala': 'scala',
+            '.sql': 'sql',
+            '.sh': 'bash',
+            '.bash': 'bash',
+            '.zsh': 'bash',
+            '.html': 'html',
+            '.css': 'css',
+            '.scss': 'scss',
+            '.sass': 'scss',
+            '.less': 'css',
+            '.json': 'json',
+            '.yaml': 'yaml',
+            '.yml': 'yaml',
+            '.md': 'markdown'
+        };
+        return languageMap[extension];
+    }
+    async importChonkieModule(specifier) {
+        // Use dynamic import() directly - preserved by TypeScript with target ES2020+
+        // even in CommonJS mode, as Node.js supports import() in CJS contexts
+        return Promise.resolve(`${specifier}`).then(s => __importStar(require(s)));
+    }
+    async chunkCode(code, sourceConfig, url, filePath, branch, repo) {
+        const logger = this.logger.child('code-chunker');
+        const normalizedPath = filePath.replace(/\\/g, '/');
+        const lang = this.detectCodeLanguage(filePath);
+        let chunks;
+        if (lang === 'markdown') {
+            const markdownChunks = await this.chunkMarkdown(code, sourceConfig, url);
+            for (const chunk of markdownChunks) {
+                if (normalizedPath) {
+                    const filePrefix = `[File: ${normalizedPath}]\n`;
+                    const searchableText = filePrefix + chunk.content;
+                    const chunkId = utils_1.Utils.generateHash(`${url}::${searchableText}`);
+                    chunk.content = searchableText;
+                    chunk.metadata.heading_hierarchy = [normalizedPath, ...chunk.metadata.heading_hierarchy.filter(Boolean)];
+                    chunk.metadata.section = normalizedPath;
+                    chunk.metadata.chunk_id = chunkId;
+                    chunk.metadata.hash = chunkId;
+                }
+                if (branch) {
+                    chunk.metadata.branch = branch;
+                }
+                if (repo) {
+                    chunk.metadata.repo = repo;
+                }
+            }
+            logger.debug(`Chunked ${normalizedPath || url}: ${markdownChunks.length} chunks created.`);
+            return markdownChunks;
+        }
+        if (lang) {
+            try {
+                const codeChunker = await this.getCodeChunker(lang, sourceConfig.chunk_size);
+                chunks = await codeChunker.chunk(code);
+            }
+            catch (error) {
+                logger.warn(`CodeChunker failed for ${normalizedPath || url}, falling back to token chunking:`, error);
+                const chunker = await this.getTokenChunker(sourceConfig.chunk_size);
+                chunks = await chunker.chunk(code);
+            }
+        }
+        else {
+            const chunker = await this.getTokenChunker(sourceConfig.chunk_size);
+            chunks = await chunker.chunk(code);
+        }
+        const documentChunks = [];
+        let chunkCounter = 0;
+        const headingHierarchy = normalizedPath ? [normalizedPath] : [];
+        const contextPrefix = normalizedPath ? `[File: ${normalizedPath}]\n` : '';
+        for (const chunk of chunks) {
+            const content = chunk.text?.trim();
+            if (!content) {
+                continue;
+            }
+            const searchableText = contextPrefix + content;
+            const chunkId = utils_1.Utils.generateHash(`${url}::${searchableText}`);
+            documentChunks.push({
+                content: searchableText,
+                metadata: {
+                    product_name: sourceConfig.product_name,
+                    version: sourceConfig.version,
+                    ...(branch ? { branch } : {}),
+                    ...(repo ? { repo } : {}),
+                    heading_hierarchy: headingHierarchy,
+                    section: normalizedPath || 'Code',
+                    chunk_id: chunkId,
+                    url: url,
+                    hash: chunkId,
+                    chunk_index: chunkCounter,
+                    total_chunks: 0
+                }
+            });
+            chunkCounter++;
+        }
+        const totalChunks = documentChunks.length;
+        for (const chunk of documentChunks) {
+            chunk.metadata.total_chunks = totalChunks;
+        }
+        logger.debug(`Chunked ${normalizedPath || url}: ${documentChunks.length} chunks created.`);
+        return documentChunks;
+    }
     async chunkMarkdown(markdown, sourceConfig, url) {
         const logger = this.logger.child('chunker');
         // --- Configuration ---