doc2vec 1.3.0 → 2.0.0

package/dist/content-processor.js

@@ -347,14 +347,32 @@ class ContentProcessor {
         // Original HTML page processing logic
         let browser = null;
         try {
+            // Use system Chromium if available (for Docker environments)
+            let executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
+            if (!executablePath) {
+                if (fs.existsSync('/usr/bin/chromium')) {
+                    executablePath = '/usr/bin/chromium';
+                }
+                else if (fs.existsSync('/usr/bin/chromium-browser')) {
+                    executablePath = '/usr/bin/chromium-browser';
+                }
+            }
             browser = await puppeteer_1.default.launch({
+                executablePath,
                 args: ['--no-sandbox', '--disable-setuid-sandbox'],
             });
             const page = await browser.newPage();
             logger.debug(`Navigating to ${url}`);
             await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
             const htmlContent = await page.evaluate(() => {
-                const mainContentElement = document.querySelector('div[role="main"].document') || document.querySelector('main') || document.body;
+                // 💡 Try specific content selectors first, then fall back to broader ones
+                const mainContentElement = document.querySelector('.docs-content') || // Common docs pattern
+                    document.querySelector('.doc-content') || // Alternative docs pattern
+                    document.querySelector('.markdown-body') || // GitHub-style
+                    document.querySelector('article') || // Semantic article
+                    document.querySelector('div[role="main"].document') ||
+                    document.querySelector('main') ||
+                    document.body;
                 return mainContentElement.innerHTML;
             });
             if (htmlContent.length > sourceConfig.max_size) {
@@ -370,10 +388,24 @@ class ContentProcessor {
                 pre.setAttribute('data-readable-content-score', '100');
                 this.markCodeParents(pre.parentElement);
             });
+            // 💡 Extract H1s BEFORE Readability - it often strips them as "chrome"
+            // We'll inject them back after Readability processing
+            const h1Elements = document.querySelectorAll('h1');
+            const extractedH1s = [];
+            logger.debug(`[Readability Debug] Found ${h1Elements.length} H1 elements before Readability`);
+            h1Elements.forEach((h1, index) => {
+                const h1Text = h1.textContent?.trim() || '';
+                // Skip empty H1s or icon-only H1s (like "link" anchors)
+                if (h1Text && h1Text.length > 3 && !h1Text.match(/^(link|#|menu|close)$/i)) {
+                    extractedH1s.push(h1Text);
+                    logger.debug(`[Readability Debug] Extracted H1[${index}]: "${h1Text.substring(0, 50)}..."`);
+                }
+                h1.classList.add('original-h1');
+            });
             logger.debug(`Applying Readability to extract main content`);
             const reader = new readability_1.Readability(document, {
                 charThreshold: 20,
-                classesToPreserve: ['article-content'],
+                classesToPreserve: ['article-content', 'original-h1'],
             });
             const article = reader.parse();
             if (!article) {
@@ -381,8 +413,35 @@ class ContentProcessor {
                 await browser.close();
                 return null;
             }
-            logger.debug(`Sanitizing HTML (${article.content.length} chars)`);
-            const cleanHtml = (0, sanitize_html_1.default)(article.content, {
+            // Debug: Log what Readability extracted
+            logger.debug(`[Readability Debug] article.title: "${article.title}"`);
+            logger.debug(`[Readability Debug] article.content length: ${article.content?.length}`);
+            logger.debug(`[Readability Debug] article.content starts with: "${article.content?.substring(0, 200)}..."`);
+            logger.debug(`[Readability Debug] Contains H1 tag: ${article.content?.includes('<h1')}`);
+            logger.debug(`[Readability Debug] Contains H2 tag: ${article.content?.includes('<h2')}`);
+            logger.debug(`[Readability Debug] Contains original-h1 class: ${article.content?.includes('original-h1')}`);
+            // 💡 Restore H1s: find elements with our marker class and convert back from H2
+            const articleDom = new jsdom_1.JSDOM(article.content);
+            const articleDoc = articleDom.window.document;
+            const originalH1Elements = articleDoc.querySelectorAll('.original-h1');
+            logger.debug(`[Readability Debug] Found ${originalH1Elements.length} elements with .original-h1 class to restore`);
+            originalH1Elements.forEach((heading, index) => {
+                logger.debug(`[Readability Debug] Restoring[${index}]: tagName=${heading.tagName}, text="${heading.textContent?.trim().substring(0, 50)}..."`);
+                // Create a new H1 element with the same content
+                const h1 = articleDoc.createElement('h1');
+                h1.innerHTML = heading.innerHTML;
+                // Copy other attributes except class
+                Array.from(heading.attributes).forEach(attr => {
+                    if (attr.name !== 'class') {
+                        h1.setAttribute(attr.name, attr.value);
+                    }
+                });
+                heading.replaceWith(h1);
+            });
+            const restoredContent = articleDoc.body.innerHTML;
+            logger.debug(`[Readability Debug] Restored content contains H1: ${restoredContent.includes('<h1')}`);
+            logger.debug(`Sanitizing HTML (${restoredContent.length} chars)`);
+            const cleanHtml = (0, sanitize_html_1.default)(restoredContent, {
                 allowedTags: [
                     'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
                     'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
@@ -397,7 +456,26 @@ class ContentProcessor {
                 }
             });
             logger.debug(`Converting HTML to Markdown`);
-            const markdown = this.turndownService.turndown(cleanHtml);
+            let markdown = this.turndownService.turndown(cleanHtml);
+            // 💡 Inject extracted H1s back if they're not in the markdown
+            // Readability often strips them as "page chrome"
+            // Use article.title as fallback if no H1 was extracted
+            const pageTitle = extractedH1s.length > 0 ? extractedH1s[0] : (article.title?.trim() || '');
+            if (pageTitle) {
+                // Check if markdown already starts with this exact H1 (allowing for leading whitespace)
+                const normalizedTitle = pageTitle.replace(/\s+/g, ' ');
+                const markdownFirstLine = markdown.trimStart().split('\n')[0] || '';
+                const existingH1Match = markdownFirstLine.match(/^#\s+(.+)$/);
+                const existingH1Text = existingH1Match ? existingH1Match[1].replace(/\s+/g, ' ').trim() : '';
+                // Only inject if markdown doesn't already start with this H1
+                if (!existingH1Match || existingH1Text !== normalizedTitle) {
+                    markdown = `# ${pageTitle}\n\n${markdown}`;
+                    logger.debug(`[Readability Debug] Injected page title as H1: "${pageTitle}"`);
+                }
+                else {
+                    logger.debug(`[Readability Debug] H1 "${pageTitle}" already present in markdown`);
+                }
+            }
             logger.debug(`Markdown conversion complete (${markdown.length} chars)`);
             return markdown;
         }
@@ -421,6 +499,68 @@ class ContentProcessor {
         }
         this.markCodeParents(node.parentElement);
     }
+    async convertDocToMarkdown(filePath, logger) {
+        logger.debug(`Converting DOC to markdown: ${filePath}`);
+        try {
+            // Dynamic import for word-extractor
+            const WordExtractor = (await Promise.resolve().then(() => __importStar(require('word-extractor')))).default;
+            const extractor = new WordExtractor();
+            const extracted = await extractor.extract(filePath);
+            const text = extracted.getBody();
+            // Create markdown with filename as title
+            let markdown = `# ${path.basename(filePath, '.doc')}\n\n`;
+            // Clean up the text and add to markdown
+            const cleanedText = text
+                .replace(/\r\n/g, '\n') // Normalize line endings
+                .replace(/\n{3,}/g, '\n\n') // Remove excessive line breaks
+                .trim();
+            markdown += cleanedText;
+            logger.debug(`Converted DOC to ${markdown.length} characters of markdown`);
+            return markdown;
+        }
+        catch (error) {
+            logger.error(`Failed to convert DOC ${filePath}:`, error);
+            throw error;
+        }
+    }
+    async convertDocxToMarkdown(filePath, logger) {
+        logger.debug(`Converting DOCX to markdown: ${filePath}`);
+        try {
+            // Dynamic import for mammoth
+            const mammoth = await Promise.resolve().then(() => __importStar(require('mammoth')));
+            const result = await mammoth.convertToHtml({ path: filePath });
+            const html = result.value;
+            // Log any warnings from mammoth
+            if (result.messages.length > 0) {
+                logger.debug(`Mammoth warnings: ${result.messages.map(m => m.message).join(', ')}`);
+            }
+            // Create markdown with filename as title
+            let markdown = `# ${path.basename(filePath, '.docx')}\n\n`;
+            // Convert HTML to Markdown using turndown
+            const cleanHtml = (0, sanitize_html_1.default)(html, {
+                allowedTags: [
+                    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
+                    'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
+                    'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td', 'br'
+                ],
+                allowedAttributes: {
+                    'a': ['href'],
+                    'pre': ['class'],
+                    'code': ['class']
+                }
+            });
+            const convertedContent = this.turndownService.turndown(cleanHtml);
+            markdown += convertedContent;
+            // Clean up excessive line breaks
+            markdown = markdown.replace(/\n{3,}/g, '\n\n').trim();
+            logger.debug(`Converted DOCX to ${markdown.length} characters of markdown`);
+            return markdown;
+        }
+        catch (error) {
+            logger.error(`Failed to convert DOCX ${filePath}:`, error);
+            throw error;
+        }
+    }
     async convertPdfToMarkdown(filePath, logger) {
         logger.debug(`Converting PDF to markdown: ${filePath}`);
         try {
@@ -611,6 +751,16 @@ class ContentProcessor {
                             logger.debug(`Processing PDF file: ${filePath}`);
                             processedContent = await this.convertPdfToMarkdown(filePath, logger);
                         }
+                        else if (extension === '.doc') {
+                            // Handle legacy Word DOC files
+                            logger.debug(`Processing DOC file: ${filePath}`);
+                            processedContent = await this.convertDocToMarkdown(filePath, logger);
+                        }
+                        else if (extension === '.docx') {
+                            // Handle modern Word DOCX files
+                            logger.debug(`Processing DOCX file: ${filePath}`);
+                            processedContent = await this.convertDocxToMarkdown(filePath, logger);
+                        }
                         else {
                             // Handle text-based files
                             content = fs.readFileSync(filePath, { encoding: encoding });
@@ -664,84 +814,157 @@ class ContentProcessor {
     }
     async chunkMarkdown(markdown, sourceConfig, url) {
         const logger = this.logger.child('chunker');
-        logger.debug(`Chunking markdown from ${url} (${markdown.length} chars)`);
+        // --- Configuration ---
         const MAX_TOKENS = 1000;
+        const MIN_TOKENS = 150; // 💡 Merges "OpenAI-compatible" sentence into the next block
+        const OVERLAP_PERCENT = 0.1; // 10% overlap for large splits
         const chunks = [];
         const lines = markdown.split("\n");
-        let currentChunk = "";
+        let buffer = "";
         let headingHierarchy = [];
-        const processChunk = () => {
-            if (currentChunk.trim()) {
-                const tokens = utils_1.Utils.tokenize(currentChunk);
-                if (tokens.length > MAX_TOKENS) {
-                    logger.debug(`Chunk exceeds max token count (${tokens.length}), splitting into smaller chunks`);
-                    let subChunk = "";
-                    let tokenCount = 0;
-                    const overlapSize = Math.floor(MAX_TOKENS * 0.05);
-                    let lastTokens = [];
-                    for (const token of tokens) {
-                        if (tokenCount + 1 > MAX_TOKENS) {
-                            chunks.push(createDocumentChunk(subChunk, headingHierarchy));
-                            subChunk = lastTokens.join("") + token;
-                            tokenCount = lastTokens.length + 1;
-                            lastTokens = [];
-                        }
-                        else {
-                            subChunk += token;
-                            tokenCount++;
-                            lastTokens.push(token);
-                            if (lastTokens.length > overlapSize) {
-                                lastTokens.shift();
-                            }
-                        }
-                    }
-                    if (subChunk) {
-                        chunks.push(createDocumentChunk(subChunk, headingHierarchy));
-                    }
-                }
-                else {
-                    chunks.push(createDocumentChunk(currentChunk, headingHierarchy));
-                }
+        let bufferHeadings = []; // Track headings in current buffer
+        let chunkCounter = 0; // Tracks chunk position within this page for ordering
+        /**
+         * Computes the topic hierarchy for merged content.
+         * When merging sibling sections (same level), uses their parent heading.
+         * Otherwise uses the current hierarchy.
+         */
+        const computeTopicHierarchy = () => {
+            if (bufferHeadings.length === 0) {
+                return headingHierarchy;
+            }
+            // Find the deepest level (most recent headings)
+            const deepestLevel = Math.max(...bufferHeadings.map(h => h.level));
+            // Get all headings at the deepest level
+            const deepestHeadings = bufferHeadings.filter(h => h.level === deepestLevel);
+            // If we have multiple sibling headings at the deepest level, use their parent
+            if (deepestHeadings.length > 1 && deepestLevel > 1) {
+                // Use parent heading (one level up from the sibling headings)
+                // headingHierarchy still contains the parent at index (deepestLevel - 2)
+                // We want everything up to (but not including) the deepest level
+                return headingHierarchy.slice(0, deepestLevel - 1);
             }
-            currentChunk = "";
+            // Single heading or different levels: use the current hierarchy
+            // This reflects the most recent heading which is appropriate
+            return headingHierarchy;
         };
+        /**
+         * Internal helper to create the final chunk object with injected context.
+         */
         const createDocumentChunk = (content, hierarchy) => {
-            const chunkId = utils_1.Utils.generateHash(content);
-            logger.debug(`Created chunk ${chunkId.substring(0, 8)}... with ${content.length} chars`);
-            return {
-                content,
+            // 💡 BREADCRUMB INJECTION
+            // We prepend the hierarchy to the text. This makes the vector highly relevant 
+            // to searches for parent topics even if the body doesn't mention them.
+            const breadcrumbs = hierarchy.filter(h => h).join(" > ");
+            const contextPrefix = breadcrumbs ? `[Topic: ${breadcrumbs}]\n` : "";
+            const searchableText = contextPrefix + content.trim();
+            const chunkId = utils_1.Utils.generateHash(searchableText);
+            const chunk = {
+                content: searchableText,
                 metadata: {
                     product_name: sourceConfig.product_name,
                     version: sourceConfig.version,
-                    heading_hierarchy: [...hierarchy],
+                    heading_hierarchy: hierarchy.filter(h => h),
                     section: hierarchy[hierarchy.length - 1] || "Introduction",
                     chunk_id: chunkId,
                     url: url,
-                    hash: utils_1.Utils.generateHash(content)
+                    hash: chunkId,
+                    chunk_index: Math.floor(chunkCounter),
+                    total_chunks: 0 // Placeholder, will be updated after all chunks are created
                 }
             };
+            chunkCounter++; // Increment for next chunk
+            return chunk;
+        };
+        /**
+         * Flushes the current buffer into the chunks array.
+         * Uses sub-splitting logic if the buffer exceeds MAX_TOKENS.
+         */
+        const flushBuffer = (force = false) => {
+            const trimmedBuffer = buffer.trim();
+            if (!trimmedBuffer)
+                return;
+            const tokenCount = utils_1.Utils.tokenize(trimmedBuffer).length;
+            // 💡 SEMANTIC MERGING
+            // If the current section is too short (like just a title or a one-liner),
+            // we don't flush yet unless it's the end of the file (force=true).
+            if (tokenCount < MIN_TOKENS && !force) {
+                return;
+            }
+            // Compute the appropriate topic hierarchy for merged content
+            const topicHierarchy = computeTopicHierarchy();
+            if (tokenCount > MAX_TOKENS) {
+                // 💡 RECURSIVE OVERLAP SPLITTING
+                // If the section is a massive guide, split it but keep headers on every sub-piece.
+                const tokens = utils_1.Utils.tokenize(trimmedBuffer);
+                const overlapSize = Math.floor(MAX_TOKENS * OVERLAP_PERCENT);
+                for (let i = 0; i < tokens.length; i += (MAX_TOKENS - overlapSize)) {
+                    const subTokens = tokens.slice(i, i + MAX_TOKENS);
+                    const subContent = subTokens.join("");
+                    chunks.push(createDocumentChunk(subContent, topicHierarchy));
+                }
+            }
+            else {
+                chunks.push(createDocumentChunk(trimmedBuffer, topicHierarchy));
+            }
+            buffer = ""; // Reset buffer after successful flush
+            bufferHeadings = []; // Reset tracked headings
         };
+        // --- Main Processing Loop ---
         for (const line of lines) {
-            if (line.startsWith("#")) {
-                processChunk();
+            const isHeading = line.startsWith("#");
+            if (isHeading) {
+                // Update Hierarchy Stack for the new heading
                 const levelMatch = line.match(/^(#+)/);
-                let level = levelMatch ? levelMatch[1].length : 1;
-                const heading = line.replace(/^#+\s*/, "").trim();
-                logger.debug(`Found heading (level ${level}): ${heading}`);
-                while (headingHierarchy.length < level - 1) {
-                    headingHierarchy.push("");
-                }
-                if (level <= headingHierarchy.length) {
-                    headingHierarchy = headingHierarchy.slice(0, level - 1);
+                const level = levelMatch ? levelMatch[1].length : 1;
+                // Clean heading: remove markdown prefix and anchor links like [](#anchor-id)
+                const headingText = line
+                    .replace(/^#+\s*/, "") // Remove ## prefix
+                    .replace(/\[.*?\]\(#[^)]*\)/g, "") // Remove [text](#anchor) patterns
+                    .replace(/\[\]\(#[^)]*\)/g, "") // Remove [](#anchor) patterns  
+                    .trim();
+                // Check if we should merge with previous content
+                const currentTokenCount = utils_1.Utils.tokenize(buffer.trim()).length;
+                const hasBufferContent = currentTokenCount > 0;
+                const bufferIsSmall = currentTokenCount < MIN_TOKENS;
+                // Only merge if:
+                // 1. Buffer has content and is small
+                // 2. Buffer has tracked headings (we're merging sections, not just content)
+                // 3. New heading is at same or deeper level than the deepest heading in buffer (siblings or children)
+                //    If new heading is shallower (e.g., H2 after H3), it's a new section - flush first
+                const deepestBufferLevel = bufferHeadings.length > 0
+                    ? Math.max(...bufferHeadings.map(h => h.level))
+                    : 0;
+                const shouldMerge = hasBufferContent && bufferIsSmall && bufferHeadings.length > 0 &&
+                    level >= deepestBufferLevel;
+                if (!shouldMerge && hasBufferContent) {
+                    // Buffer is large enough OR new heading starts a new section - flush first
+                    flushBuffer();
                 }
-                headingHierarchy[level - 1] = heading;
+                // If shouldMerge is true, we keep the buffer and merge the sections
+                // Reset hierarchy below this level (e.g., H2 reset should clear previous H3s)
+                headingHierarchy = headingHierarchy.slice(0, level - 1);
+                headingHierarchy[level - 1] = headingText;
+                // Track this heading in the buffer
+                bufferHeadings.push({ level, text: headingText });
+                buffer += `${line}\n`;
             }
             else {
-                currentChunk += `${line}\n`;
+                buffer += `${line}\n`;
+                // Safety valve: if a single section is huge, flush it periodically
+                if (utils_1.Utils.tokenize(buffer).length >= MAX_TOKENS) {
+                    flushBuffer();
+                }
             }
         }
-        processChunk();
-        logger.debug(`Chunking complete, created ${chunks.length} chunks`);
+        // Final sweep
+        flushBuffer(true);
+        // Update all chunks with the final total count
+        const totalChunks = Math.floor(chunks.length);
+        for (const chunk of chunks) {
+            chunk.metadata.total_chunks = totalChunks;
+        }
+        logger.debug(`Chunking complete: ${chunks.length} rich context chunks created.`);
         return chunks;
     }
 }

package/dist/database.js

@@ -64,7 +64,9 @@ class DatabaseManager {
                     chunk_id TEXT UNIQUE,
                     content TEXT,
                     url TEXT,
-                    hash TEXT
+                    hash TEXT,
+                    chunk_index INTEGER,
+                    total_chunks INTEGER
                 );
             `);
             logger.info(`SQLite database initialized successfully`);
@@ -226,12 +228,12 @@ class DatabaseManager {
     static prepareSQLiteStatements(db) {
         return {
             insertStmt: db.prepare(`
-                INSERT INTO vec_items (embedding, product_name, version, heading_hierarchy, section, chunk_id, content, url, hash)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                INSERT INTO vec_items (embedding, product_name, version, heading_hierarchy, section, chunk_id, content, url, hash, chunk_index, total_chunks)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             `),
             checkHashStmt: db.prepare(`SELECT hash FROM vec_items WHERE chunk_id = ?`),
             updateStmt: db.prepare(`
-                UPDATE vec_items SET embedding = ?, product_name = ?, version = ?, heading_hierarchy = ?, section = ?, content = ?, url = ?, hash = ?
+                UPDATE vec_items SET embedding = ?, product_name = ?, version = ?, heading_hierarchy = ?, section = ?, content = ?, url = ?, hash = ?, chunk_index = ?, total_chunks = ?
                 WHERE chunk_id = ?
             `),
             getAllChunkIdsStmt: db.prepare(`SELECT chunk_id FROM vec_items`),
@@ -242,22 +244,14 @@ class DatabaseManager {
         const { insertStmt, updateStmt } = this.prepareSQLiteStatements(db);
         const hash = chunkHash || utils_1.Utils.generateHash(chunk.content);
         const transaction = db.transaction(() => {
-            const params = [
-                new Float32Array(embedding),
-                chunk.metadata.product_name,
-                chunk.metadata.version,
-                JSON.stringify(chunk.metadata.heading_hierarchy),
-                chunk.metadata.section,
-                chunk.metadata.chunk_id,
-                chunk.content,
-                chunk.metadata.url,
-                hash
-            ];
+            // Use BigInt for true integer representation in SQLite vec0
+            const chunkIndex = BigInt(chunk.metadata.chunk_index | 0);
+            const totalChunks = BigInt(chunk.metadata.total_chunks | 0);
             try {
-                insertStmt.run(params);
+                insertStmt.run(new Float32Array(embedding), chunk.metadata.product_name, chunk.metadata.version, JSON.stringify(chunk.metadata.heading_hierarchy), chunk.metadata.section, chunk.metadata.chunk_id, chunk.content, chunk.metadata.url, hash, chunkIndex, totalChunks);
             }
             catch (error) {
-                updateStmt.run([...params.slice(0, 8), chunk.metadata.chunk_id]);
+                updateStmt.run(new Float32Array(embedding), chunk.metadata.product_name, chunk.metadata.version, JSON.stringify(chunk.metadata.heading_hierarchy), chunk.metadata.section, chunk.content, chunk.metadata.url, hash, chunkIndex, totalChunks, chunk.metadata.chunk_id);
             }
         });
         transaction();
@@ -288,6 +282,8 @@ class DatabaseManager {
                     url: chunk.metadata.url,
                     hash: hash,
                     original_chunk_id: chunk.metadata.chunk_id,
+                    chunk_index: chunk.metadata.chunk_index,
+                    total_chunks: chunk.metadata.total_chunks,
                 },
             };
             await client.upsert(collectionName, {

package/dist/doc2vec.js

@@ -123,26 +123,64 @@ class Doc2Vec {
         const fetchWithRetry = async (url, params = {}, retries = 5, delay = 5000) => {
             for (let attempt = 0; attempt < retries; attempt++) {
                 try {
+                    // Only log on retries to reduce noise during pagination
+                    if (attempt > 0) {
+                        logger.debug(`GitHub API retry: ${url} (attempt ${attempt + 1}/${retries})`);
+                    }
                     const response = await axios_1.default.get(url, {
                         headers: {
                             Authorization: `token ${GITHUB_TOKEN}`,
                             Accept: 'application/vnd.github.v3+json',
                         },
                         params,
+                        timeout: 30000, // 30 second timeout
                     });
                     return response.data;
                 }
                 catch (error) {
+                    // Enhanced error logging for debugging
+                    const errorDetails = {
+                        code: error.code,
+                        message: error.message,
+                        status: error.response?.status,
+                        isTimeout: error.code === 'ECONNABORTED' || error.message?.includes('timeout'),
+                        isNetworkError: !error.response && error.code,
+                    };
+                    logger.debug(`GitHub API error details: ${JSON.stringify(errorDetails)}`);
                     if (error.response && error.response.status === 403) {
+                        // Check if this is actually a rate limit error
+                        const rateLimitRemaining = error.response.headers['x-ratelimit-remaining'];
                         const resetTime = error.response.headers['x-ratelimit-reset'];
-                        const currentTime = Math.floor(Date.now() / 1000);
-                        const waitTime = resetTime ? (resetTime - currentTime) * 1000 : delay * 2;
-                        logger.warn(`GitHub rate limit exceeded. Waiting ${waitTime / 1000}s`);
-                        await new Promise(res => setTimeout(res, waitTime));
+                        if (rateLimitRemaining === '0' && resetTime) {
+                            const currentTime = Math.floor(Date.now() / 1000);
+                            const resetTimestamp = parseInt(resetTime, 10);
+                            let waitTime = (resetTimestamp - currentTime) * 1000;
+                            // Ensure waitTime is at least 1 second (in case resetTime is in the past)
+                            if (waitTime < 1000) {
+                                waitTime = 1000;
+                            }
+                            logger.warn(`GitHub rate limit exceeded. Waiting ${Math.ceil(waitTime / 1000)}s (attempt ${attempt + 1}/${retries})`);
+                            await new Promise(res => setTimeout(res, waitTime));
+                            // Retry the request after waiting
+                            continue;
+                        }
+                        else {
+                            // Other 403 errors (e.g., forbidden access)
+                            logger.error(`GitHub API returned 403 (not rate limit): ${error.message}`);
+                            throw error;
+                        }
                     }
                     else {
-                        logger.error(`GitHub fetch failed: ${error.message}`);
-                        throw error;
+                        // For non-403 errors, wait before retrying (exponential backoff)
+                        if (attempt < retries - 1) {
+                            const backoffDelay = delay * Math.pow(2, attempt);
+                            logger.warn(`GitHub fetch failed (attempt ${attempt + 1}/${retries}): ${error.message}. Retrying in ${backoffDelay}ms`);
+                            await new Promise(res => setTimeout(res, backoffDelay));
+                        }
+                        else {
+                            logger.error(`GitHub fetch failed after ${retries} attempts: ${error.message} (code: ${error.code || 'unknown'})`);
+                            throw error;
+                        }
                     }
                 }
             }
@@ -154,6 +192,10 @@ class Doc2Vec {
             const perPage = 100;
             const sinceTimestamp = new Date(sinceDate);
             while (true) {
+                // Log progress every 10 pages to reduce noise
+                if (page === 1 || page % 10 === 0) {
+                    logger.debug(`Fetching issues page ${page}... (${issues.length} issues so far)`);
+                }
                 const data = await fetchWithRetry(GITHUB_API_URL, {
                     per_page: perPage,
                     page,