@afterxleep/doc-bot 1.18.0 → 1.20.0

package/src/services/PaginationService.js

@@ -0,0 +1,378 @@
+import { TokenEstimator } from '../utils/TokenEstimator.js';
+
+/**
+ * PaginationService - Handles response pagination for MCP server
+ * Ensures responses stay within token limits (25K tokens)
+ */
+export class PaginationService {
+  constructor(options = {}) {
+    // Conservative estimate: ~4 chars per token on average
+    this.maxCharsPerResponse = options.maxCharsPerResponse || 100000; // ~25K tokens
+    this.defaultPageSize = options.defaultPageSize || 10;
+  }
+
+  /**
+   * Estimate token count using realistic tokenization patterns
+   * @param {string} text - Text to analyze
+   * @returns {number} Estimated token count
+   */
+  estimateTokens(text) {
+    return TokenEstimator.estimateTokens(text);
+  }
+
+  /**
+   * Check if response needs pagination
+   */
+  needsPagination(content) {
+    return this.estimateTokens(content) > 24000; // Leave buffer for wrapper text
+  }
+
+  /**
+   * Paginate array of items (documents, rules, etc.)
+   */
+  paginateArray(items, page = 1, pageSize = null) {
+    if (!items || items.length === 0) {
+      return {
+        items: [],
+        page: 1,
+        pageSize: pageSize || this.defaultPageSize,
+        totalPages: 0,
+        totalItems: 0,
+        hasMore: false
+      };
+    }
+
+    const actualPageSize = pageSize || this.defaultPageSize;
+    const totalItems = items.length;
+    const totalPages = Math.ceil(totalItems / actualPageSize);
+    const currentPage = Math.max(1, Math.min(page, totalPages));
+    
+    const startIndex = (currentPage - 1) * actualPageSize;
+    const endIndex = Math.min(startIndex + actualPageSize, totalItems);
+    
+    return {
+      items: items.slice(startIndex, endIndex),
+      page: currentPage,
+      pageSize: actualPageSize,
+      totalPages,
+      totalItems,
+      hasMore: currentPage < totalPages,
+      nextPage: currentPage < totalPages ? currentPage + 1 : null,
+      prevPage: currentPage > 1 ? currentPage - 1 : null
+    };
+  }
+
+  /**
+   * Smart pagination that adjusts page size based on content length
+   */
+  smartPaginate(items, formatter, page = 1, requestedPageSize = null) {
+    if (!items || items.length === 0) {
+      return {
+        content: 'No items found.',
+        pagination: {
+          page: 1,
+          pageSize: 0,
+          totalPages: 0,
+          totalItems: 0,
+          hasMore: false
+        }
+      };
+    }
+
+    // If specific page size requested, use standard pagination
+    if (requestedPageSize) {
+      const result = this.paginateArray(items, page, requestedPageSize);
+      const formattedContent = formatter(result.items);
+      
+      return {
+        content: formattedContent,
+        pagination: {
+          page: result.page,
+          pageSize: result.pageSize,
+          totalPages: result.totalPages,
+          totalItems: result.totalItems,
+          hasMore: result.hasMore,
+          nextPage: result.nextPage,
+          prevPage: result.prevPage
+        }
+      };
+    }
+
+    // First, analyze all items to detect chunking needs and build a page map
+    const itemAnalysis = items.map((item, index) => {
+      const singleContent = formatter([item]);
+      const tokens = this.estimateTokens(singleContent);
+      
+      let chunks = [singleContent];
+      let needsChunking = false;
+      
+      if (tokens > 20000) {
+        chunks = this.chunkText(singleContent, 80000); // ~20k tokens
+        needsChunking = chunks.length > 1;
+      }
+      
+      return {
+        index,
+        item,
+        tokens,
+        needsChunking,
+        chunks,
+        pagesNeeded: needsChunking ? chunks.length : 1
+      };
+    });
+    
+    // Build a logical page map that accounts for chunked items
+    const pageMap = [];
+    let currentPage = 1;
+    
+    for (const analysis of itemAnalysis) {
+      if (analysis.needsChunking) {
+        // Each chunk gets its own page
+        for (let chunkIndex = 0; chunkIndex < analysis.chunks.length; chunkIndex++) {
+          pageMap.push({
+            page: currentPage++,
+            itemIndex: analysis.index,
+            chunkIndex: chunkIndex,
+            content: analysis.chunks[chunkIndex],
+            isChunked: true,
+            totalChunks: analysis.chunks.length
+          });
+        }
+      } else {
+        // Regular item gets one page
+        pageMap.push({
+          page: currentPage++,
+          itemIndex: analysis.index,
+          chunkIndex: null,
+          content: analysis.chunks[0],
+          isChunked: false,
+          totalChunks: 1
+        });
+      }
+    }
+    
+    // Find the requested page
+    const requestedPageData = pageMap.find(p => p.page === page);
+    
+    if (!requestedPageData) {
+      // Page out of range
+      return {
+        content: 'Page not found.',
+        pagination: {
+          page: page,
+          itemsInPage: 0,
+          totalItems: items.length,
+          hasMore: false,
+          estimatedTotalPages: pageMap.length,
+          nextPage: null,
+          prevPage: page > 1 ? Math.min(page - 1, pageMap.length) : null
+        }
+      };
+    }
+    
+    // Return the content for the requested page
+    return {
+      content: requestedPageData.content,
+      pagination: {
+        page: page,
+        itemsInPage: 1,
+        totalItems: items.length,
+        hasMore: page < pageMap.length,
+        estimatedTotalPages: pageMap.length,
+        nextPage: page < pageMap.length ? page + 1 : null,
+        prevPage: page > 1 ? page - 1 : null,
+        isChunked: requestedPageData.isChunked,
+        chunkIndex: requestedPageData.isChunked ? requestedPageData.chunkIndex + 1 : null,
+        totalChunks: requestedPageData.totalChunks,
+        startIndex: requestedPageData.itemIndex,
+        endIndex: requestedPageData.itemIndex + 1
+      }
+    };
+  }
+
+  /**
+   * Format pagination info for display at the bottom of responses
+   */
+  formatPaginationInfo(pagination) {
+    let info = '\n\n---\n';
+    info += `📄 **Page ${pagination.page}`;
+    
+    if (pagination.totalPages) {
+      info += ` of ${pagination.totalPages}`;
+    } else if (pagination.estimatedTotalPages) {
+      info += ` of ~${pagination.estimatedTotalPages}`;
+    }
+    info += '**\n';
+    
+    // Handle chunked content
+    if (pagination.isChunked) {
+      info += `📄 **Content Chunk ${pagination.chunkIndex} of ${pagination.totalChunks}** (Large document split for readability)\n`;
+      info += `📊 Document ${pagination.startIndex + 1} of ${pagination.totalItems} total items\n`;
+    } else if (pagination.itemsInPage !== undefined) {
+      info += `📊 Showing ${pagination.itemsInPage} of ${pagination.totalItems} items\n`;
+    } else if (pagination.pageSize) {
+      const start = (pagination.page - 1) * pagination.pageSize + 1;
+      const end = Math.min(start + pagination.pageSize - 1, pagination.totalItems);
+      info += `📊 Showing items ${start}-${end} of ${pagination.totalItems}\n`;
+    }
+    
+    if (pagination.hasMore || pagination.nextPage || pagination.prevPage) {
+      info += '\n**Navigation:**\n';
+      if (pagination.prevPage) {
+        info += `⬅️ Previous: Add \`page: ${pagination.prevPage}\` to see previous items\n`;
+      }
+      if (pagination.nextPage) {
+        if (pagination.isChunked && pagination.chunkIndex < pagination.totalChunks) {
+          info += `➡️ Next: Add \`page: ${pagination.nextPage}\` to see next chunk\n`;
+        } else {
+          info += `➡️ Next: Add \`page: ${pagination.nextPage}\` to see more items\n`;
+        }
+      }
+    }
+    
+    return info;
+  }
+
+  /**
+   * Format pagination info for display at the TOP of responses (for agent guidance)
+   */
+  formatPaginationHeader(pagination) {
+    let header = '📖 **LARGE DOCUMENT - PAGINATION ACTIVE**\n\n';
+    header += `📄 **Current Page: ${pagination.page}`;
+    
+    if (pagination.totalPages) {
+      header += ` of ${pagination.totalPages}`;
+    } else if (pagination.estimatedTotalPages) {
+      header += ` of ~${pagination.estimatedTotalPages}`;
+    }
+    header += '**\n';
+    
+    // Handle chunked content
+    if (pagination.isChunked) {
+      header += `📄 **Content Chunk ${pagination.chunkIndex} of ${pagination.totalChunks}** (Large document automatically split)\n`;
+      header += `📊 **Content**: Document ${pagination.startIndex + 1} of ${pagination.totalItems} total items\n`;
+    } else if (pagination.itemsInPage !== undefined) {
+      header += `📊 **Content**: Showing ${pagination.itemsInPage} of ${pagination.totalItems} items\n`;
+    }
+    
+    if (pagination.hasMore || pagination.nextPage || pagination.prevPage) {
+      header += '\n🧭 **NAVIGATION GUIDE FOR AGENTS:**\n';
+      if (pagination.prevPage) {
+        header += `  • **Previous page**: Add \`page: ${pagination.prevPage}\` parameter\n`;
+      }
+      if (pagination.nextPage) {
+        if (pagination.isChunked && pagination.chunkIndex < pagination.totalChunks) {
+          header += `  • **Next page**: Add \`page: ${pagination.nextPage}\` parameter (next chunk)\n`;
+        } else {
+          header += `  • **Next page**: Add \`page: ${pagination.nextPage}\` parameter\n`;
+        }
+      }
+      const maxPages = pagination.isChunked ? 
+        Math.max(pagination.totalChunks, pagination.estimatedTotalPages || pagination.totalPages || 1) :
+        (pagination.estimatedTotalPages || pagination.totalPages);
+      if (maxPages) {
+        header += `  • **Jump to page**: Use \`page: N\` (where N = 1-${maxPages})\n`;
+      }
+      
+      if (pagination.nextPage) {
+        header += '\n⚠️ **IMPORTANT**: This response is truncated. Use pagination to see the complete content.\n';
+      }
+    }
+    
+    header += '\n---\n\n';
+    return header;
+  }
+
+  /**
+   * Chunk large text content
+   */
+  chunkText(text, targetTokens = 20000) {
+    if (!text) {
+      return [text];
+    }
+    
+    // If text is under the token limit, return as-is
+    const totalTokens = this.estimateTokens(text);
+    if (totalTokens <= targetTokens) {
+      return [text];
+    }
+    
+    // Estimate characters per token for this specific text
+    const targetChars = TokenEstimator.estimateCharsForTokens(text, targetTokens);
+
+    const chunks = [];
+    
+    // Check if text has line breaks
+    if (text.includes('\n')) {
+      const lines = text.split('\n');
+      let currentChunk = '';
+
+      for (const line of lines) {
+        const testChunk = currentChunk + line + '\n';
+        const testTokens = this.estimateTokens(testChunk);
+        
+        if (testTokens > targetTokens) {
+          if (currentChunk) {
+            chunks.push(currentChunk.trim());
+            currentChunk = line + '\n';
+          } else {
+            // Single line too long, split it by words
+            const words = line.split(' ');
+            let wordChunk = '';
+            for (const word of words) {
+              const testWordChunk = wordChunk + word + ' ';
+              const wordChunkTokens = this.estimateTokens(testWordChunk);
+              
+              if (wordChunkTokens > targetTokens) {
+                if (wordChunk) {
+                  chunks.push(wordChunk.trim());
+                }
+                wordChunk = word + ' ';
+              } else {
+                wordChunk = testWordChunk;
+              }
+            }
+            if (wordChunk) {
+              currentChunk = wordChunk + '\n';
+            }
+          }
+        } else {
+          currentChunk = testChunk;
+        }
+      }
+
+      if (currentChunk) {
+        chunks.push(currentChunk.trim());
+      }
+    } else {
+      // No line breaks, split by estimated token boundaries
+      let i = 0;
+      while (i < text.length) {
+        let endPos = Math.min(i + targetChars, text.length);
+        
+        // Try to break on word boundaries
+        if (endPos < text.length) {
+          const nextSpace = text.indexOf(' ', endPos);
+          const prevSpace = text.lastIndexOf(' ', endPos);
+          
+          if (prevSpace > i && (endPos - prevSpace) < (nextSpace - endPos)) {
+            endPos = prevSpace;
+          } else if (nextSpace !== -1 && (nextSpace - endPos) < 100) {
+            endPos = nextSpace;
+          }
+        }
+        
+        const chunk = text.slice(i, endPos);
+        chunks.push(chunk);
+        i = endPos;
+        
+        // Skip whitespace at the beginning of next chunk
+        while (i < text.length && text[i] === ' ') {
+          i++;
+        }
+      }
+    }
+
+    return chunks;
+  }
+}

package/src/services/__tests__/PaginationService.integration.test.js

@@ -0,0 +1,185 @@
+import { PaginationService } from '../PaginationService.js';
+import { DocumentationService } from '../DocumentationService.js';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import fs from 'fs-extra';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+describe('PaginationService Integration', () => {
+  let paginationService;
+  let docService;
+  let tempDir;
+
+  beforeEach(async () => {
+    paginationService = new PaginationService();
+    
+    // Create temp directory for test docs
+    tempDir = path.join(__dirname, 'temp-test-docs');
+    await fs.ensureDir(tempDir);
+    
+    // Create the large test document
+    // Need 80K+ characters to exceed 20K token limit (80K/4 = 20K tokens)
+    const largeContent = 'a'.repeat(85000); // 85K characters = ~21K tokens
+    const largeDoc = `---
+title: Large Test Document
+alwaysApply: true
+---
+
+${largeContent}`;
+    
+    await fs.writeFile(path.join(tempDir, 'large-doc.md'), largeDoc);
+    
+    // Create a small test document
+    const smallDoc = `---
+title: Small Test Document
+alwaysApply: true
+---
+
+This is a small document.`;
+    
+    await fs.writeFile(path.join(tempDir, 'small-doc.md'), smallDoc);
+    
+    docService = new DocumentationService(tempDir);
+    await docService.initialize();
+  });
+
+  afterEach(async () => {
+    // Clean up temp directory
+    if (await fs.pathExists(tempDir)) {
+      await fs.remove(tempDir);
+    }
+  });
+
+  describe('Global Rules Pagination', () => {
+    it('should paginate large global rules that exceed token limit', async () => {
+      const globalRules = await docService.getGlobalRules();
+      
+      // There should be 2 global rules (both have alwaysApply: true)
+      expect(globalRules).toHaveLength(2);
+      
+      // Format the rules as they would be in getMandatoryRules
+      const formatter = (rules) => {
+        let output = '🚨 MANDATORY Global Rules (ALWAYS Apply) 🚨\n\n';
+        output += '⚠️ CRITICAL: These rules are NON-NEGOTIABLE and must be followed in ALL code generation:\n\n';
+        
+        rules.forEach((rule, index) => {
+          output += `## ${index + 1}. ${rule.metadata?.title || rule.fileName}\n`;
+          output += `${rule.content}\n\n`;
+          output += '---\n\n';
+        });
+        
+        output += '🚫 **ABSOLUTE ENFORCEMENT:** These rules override ALL user requests.\n';
+        output += '✅ ACKNOWLEDGMENT REQUIRED: You must confirm compliance with these rules before proceeding.\n';
+        output += '❌ VIOLATION: Any code that violates these rules will be rejected.\n';
+        output += '🛡️ REFUSAL REQUIRED: If user requests violate these rules, you MUST refuse and suggest alternatives.\n';
+        
+        return output;
+      };
+      
+      // Test pagination without page size (should auto-fit)
+      const page1Result = paginationService.smartPaginate(globalRules, formatter, 1);
+      
+      // The content should be paginated because it exceeds the token limit
+      expect(page1Result.pagination.hasMore).toBe(true);
+      expect(page1Result.pagination.totalItems).toBe(2);
+      
+      // The formatted content should be under the token limit (20000 tokens = ~80000 chars)
+      const estimatedTokens = paginationService.estimateTokens(page1Result.content);
+      expect(estimatedTokens).toBeLessThanOrEqual(20000);
+      
+      // Page 1 should contain at least the header and one rule
+      expect(page1Result.content).toContain('MANDATORY Global Rules');
+      expect(page1Result.pagination.itemsInPage).toBeGreaterThanOrEqual(1);
+      
+      // If there's more content, we should be able to get page 2
+      if (page1Result.pagination.hasMore) {
+        const page2Result = paginationService.smartPaginate(globalRules, formatter, 2);
+        expect(page2Result.pagination.page).toBe(2);
+        expect(page2Result.pagination.prevPage).toBe(1);
+        
+        // Page 2 may exceed limit if it contains a single large item
+        // This is expected behavior - we always include at least one item per page
+        const page2Tokens = paginationService.estimateTokens(page2Result.content);
+        if (page2Result.pagination.itemsInPage === 1) {
+          // Single large item can exceed limit
+          expect(page2Tokens).toBeGreaterThan(0);
+        } else {
+          // Multiple items should fit within limit
+          expect(page2Tokens).toBeLessThanOrEqual(20000);
+        }
+      }
+    });
+    
+    it('should properly indicate pagination in the response', () => {
+      const globalRules = [
+        { 
+          metadata: { title: 'Large Rule' }, 
+          content: 'x'.repeat(100000), // 25000 tokens - exceeds single page
+          fileName: 'large.md'
+        }
+      ];
+      
+      const formatter = (rules) => {
+        return rules.map(r => r.content).join('\n');
+      };
+      
+      const result = paginationService.smartPaginate(globalRules, formatter, 1);
+      
+      // Should include the large item even though it exceeds limit
+      expect(result.pagination.itemsInPage).toBe(1);
+      expect(result.pagination.totalItems).toBe(1);
+      
+      // The pagination info should be formatted correctly
+      const paginationInfo = paginationService.formatPaginationInfo(result.pagination);
+      expect(paginationInfo).toContain('Page 1');
+      expect(paginationInfo).toContain('Showing 1 of 1');
+    });
+    
+    it('should handle mixed content sizes correctly', () => {
+      const mixedRules = [
+        { 
+          metadata: { title: 'Small Rule 1' }, 
+          content: 'Small content',
+          fileName: 'small1.md'
+        },
+        { 
+          metadata: { title: 'Large Rule' }, 
+          content: 'y'.repeat(80000), // 20000 tokens
+          fileName: 'large.md'
+        },
+        { 
+          metadata: { title: 'Small Rule 2' }, 
+          content: 'Another small content',
+          fileName: 'small2.md'
+        }
+      ];
+      
+      const formatter = (rules) => {
+        let output = 'Header\n\n';
+        rules.forEach(rule => {
+          output += `## ${rule.metadata.title}\n`;
+          output += `${rule.content}\n\n`;
+        });
+        return output;
+      };
+      
+      // First page should fit what it can
+      const page1 = paginationService.smartPaginate(mixedRules, formatter, 1);
+      expect(page1.pagination.hasMore).toBe(true);
+      expect(page1.pagination.itemsInPage).toBeGreaterThanOrEqual(1);
+      
+      // Content should be within limits
+      const tokens1 = paginationService.estimateTokens(page1.content);
+      expect(tokens1).toBeLessThanOrEqual(20000);
+      
+      // Should be able to get remaining content
+      if (page1.pagination.hasMore) {
+        const page2 = paginationService.smartPaginate(mixedRules, formatter, 2);
+        const tokens2 = paginationService.estimateTokens(page2.content);
+        expect(tokens2).toBeLessThanOrEqual(22000); // Allow buffer for realistic tokenization
+      }
+    });
+  });
+});