@afterxleep/doc-bot 1.18.0 → 1.20.0

package/src/services/__tests__/PaginationService.test.js

@@ -0,0 +1,398 @@
+import { PaginationService } from '../PaginationService.js';
+
+describe('PaginationService', () => {
+  let paginationService;
+
+  beforeEach(() => {
+    paginationService = new PaginationService();
+  });
+
+  describe('estimateTokens', () => {
+    it('should estimate tokens using realistic tokenization', () => {
+      expect(paginationService.estimateTokens('test')).toBe(2); // Realistic: single word + overhead
+      expect(paginationService.estimateTokens('a'.repeat(100))).toBeGreaterThan(25); // More realistic than 4:1 ratio
+      expect(paginationService.estimateTokens('a'.repeat(1000))).toBeGreaterThan(250); // More realistic than 4:1 ratio
+    });
+
+    it('should handle empty or null input', () => {
+      expect(paginationService.estimateTokens('')).toBe(0);
+      expect(paginationService.estimateTokens(null)).toBe(0);
+      expect(paginationService.estimateTokens(undefined)).toBe(0);
+    });
+  });
+
+  describe('needsPagination', () => {
+    it('should return true for content over 24000 tokens', () => {
+      const largeContent = 'a'.repeat(100000); // 25000 tokens
+      expect(paginationService.needsPagination(largeContent)).toBe(true);
+    });
+
+    it('should return false for content under 24000 tokens', () => {
+      const smallContent = 'a'.repeat(50000); // 12500 tokens
+      expect(paginationService.needsPagination(smallContent)).toBe(false);
+    });
+  });
+
+  describe('paginateArray', () => {
+    const testItems = Array.from({ length: 25 }, (_, i) => ({ id: i + 1, name: `Item ${i + 1}` }));
+
+    it('should paginate array with default page size', () => {
+      const result = paginationService.paginateArray(testItems, 1);
+      
+      expect(result.items).toHaveLength(10);
+      expect(result.page).toBe(1);
+      expect(result.pageSize).toBe(10);
+      expect(result.totalPages).toBe(3);
+      expect(result.totalItems).toBe(25);
+      expect(result.hasMore).toBe(true);
+      expect(result.nextPage).toBe(2);
+      expect(result.prevPage).toBe(null);
+    });
+
+    it('should paginate array with custom page size', () => {
+      const result = paginationService.paginateArray(testItems, 1, 5);
+      
+      expect(result.items).toHaveLength(5);
+      expect(result.pageSize).toBe(5);
+      expect(result.totalPages).toBe(5);
+      expect(result.items[0].id).toBe(1);
+      expect(result.items[4].id).toBe(5);
+    });
+
+    it('should handle page 2 correctly', () => {
+      const result = paginationService.paginateArray(testItems, 2, 10);
+      
+      expect(result.items).toHaveLength(10);
+      expect(result.page).toBe(2);
+      expect(result.items[0].id).toBe(11);
+      expect(result.items[9].id).toBe(20);
+      expect(result.prevPage).toBe(1);
+      expect(result.nextPage).toBe(3);
+    });
+
+    it('should handle last page correctly', () => {
+      const result = paginationService.paginateArray(testItems, 3, 10);
+      
+      expect(result.items).toHaveLength(5);
+      expect(result.page).toBe(3);
+      expect(result.hasMore).toBe(false);
+      expect(result.nextPage).toBe(null);
+      expect(result.prevPage).toBe(2);
+    });
+
+    it('should handle empty array', () => {
+      const result = paginationService.paginateArray([], 1, 10);
+      
+      expect(result.items).toHaveLength(0);
+      expect(result.totalPages).toBe(0);
+      expect(result.totalItems).toBe(0);
+      expect(result.hasMore).toBe(false);
+    });
+
+    it('should handle out of bounds page number', () => {
+      const result = paginationService.paginateArray(testItems, 10, 10);
+      
+      expect(result.page).toBe(3); // Should cap at max page
+      expect(result.items).toHaveLength(5);
+    });
+
+    it('should handle negative page number', () => {
+      const result = paginationService.paginateArray(testItems, -1, 10);
+      
+      expect(result.page).toBe(1); // Should default to page 1
+      expect(result.items[0].id).toBe(1);
+    });
+  });
+
+  describe('smartPaginate', () => {
+    const createLargeItems = (count) => {
+      return Array.from({ length: count }, (_, i) => ({
+        id: i + 1,
+        content: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '.repeat(100) // ~1400 chars each
+      }));
+    };
+
+    const formatter = (items) => {
+      return items.map(item => `Item ${item.id}: ${item.content}`).join('\n\n');
+    };
+
+    it('should auto-fit items within token limit when no page size specified', () => {
+      const items = createLargeItems(50);
+      const result = paginationService.smartPaginate(items, formatter, 1);
+      
+      expect(result.pagination.totalItems).toBe(50);
+      expect(result.pagination.page).toBe(1);
+      expect(result.pagination.hasMore).toBe(true);
+      
+      // Content should be under 20000 tokens
+      const estimatedTokens = paginationService.estimateTokens(result.content);
+      expect(estimatedTokens).toBeLessThanOrEqual(20000);
+      expect(estimatedTokens).toBeGreaterThan(0);
+    });
+
+    it('should use specified page size when provided', () => {
+      const items = createLargeItems(20);
+      const result = paginationService.smartPaginate(items, formatter, 1, 5);
+      
+      expect(result.pagination.page).toBe(1);
+      expect(result.pagination.pageSize).toBe(5);
+      expect(result.pagination.totalPages).toBe(4);
+      expect(result.pagination.totalItems).toBe(20);
+    });
+
+    it('should handle page navigation in smart mode', () => {
+      const items = createLargeItems(100);
+      const page1 = paginationService.smartPaginate(items, formatter, 1);
+      const page2 = paginationService.smartPaginate(items, formatter, 2);
+      
+      expect(page1.pagination.page).toBe(1);
+      expect(page1.pagination.nextPage).toBe(2);
+      expect(page1.pagination.prevPage).toBe(null);
+      
+      expect(page2.pagination.page).toBe(2);
+      expect(page2.pagination.prevPage).toBe(1);
+    });
+
+    it('should handle empty items array', () => {
+      const result = paginationService.smartPaginate([], formatter, 1);
+      
+      expect(result.content).toBe('No items found.');
+      expect(result.pagination.totalItems).toBe(0);
+      expect(result.pagination.hasMore).toBe(false);
+    });
+
+    it('should handle single small item', () => {
+      const items = [{ id: 1, content: 'Small content' }];
+      const result = paginationService.smartPaginate(items, formatter, 1);
+      
+      expect(result.pagination.itemsInPage).toBe(1);
+      expect(result.pagination.totalItems).toBe(1);
+      expect(result.pagination.hasMore).toBe(false);
+    });
+  });
+
+  describe('formatPaginationInfo', () => {
+    it('should format pagination info with total pages', () => {
+      const pagination = {
+        page: 2,
+        totalPages: 5,
+        itemsInPage: 10,
+        totalItems: 50,
+        hasMore: true,
+        nextPage: 3,
+        prevPage: 1
+      };
+      
+      const formatted = paginationService.formatPaginationInfo(pagination);
+      
+      expect(formatted).toContain('Page 2 of 5');
+      expect(formatted).toContain('Showing 10 of 50 items');
+      expect(formatted).toContain('Previous: Add `page: 1`');
+      expect(formatted).toContain('Next: Add `page: 3`');
+    });
+
+    it('should format pagination info with estimated pages', () => {
+      const pagination = {
+        page: 1,
+        estimatedTotalPages: 3,
+        itemsInPage: 15,
+        totalItems: 45,
+        hasMore: true,
+        nextPage: 2,
+        prevPage: null
+      };
+      
+      const formatted = paginationService.formatPaginationInfo(pagination);
+      
+      expect(formatted).toContain('Page 1 of ~3');
+      expect(formatted).not.toContain('Previous:');
+      expect(formatted).toContain('Next: Add `page: 2`');
+    });
+
+    it('should format pagination info without navigation on last page', () => {
+      const pagination = {
+        page: 3,
+        totalPages: 3,
+        pageSize: 10,
+        totalItems: 25,
+        hasMore: false,
+        nextPage: null,
+        prevPage: 2
+      };
+      
+      const formatted = paginationService.formatPaginationInfo(pagination);
+      
+      expect(formatted).toContain('Page 3 of 3');
+      expect(formatted).toContain('Previous: Add `page: 2`');
+      expect(formatted).not.toContain('Next:');
+    });
+
+    it('should handle pageSize display correctly', () => {
+      const pagination = {
+        page: 2,
+        pageSize: 20,
+        totalPages: 3,
+        totalItems: 50,
+        hasMore: true,
+        nextPage: 3,
+        prevPage: 1
+      };
+      
+      const formatted = paginationService.formatPaginationInfo(pagination);
+      
+      expect(formatted).toContain('Showing items 21-40 of 50');
+    });
+  });
+
+  describe('chunkText', () => {
+    it('should not chunk text under token limit', () => {
+      const text = 'a'.repeat(50000); // 12500 tokens
+      const chunks = paginationService.chunkText(text);
+      
+      expect(chunks).toHaveLength(1);
+      expect(chunks[0]).toBe(text);
+    });
+
+    it('should chunk large text into multiple parts', () => {
+      const text = 'a'.repeat(100000); // 25000 tokens
+      const chunks = paginationService.chunkText(text);
+      
+      expect(chunks).toHaveLength(2);
+      chunks.forEach(chunk => {
+        const tokens = paginationService.estimateTokens(chunk);
+        expect(tokens).toBeLessThanOrEqual(20500); // Allow small buffer for realistic tokenization
+      });
+    });
+
+    it('should preserve line breaks when chunking', () => {
+      const lines = Array.from({ length: 1000 }, (_, i) => 'a'.repeat(100)).join('\n');
+      const chunks = paginationService.chunkText(lines);
+      
+      expect(chunks.length).toBeGreaterThan(1);
+      chunks.forEach(chunk => {
+        // Each chunk should be properly formatted (allow trimming)
+        expect(chunk.length).toBeGreaterThan(0);
+        expect(typeof chunk).toBe('string');
+      });
+    });
+
+    it('should handle very long single lines', () => {
+      const longLine = 'word '.repeat(20000); // Single line with ~100000 chars
+      const chunks = paginationService.chunkText(longLine);
+      
+      expect(chunks.length).toBeGreaterThan(1);
+      chunks.forEach(chunk => {
+        const tokens = paginationService.estimateTokens(chunk);
+        expect(tokens).toBeLessThanOrEqual(20500); // Allow small buffer for realistic tokenization
+      });
+    });
+
+    it('should handle empty text', () => {
+      const chunks = paginationService.chunkText('');
+      expect(chunks).toEqual(['']);
+    });
+
+    it('should handle null text', () => {
+      const chunks = paginationService.chunkText(null);
+      expect(chunks).toEqual([null]);
+    });
+
+    it('should respect target token parameter', () => {
+      const text = 'word '.repeat(50000); // More complex text that tokenizes differently
+      const chunks = paginationService.chunkText(text, 10000); // 10000 token target
+      
+      expect(chunks.length).toBeGreaterThan(1); // Should need chunking
+      chunks.forEach(chunk => {
+        const tokens = paginationService.estimateTokens(chunk);
+        expect(tokens).toBeLessThanOrEqual(10500); // Allow small buffer
+      });
+    });
+  });
+
+  describe('Edge Cases', () => {
+    it('should handle mixed content sizes in smart pagination', () => {
+      const items = [
+        { id: 1, content: 'a'.repeat(50000) }, // Large item
+        { id: 2, content: 'b'.repeat(10) },    // Small item
+        { id: 3, content: 'c'.repeat(30000) }, // Medium item
+      ];
+      
+      const formatter = (items) => items.map(i => i.content).join('');
+      const result = paginationService.smartPaginate(items, formatter, 1);
+      
+      // Should include at least the first item, maybe more
+      expect(result.pagination.itemsInPage).toBeGreaterThanOrEqual(1);
+      const tokens = paginationService.estimateTokens(result.content);
+      expect(tokens).toBeLessThanOrEqual(20000);
+    });
+
+    it('should handle single item exceeding token limit', () => {
+      const items = [
+        { id: 1, content: 'x'.repeat(100000) } // 25000 tokens - exceeds limit
+      ];
+      
+      const formatter = (items) => items.map(i => i.content).join('');
+      const result = paginationService.smartPaginate(items, formatter, 1);
+      
+      // Should still include the item even if it exceeds limit
+      expect(result.pagination.itemsInPage).toBe(1);
+      expect(result.pagination.hasMore).toBe(false);
+    });
+
+    it('should handle unicode characters correctly', () => {
+      const text = '你好世界🌍'.repeat(1000);
+      const tokens = paginationService.estimateTokens(text);
+      
+      // Unicode chars should still be counted
+      expect(tokens).toBeGreaterThan(0);
+      
+      const chunks = paginationService.chunkText(text.repeat(10));
+      expect(chunks.length).toBeGreaterThanOrEqual(1);
+    });
+
+    it('should handle formatter that returns empty string', () => {
+      const items = [{ id: 1 }, { id: 2 }];
+      const formatter = () => '';
+      
+      const result = paginationService.smartPaginate(items, formatter, 1);
+      expect(result.content).toBe('');
+      expect(result.pagination.itemsInPage).toBe(1); // New pagination logic: 1 item per page
+    });
+
+    it('should handle formatter that throws error gracefully', () => {
+      const items = [{ id: 1 }];
+      const formatter = () => {
+        throw new Error('Formatter error');
+      };
+      
+      expect(() => {
+        paginationService.smartPaginate(items, formatter, 1);
+      }).toThrow('Formatter error');
+    });
+  });
+
+  describe('Performance', () => {
+    it('should handle very large arrays efficiently', () => {
+      const largeArray = Array.from({ length: 10000 }, (_, i) => ({ id: i }));
+      
+      const startTime = Date.now();
+      const result = paginationService.paginateArray(largeArray, 1, 100);
+      const endTime = Date.now();
+      
+      expect(result.items).toHaveLength(100);
+      expect(result.totalItems).toBe(10000);
+      expect(endTime - startTime).toBeLessThan(100); // Should be fast
+    });
+
+    it('should handle very large text efficiently', () => {
+      const largeText = 'a'.repeat(1000000); // 1M chars
+      
+      const startTime = Date.now();
+      const chunks = paginationService.chunkText(largeText);
+      const endTime = Date.now();
+      
+      expect(chunks.length).toBeGreaterThanOrEqual(12); // 1M chars / 80K = ~12.5 chunks
+      expect(endTime - startTime).toBeLessThan(500); // Should complete quickly
+    });
+  });
+});

package/src/utils/TokenEstimator.js

@@ -0,0 +1,134 @@
+/**
+ * TokenEstimator - Realistic token counting for LLM content
+ * 
+ * Implements proper tokenization estimation based on modern LLM patterns.
+ * Much more accurate than naive character-count approaches.
+ */
+export class TokenEstimator {
+  
+  /**
+   * Estimate token count using realistic tokenization patterns
+   * Based on GPT-style tokenization rules and observed patterns
+   * 
+   * @param {string} text - Text to analyze
+   * @returns {number} Estimated token count
+   */
+  static estimateTokens(text) {
+    if (!text) return 0;
+    
+    let tokens = 0;
+    
+    // Split by whitespace and punctuation patterns
+    const words = text.split(/(\s+|[^\w\s])/);
+    
+    for (const word of words) {
+      if (!word) continue;
+      
+      // Whitespace is often merged with adjacent tokens
+      if (/^\s+$/.test(word)) {
+        continue; // Don't count pure whitespace as tokens
+      }
+      
+      // Single punctuation marks are usually 1 token
+      if (/^[^\w\s]$/.test(word)) {
+        tokens += 1;
+        continue;
+      }
+      
+      // Handle different word types
+      if (/^\w+$/.test(word)) {
+        // Regular words: estimate based on length and common patterns
+        if (word.length <= 3) {
+          tokens += 1;  // Short words: 1 token
+        } else if (word.length <= 6) {
+          tokens += 1;  // Medium words: usually 1 token
+        } else if (word.length <= 10) {
+          tokens += Math.ceil(word.length / 5); // Longer words: ~5 chars per token
+        } else {
+          // Very long words (often technical terms): ~4 chars per token
+          tokens += Math.ceil(word.length / 4);
+        }
+      } else {
+        // Mixed content (URLs, emails, code, etc.)
+        // These are often tokenized more aggressively
+        if (word.includes('://') || word.includes('@')) {
+          // URLs and emails: roughly 3-4 chars per token
+          tokens += Math.ceil(word.length / 3.5);
+        } else if (/[A-Z]{2,}/.test(word) || /\d+/.test(word)) {
+          // Acronyms and numbers: often 2-3 chars per token
+          tokens += Math.ceil(word.length / 2.5);
+        } else {
+          // Other mixed content: 4 chars per token
+          tokens += Math.ceil(word.length / 4);
+        }
+      }
+    }
+    
+    // Account for special sequences that are tokenized differently
+    // Code blocks, markdown, JSON, etc. tend to have more tokens
+    const specialPatterns = [
+      /```[\s\S]*?```/g,     // Code blocks
+      /`[^`]+`/g,            // Inline code  
+      /\[[^\]]*\]\([^)]*\)/g, // Markdown links
+      /\*\*[^*]+\*\*/g,      // Bold text
+      /\*[^*]+\*/g,          // Italic text
+      /{[^}]*}/g,            // JSON-like structures
+      /\([^)]*\)/g,          // Parenthetical content
+    ];
+    
+    let specialTokens = 0;
+    for (const pattern of specialPatterns) {
+      const matches = text.match(pattern);
+      if (matches) {
+        for (const match of matches) {
+          // Special content has higher token density
+          specialTokens += Math.ceil(match.length / 3);
+        }
+      }
+    }
+    
+    // Use the higher of the two estimates (word-based vs special-pattern-based)
+    // This accounts for content that's heavily formatted vs plain text
+    const wordBasedEstimate = tokens;
+    const specialContentRatio = specialTokens / Math.max(1, text.length);
+    
+    if (specialContentRatio > 0.1) {
+      // High special content - use pattern-based estimate with adjustment
+      tokens = Math.max(wordBasedEstimate, Math.ceil(text.length / 3.2));
+    } else {
+      // Regular content - use word-based estimate
+      tokens = wordBasedEstimate;
+    }
+    
+    // Add buffer for control tokens, formatting, etc. (5-10% overhead)
+    tokens = Math.ceil(tokens * 1.08);
+    
+    return tokens;
+  }
+  
+  /**
+   * Get the average characters per token for specific text
+   * Useful for chunking operations
+   */
+  static getAvgCharsPerToken(text) {
+    if (!text) return 4; // Fallback
+    const tokens = this.estimateTokens(text);
+    return tokens > 0 ? text.length / tokens : 4;
+  }
+  
+  /**
+   * Check if text exceeds a token limit
+   */
+  static exceedsLimit(text, maxTokens) {
+    return this.estimateTokens(text) > maxTokens;
+  }
+  
+  /**
+   * Estimate how many characters would fit within a token budget
+   * for a given text style
+   */
+  static estimateCharsForTokens(sampleText, targetTokens) {
+    const avgCharsPerToken = this.getAvgCharsPerToken(sampleText);
+    return Math.floor(targetTokens * avgCharsPerToken);
+  }
+}