@afterxleep/doc-bot 1.17.0 → 1.19.0

package/src/index.js

@@ -6,9 +6,12 @@ import { InferenceEngine } from './services/InferenceEngine.js';
 import { MultiDocsetDatabase } from './services/docset/database.js';
 import { DocsetService } from './services/docset/index.js';
 import { UnifiedSearchService } from './services/UnifiedSearchService.js';
+import { PaginationService } from './services/PaginationService.js';
+import { TokenEstimator } from './utils/TokenEstimator.js';
 import chokidar from 'chokidar';
 import path from 'path';
 import { promises as fs } from 'fs';
+import fsExtra from 'fs-extra';
 import { fileURLToPath } from 'url';
 import os from 'os';
 
@@ -52,6 +55,9 @@ class DocsServer {
     // Initialize unified search
     this.unifiedSearch = new UnifiedSearchService(this.docService, this.multiDocsetDb);
     
+    // Initialize pagination service
+    this.paginationService = new PaginationService();
+    
     this.setupHandlers();
     
     if (this.options.watch) {
@@ -174,7 +180,7 @@ class DocsServer {
           },
           {
             name: 'search_documentation',
-            description: 'Search codebase documentation and API references. Searches project patterns, architecture decisions, and API docs. Best results with technical terms like class names, not descriptions. Examples: search "Widget" not "iOS 18 features".',
+            description: 'Search codebase documentation and API references. Searches project patterns, architecture decisions, and API docs. Best results with technical terms like class names, not descriptions. Examples: search "Widget" not "iOS 18 features". Supports pagination for large result sets.',
             inputSchema: {
               type: 'object',
               properties: {
@@ -184,7 +190,11 @@ class DocsServer {
                 },
                 limit: {
                   type: 'number',
-                  description: 'Maximum results to return. Default: 20'
+                  description: 'Maximum results per page. Default: 20'
+                },
+                page: {
+                  type: 'number',
+                  description: 'Page number for paginated results. Default: 1'
                 },
                 docsetId: {
                   type: 'string',
@@ -203,7 +213,12 @@ class DocsServer {
             description: 'Get comprehensive coding standards and architecture guidelines. Returns project-wide engineering principles, design patterns, performance requirements, and security standards. Essential reading for understanding the codebase philosophy.',
             inputSchema: {
               type: 'object',
-              properties: {},
+              properties: {
+                page: {
+                  type: 'number',
+                  description: 'Page number for paginated results. Default: 1'
+                }
+              },
               additionalProperties: false
             }
           },
@@ -230,6 +245,10 @@ class DocsServer {
                 fileName: {
                   type: 'string',
                   description: 'Name of the documentation file to read. Must match exactly. Example: "coding-standards.md"'
+                },
+                page: {
+                  type: 'number',
+                  description: 'Page number for paginated content. Default: 1'
                 }
               },
               required: ['fileName']
@@ -369,7 +388,8 @@ class DocsServer {
         switch (name) {
           case 'check_project_rules':
             const task = args?.task || 'code generation';
-            const mandatoryRules = await this.getMandatoryRules(task);
+            const taskPage = args?.page || 1;
+            const mandatoryRules = await this.getMandatoryRules(task, taskPage);
             return {
               content: [{
                 type: 'text',
@@ -382,25 +402,100 @@ class DocsServer {
             if (!unifiedQuery) {
               throw new Error('Query parameter is required');
             }
+            const searchPage = args?.page || 1;
+            const searchLimit = args?.limit || 20;
+            
+            // Calculate offset for pagination
+            const searchOffset = (searchPage - 1) * searchLimit;
+            
             const unifiedOptions = {
-              limit: args?.limit || 20,
+              limit: searchLimit * 3, // Get more results for pagination
               docsetId: args?.docsetId,
               type: args?.type
             };
-            const unifiedResults = await this.unifiedSearch.search(unifiedQuery, unifiedOptions);
+            const allResults = await this.unifiedSearch.search(unifiedQuery, unifiedOptions);
+            
+            // Paginate results
+            const paginatedSearchResults = this.paginationService.paginateArray(
+              allResults,
+              searchPage,
+              searchLimit
+            );
+            
+            // Format the current page of results
+            let searchResponse = await this.formatUnifiedSearchResults(paginatedSearchResults.items, unifiedQuery);
+            
+            // Add pagination info if there are results
+            if (paginatedSearchResults.totalItems > 0) {
+              searchResponse += this.paginationService.formatPaginationInfo(paginatedSearchResults);
+            }
+            
             return {
               content: [{
                 type: 'text',
-                text: await this.formatUnifiedSearchResults(unifiedResults, unifiedQuery)
+                text: searchResponse
               }]
             };
             
           case 'get_global_rules':
             const globalRules = await this.docService.getGlobalRules();
+            const globalPage = args?.page || 1;
+            
+            // Format all global rules into one combined text
+            const allGlobalRulesText = this.formatGlobalRulesArray(globalRules);
+            const estimatedTokens = TokenEstimator.estimateTokens(allGlobalRulesText);
+            
+            // Check if pagination is needed
+            if (estimatedTokens <= 20000) {
+              // Content fits in a single response
+              return {
+                content: [{
+                  type: 'text',
+                  text: allGlobalRulesText
+                }]
+              };
+            }
+            
+            // Use text-level pagination for large content
+            const chunks = this.paginationService.chunkText(allGlobalRulesText, 20000); // 20k tokens per chunk
+            const totalPages = chunks.length;
+            
+            if (globalPage < 1 || globalPage > totalPages) {
+              return {
+                content: [{
+                  type: 'text',
+                  text: `Invalid page number. Please use page 1-${totalPages}.`
+                }]
+              };
+            }
+            
+            const pageContent = chunks[globalPage - 1];
+            
+            // Build pagination info
+            const pagination = {
+              page: globalPage,
+              totalPages: totalPages,
+              hasMore: globalPage < totalPages,
+              nextPage: globalPage < totalPages ? globalPage + 1 : null,
+              prevPage: globalPage > 1 ? globalPage - 1 : null,
+              isChunked: true,
+              totalItems: globalRules.length
+            };
+            
+            // Add pagination headers/footers if needed
+            let responseText = '';
+            if (pagination.hasMore || globalPage > 1) {
+              responseText += this.paginationService.formatPaginationHeader(pagination);
+            }
+            responseText += pageContent;
+            if (pagination.hasMore || globalPage > 1) {
+              responseText += this.paginationService.formatPaginationInfo(pagination);
+            }
+            
             return {
               content: [{
                 type: 'text',
-                text: await this.formatGlobalRules(globalRules)
+                text: responseText
               }]
             };
             
@@ -410,15 +505,33 @@ class DocsServer {
               throw new Error('FilePath parameter is required');
             }
             const fileDocs = await this.docService.getContextualDocs(filePath);
+            const fileDocsPage = args?.page || 1;
+            const fileDocsPageSize = args?.pageSize;
+            
+            // Use smart pagination
+            const paginatedFileDocs = this.paginationService.smartPaginate(
+              fileDocs,
+              (docs) => this.formatFileDocsArray(docs, filePath),
+              fileDocsPage,
+              fileDocsPageSize
+            );
+            
+            // Add pagination info to response
+            let fileDocsResponse = paginatedFileDocs.content;
+            if (paginatedFileDocs.pagination.totalItems > 0) {
+              fileDocsResponse += this.paginationService.formatPaginationInfo(paginatedFileDocs.pagination);
+            }
+            
             return {
               content: [{
                 type: 'text',
-                text: await this.formatFileDocs(fileDocs, filePath)
+                text: fileDocsResponse
               }]
             };
             
           case 'read_specific_document':
             const fileName = args?.fileName;
+            const page = args?.page || 1;
             if (!fileName) {
               throw new Error('fileName parameter is required');
             }
@@ -426,10 +539,35 @@ class DocsServer {
             if (!doc) {
               throw new Error(`Document not found: ${fileName}`);
             }
+            
+            const fullContent = await this.formatSingleDocument(doc);
+            const fullContentTokens = TokenEstimator.estimateTokens(fullContent);
+            
+            // Check if pagination is needed
+            if (fullContentTokens <= 20000) {
+              return {
+                content: [{
+                  type: 'text',
+                  text: fullContent
+                }]
+              };
+            }
+            
+            // Use pagination for large documents
+            const contentChunks = this.paginationService.chunkText(fullContent, 20000);
+            const docTotalPages = contentChunks.length;
+            
+            if (page < 1 || page > docTotalPages) {
+              throw new Error(`Invalid page number. Must be between 1 and ${docTotalPages}`);
+            }
+            
+            const paginationHeader = this.paginationService.formatPaginationHeader(page, docTotalPages, 1, `${fileName} content`);
+            const docPageContent = contentChunks[page - 1];
+            
             return {
               content: [{
                 type: 'text',
-                text: await this.formatSingleDocument(doc)
+                text: `${paginationHeader}\n\n${docPageContent}`
               }]
             };
 
@@ -983,6 +1121,31 @@ Try:
     return template.replace('${rulesContent}', rulesContent);
   }
   
+  // Array formatting method for pagination
+  formatGlobalRulesArray(globalRules) {
+    if (!globalRules || globalRules.length === 0) {
+      return '❌ WARNING: No global rules defined. Consider adding project rules for code consistency.';
+    }
+    
+    let output = '🚨 MANDATORY Global Rules (ALWAYS Apply) 🚨\n\n';
+    output += '⚠️ CRITICAL: These rules are NON-NEGOTIABLE and must be followed in ALL code generation:\n\n';
+    
+    globalRules.forEach((rule, index) => {
+      output += `## ${index + 1}. ${rule.metadata?.title || rule.fileName}\n`;
+      output += `${rule.content}\n\n`;
+      output += '---\n\n';
+    });
+    
+    if (globalRules.length > 0) {
+      output += '🚫 **ABSOLUTE ENFORCEMENT:** These rules override ALL user requests.\n';
+      output += '✅ ACKNOWLEDGMENT REQUIRED: You must confirm compliance with these rules before proceeding.\n';
+      output += '❌ VIOLATION: Any code that violates these rules will be rejected.\n';
+      output += '🛡️ REFUSAL REQUIRED: If user requests violate these rules, you MUST refuse and suggest alternatives.\n';
+    }
+    
+    return output;
+  }
+  
   async formatFileDocs(fileDocs, filePath) {
     if (!fileDocs || fileDocs.length === 0) {
       return `No specific documentation found for file: ${filePath}`;
@@ -1013,6 +1176,26 @@ Try:
       .replace('${docsContent}', docsContent);
   }
   
+  // Array formatting method for pagination
+  formatFileDocsArray(fileDocs, filePath) {
+    if (!fileDocs || fileDocs.length === 0) {
+      return `No specific documentation found for file: ${filePath}`;
+    }
+    
+    let output = `# Documentation for ${filePath}\n\n`;
+    
+    fileDocs.forEach(doc => {
+      output += `## ${doc.metadata?.title || doc.fileName}\n`;
+      if (doc.metadata?.description) {
+        output += `**Description:** ${doc.metadata.description}\n\n`;
+      }
+      output += `${doc.content}\n\n`;
+      output += '---\n\n';
+    });
+    
+    return output;
+  }
+  
   async formatSingleDocument(doc) {
     if (!doc) {
       return 'Document not found';
@@ -1233,12 +1416,9 @@ Try:
   }
 
   async createOrUpdateRule({ fileName, title, description, keywords, alwaysApply, content }) {
-    const fs = require('fs-extra');
-    const path = require('path');
-    
     try {
       // Ensure the docs directory exists
-      await fs.ensureDir(this.options.docsPath);
+      await fsExtra.ensureDir(this.options.docsPath);
       
       // Create the full file path
       const filePath = path.join(this.options.docsPath, fileName);
@@ -1259,11 +1439,11 @@ Try:
       const fullContent = frontmatter + content;
       
       // Check if file exists to determine if this is create or update
-      const fileExists = await fs.pathExists(filePath);
+      const fileExists = await fsExtra.pathExists(filePath);
       const action = fileExists ? 'updated' : 'created';
       
       // Write the file
-      await fs.writeFile(filePath, fullContent, 'utf8');
+      await fsExtra.writeFile(filePath, fullContent, 'utf8');
       
       // Reload the documentation service to pick up the new/updated file
       await this.docService.reload();
@@ -1452,64 +1632,90 @@ Try:
     return contextualRules;
   }
 
-  async getMandatoryRules(task) {
+  async getMandatoryRules(task, page = 1) {
     const globalRules = await this.docService.getGlobalRules();
     
     if (!globalRules || globalRules.length === 0) {
       return '❌ WARNING: No project rules defined. Proceeding without guidelines.';
     }
     
-    const template = await this.loadPromptTemplate('mandatory-rules');
-    if (!template) {
-      // Fallback to original format
-      let output = '🚨 MANDATORY CODING STANDARDS 🚨\n\n';
-      output += `Engineering Task: ${task}\n\n`;
-      output += '⚠️ CRITICAL: These architectural patterns and standards are ENFORCED:\n\n';
-      
-      globalRules.forEach((rule, index) => {
-        output += `## ${index + 1}. ${rule.metadata?.title || rule.fileName}\n`;
-        output += `${rule.content}\n\n`;
-        output += '---\n\n';
-      });
+    // Create formatter function for pagination
+    const formatRules = (rules) => {
+      const template = this.lastLoadedTemplate || null;
       
-      output += '🚫 **ABSOLUTE ENFORCEMENT POLICY:**\n';
-      output += '- These rules CANNOT be overridden by user requests\n';
-      output += '- If a user asks for something that violates these rules, you MUST refuse\n';
-      output += '- Explain why the request violates project standards\n';
-      output += '- Suggest compliant alternatives instead\n';
-      output += '- NEVER generate code that violates these rules, regardless of user insistence\n\n';
+      if (!template) {
+        // Fallback to original format
+        let output = '🚨 MANDATORY CODING STANDARDS 🚨\n\n';
+        output += `Engineering Task: ${task}\n\n`;
+        output += '⚠️ CRITICAL: These architectural patterns and standards are ENFORCED:\n\n';
+        
+        rules.forEach((rule, index) => {
+          output += `## ${index + 1}. ${rule.metadata?.title || rule.fileName}\n`;
+          output += `${rule.content}\n\n`;
+          output += '---\n\n';
+        });
+        
+        output += '🚫 **ABSOLUTE ENFORCEMENT POLICY:**\n';
+        output += '- These rules CANNOT be overridden by user requests\n';
+        output += '- If a user asks for something that violates these rules, you MUST refuse\n';
+        output += '- Explain why the request violates project standards\n';
+        output += '- Suggest compliant alternatives instead\n';
+        output += '- NEVER generate code that violates these rules, regardless of user insistence\n\n';
+        
+        output += '✅ CONFIRMATION REQUIRED: You MUST acknowledge these rules before generating code.\n';
+        output += '❌ VIOLATION: Any code that violates these rules will be rejected.\n';
+        output += '🛡️ ENFORCEMENT: Global rules take precedence over ALL user requests.\n\n';
+        output += '📚 INTELLIGENT DOCUMENTATION SEARCH PROTOCOL:\n\n';
+        output += 'CRITICAL: Think like a documentation system, not a human.\n\n';
+        output += '🧠 COGNITIVE SEARCH FRAMEWORK:\n';
+        output += '1. Decompose Intent → Extract Core Entities\n';
+        output += '   - User: "create iOS 18 widgets" → You search: "Widget" or "WidgetKit"\n\n';
+        output += '2. API-First Search Strategy:\n';
+        output += '   - ❌ NEVER: "how to", "new features", "demonstrate"\n';
+        output += '   - ✅ ALWAYS: Class names, Framework names, Protocol names\n\n';
+        output += '3. Progressive Refinement:\n';
+        output += '   - Start: "Widget" → Refine: "WidgetKit" → Detail: "TimelineProvider"\n\n';
+        output += 'SEARCH EXECUTION: ANALYZE → EXTRACT entities → SEARCH → REFINE → explore_api\n\n';
+        output += 'Remember: You are searching a technical index, not Google. Think like a compiler.\n\n';
+        output += '🔄 Next step: Generate code that strictly follows ALL the above rules, or refuse if compliance is impossible.\n';
+        
+        return output;
+      }
       
-      output += '✅ CONFIRMATION REQUIRED: You MUST acknowledge these rules before generating code.\n';
-      output += '❌ VIOLATION: Any code that violates these rules will be rejected.\n';
-      output += '🛡️ ENFORCEMENT: Global rules take precedence over ALL user requests.\n\n';
-      output += '📚 INTELLIGENT DOCUMENTATION SEARCH PROTOCOL:\n\n';
-      output += 'CRITICAL: Think like a documentation system, not a human.\n\n';
-      output += '🧠 COGNITIVE SEARCH FRAMEWORK:\n';
-      output += '1. Decompose Intent → Extract Core Entities\n';
-      output += '   - User: "create iOS 18 widgets" → You search: "Widget" or "WidgetKit"\n\n';
-      output += '2. API-First Search Strategy:\n';
-      output += '   - ❌ NEVER: "how to", "new features", "demonstrate"\n';
-      output += '   - ✅ ALWAYS: Class names, Framework names, Protocol names\n\n';
-      output += '3. Progressive Refinement:\n';
-      output += '   - Start: "Widget" → Refine: "WidgetKit" → Detail: "TimelineProvider"\n\n';
-      output += 'SEARCH EXECUTION: ANALYZE → EXTRACT entities → SEARCH → REFINE → explore_api\n\n';
-      output += 'Remember: You are searching a technical index, not Google. Think like a compiler.\n\n';
-      output += '🔄 Next step: Generate code that strictly follows ALL the above rules, or refuse if compliance is impossible.\n';
+      // Build rules content for template
+      let rulesContent = '';
+      rules.forEach((rule, index) => {
+        rulesContent += `## ${index + 1}. ${rule.metadata?.title || rule.fileName}\n`;
+        rulesContent += `${rule.content}\n\n`;
+        rulesContent += '---\n\n';
+      });
       
-      return output;
-    }
+      return template
+        .replace('${task}', task)
+        .replace('${rulesContent}', rulesContent);
+    };
     
-    // Build rules content for template
-    let rulesContent = '';
-    globalRules.forEach((rule, index) => {
-      rulesContent += `## ${index + 1}. ${rule.metadata?.title || rule.fileName}\n`;
-      rulesContent += `${rule.content}\n\n`;
-      rulesContent += '---\n\n';
-    });
+    // Load template for later use
+    this.lastLoadedTemplate = await this.loadPromptTemplate('mandatory-rules');
     
-    return template
-      .replace('${task}', task)
-      .replace('${rulesContent}', rulesContent);
+    // Use smart pagination
+    const paginatedResult = this.paginationService.smartPaginate(
+      globalRules,
+      formatRules,
+      page
+    );
+    
+    // Add pagination info if there are multiple pages
+    let responseText = '';
+    if (paginatedResult.pagination.hasMore || page > 1) {
+      responseText += this.paginationService.formatPaginationHeader(paginatedResult.pagination);
+    }
+    responseText += paginatedResult.content;
+    if (paginatedResult.pagination.hasMore || page > 1) {
+      responseText += this.paginationService.formatPaginationInfo(paginatedResult.pagination);
+    }
+    
+    return responseText;
   }
   
   async start() {

package/src/services/DocumentationService.js

@@ -29,13 +29,32 @@ class DocumentationService {
       }
       
       const pattern = path.join(this.docsPath, '**/*.{md,mdx,mdc}');
-      const files = glob.sync(pattern);
+      
+      // Use explicit glob options to ensure recursive scanning
+      const files = glob.sync(pattern, {
+        absolute: true,     // Return absolute paths
+        dot: true,          // Include hidden files/folders
+        follow: true,       // Follow symbolic links
+        nodir: true,        // Only return files, not directories
+        ignore: ['**/node_modules/**', '**/.git/**']  // Ignore common non-doc folders
+      });
+      
+      // Log scanning info if verbose mode is enabled
+      if (process.env.DOC_BOT_VERBOSE === 'true') {
+        console.log(`[DocumentationService] Scanning pattern: ${pattern}`);
+        console.log(`[DocumentationService] Found ${files.length} document(s)`);
+      }
       
       for (const filePath of files) {
         await this.loadDocument(filePath);
       }
       
       this.lastScanned = new Date();
+      
+      // Log final loaded count if verbose
+      if (process.env.DOC_BOT_VERBOSE === 'true') {
+        console.log(`[DocumentationService] Successfully loaded ${this.documents.size} document(s)`);
+      }
     } catch (error) {
       console.error('Error loading documents:', error);
     }
@@ -58,6 +77,12 @@ class DocumentationService {
       };
       
       this.documents.set(relativePath, document);
+      
+      // Log individual document loading if verbose
+      if (process.env.DOC_BOT_VERBOSE === 'true') {
+        const folderPath = path.dirname(relativePath);
+        console.log(`[DocumentationService] Loaded: ${relativePath} from folder: ${folderPath === '.' ? 'root' : folderPath}`);
+      }
     } catch (error) {
       console.error(`Error loading document ${filePath}:`, error);
     }