@chanaka_nakandala/integration-agent 1.0.0

package/dist/index.js

@@ -0,0 +1,493 @@
+#!/usr/bin/env node
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { ListToolsRequestSchema, CallToolRequestSchema, ListPromptsRequestSchema, GetPromptRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
+import { initCommand } from './cli/init-command.js';
+import { SearchTool } from './tools/search-tool.js';
+import { CodeGenerationTool } from './tools/code-generation-tool.js';
+import { FileCache } from './cache/file-cache.js';
+import { DocsScraper } from './scrapers/docs-scraper.js';
+// Check for CLI commands
+const args = process.argv.slice(2);
+if (args[0] === 'init') {
+    initCommand()
+        .then(() => process.exit(0))
+        .catch((error) => {
+        console.error('Init failed:', error);
+        process.exit(1);
+    });
+}
+else if (args[0] === 'mcp-server' || args.length === 0) {
+    // Start MCP server
+    const server = new Server({
+        name: 'grubtech-integration-support',
+        version: '1.0.0',
+    }, {
+        capabilities: {
+            tools: {},
+            prompts: {},
+        },
+    });
+    // Initialize tools and cache
+    const cache = new FileCache();
+    const searchTool = new SearchTool();
+    const codeGenerationTool = new CodeGenerationTool();
+    const docsScraper = new DocsScraper(cache);
+    // Track scraping state
+    let isScrapingInProgress = false;
+    let lastScrapedAt = null;
+    // Register MCP prompt handlers
+    server.setRequestHandler(ListPromptsRequestSchema, async () => {
+        return {
+            prompts: [
+                {
+                    name: 'get_started',
+                    description: 'WORKFLOW GUIDE: Start here if this is your first time using the agent. Provides a comprehensive 4-step workflow guide explaining how to use all tools and prompts. Shows example workflows and important notes. Use this prompt when the user asks "how do I use this" or "how does this work" or when starting a new integration project.',
+                    arguments: [],
+                },
+                {
+                    name: 'grubtech_api_question',
+                    description: 'ANSWER QUESTIONS: Use this prompt to answer specific questions about Grubtech API integration. Automatically searches documentation and provides context-aware answers. CRITICAL: Only returns information from official Grubtech documentation - never makes up information. If information is not found in documentation, explicitly states it. Use this when user asks specific "how to" questions like "How do I authenticate?" or "What is the order webhook payload structure?" PREREQUISITE: Documentation must be scraped first using scrape_grubtech_docs tool.',
+                    arguments: [
+                        {
+                            name: 'question',
+                            description: 'The specific question about Grubtech API integration (e.g., "How do I authenticate with API keys?" or "What is the menu sync endpoint?")',
+                            required: true,
+                        },
+                    ],
+                },
+                {
+                    name: 'integration_guide',
+                    description: 'STEP-BY-STEP GUIDES: Generates detailed step-by-step integration guides for specific operations. Searches documentation and creates structured guides with implementation steps. Best used when user needs a complete walkthrough for implementing a specific feature. Example: "Give me a guide for menu synchronization" or "Show me how to handle order webhooks step-by-step". PREREQUISITE: Documentation must be scraped first.',
+                    arguments: [
+                        {
+                            name: 'operation',
+                            description: 'The integration operation to create a guide for. Options: "authentication" (API keys, OAuth), "menu_sync" (uploading menus), "order_handling" (receiving order webhooks), "item_availability" (updating item stock)',
+                            required: true,
+                        },
+                    ],
+                },
+            ],
+        };
+    });
+    server.setRequestHandler(GetPromptRequestSchema, async (request) => {
+        if (request.params.name === 'get_started') {
+            return {
+                messages: [
+                    {
+                        role: 'user',
+                        content: {
+                            type: 'text',
+                            text: `Welcome to the Grubtech Integration Support Agent! This MCP server helps you integrate with Grubtech APIs.
+
+## 🚀 Quick Start Workflow
+
+**Step 1: Download Documentation**
+First, run the \`scrape_grubtech_docs\` tool to download and cache all Grubtech API documentation from https://docs.grubtech.io
+
+\`\`\`
+Use tool: scrape_grubtech_docs
+Parameters: (none required, or force: true to refresh)
+\`\`\`
+
+This downloads 210+ documentation pages covering:
+- Authentication (API Key, OAuth)
+- Ordering Platforms (Store, Menu, Order Integration)
+- POS Applications
+- Logistic Platforms
+- Data API, gData
+- And more...
+
+**Step 2: Search Documentation**
+Use the \`search_grubtech_docs\` tool to find relevant information:
+
+\`\`\`
+Use tool: search_grubtech_docs
+Parameters:
+  - query: "your search keywords"
+  - integrationPhase: (optional) AUTH, MENU_SYNC, ORDER_CREATION, ORDER_STATUS, GO_LIVE
+\`\`\`
+
+**Step 3: Ask Questions**
+Use the \`grubtech_api_question\` prompt to ask specific questions:
+
+\`\`\`
+Use prompt: grubtech_api_question
+Parameters:
+  - question: "your question about Grubtech API"
+\`\`\`
+
+**Step 4: Generate Code**
+Use the \`generate_integration_code\` tool to create code templates:
+
+\`\`\`
+Use tool: generate_integration_code
+Parameters:
+  - operation: authenticate | create_menu | receive_order | update_order_status | update_item_availability
+  - language: typescript | python | java | curl
+\`\`\`
+
+## 📚 Available Tools
+
+1. **scrape_grubtech_docs** - Downloads all documentation (run this first!)
+2. **search_grubtech_docs** - Search cached documentation by keywords
+3. **generate_integration_code** - Generate code snippets in 4 languages
+
+## 💡 Available Prompts
+
+1. **get_started** (this prompt) - Workflow guide
+2. **grubtech_api_question** - Ask questions about Grubtech API
+3. **integration_guide** - Get step-by-step guides for specific operations
+
+## 🎯 Example Workflow
+
+\`\`\`
+1. "Use scrape_grubtech_docs tool to download documentation"
+2. "Search for 'authentication' using search_grubtech_docs"
+3. "Ask: How do I authenticate with the Grubtech API?"
+4. "Generate TypeScript authentication code"
+\`\`\`
+
+## ⚠️ Important Notes
+
+- **Always run scrape_grubtech_docs first** if you haven't already
+- Documentation is cached for 24 hours
+- All answers are based ONLY on official Grubtech documentation
+- If information is not in the documentation, the agent will tell you
+
+Ready to start? Run the scrape_grubtech_docs tool now!`,
+                        },
+                    },
+                ],
+            };
+        }
+        if (request.params.name === 'grubtech_api_question') {
+            const question = request.params.arguments?.question || '';
+            // Search for relevant documentation
+            const searchResults = await searchTool.search(question);
+            let context = '';
+            if (searchResults.results.length === 0) {
+                context = 'No relevant documentation found in the Grubtech documentation portal.';
+            }
+            else {
+                context = 'Relevant documentation from Grubtech:\n\n';
+                searchResults.results.forEach((result, idx) => {
+                    context += `### Source ${idx + 1}: ${result.title}\n`;
+                    context += `URL: ${result.sourceUrl}\n`;
+                    context += `${result.snippet}\n\n`;
+                });
+            }
+            return {
+                messages: [
+                    {
+                        role: 'user',
+                        content: {
+                            type: 'text',
+                            text: `You are a Grubtech API integration assistant. Answer the following question ONLY using the provided documentation context. If the answer is not in the documentation, explicitly state "This information is not available in the Grubtech documentation portal."
+
+CRITICAL RULES:
+- DO NOT make up or assume any information
+- DO NOT use your general knowledge about APIs
+- ONLY use information from the documentation context below
+- If the documentation is insufficient, say so
+
+Question: ${question}
+
+Documentation Context:
+${context}
+
+Please provide an accurate answer based ONLY on the documentation context above.`,
+                        },
+                    },
+                ],
+            };
+        }
+        if (request.params.name === 'integration_guide') {
+            const operation = request.params.arguments?.operation || '';
+            // Map operation to search query
+            const queryMap = {
+                'authentication': 'authentication API key token',
+                'menu_sync': 'menu synchronization upload',
+                'order_handling': 'order webhook receive',
+                'item_availability': 'item availability update',
+            };
+            const query = queryMap[operation] || operation;
+            const searchResults = await searchTool.search(query);
+            let context = '';
+            if (searchResults.results.length === 0) {
+                context = `No documentation found for ${operation} in the Grubtech documentation portal.`;
+            }
+            else {
+                context = 'Step-by-step guide from Grubtech documentation:\n\n';
+                searchResults.results.forEach((result, idx) => {
+                    context += `### Step ${idx + 1}: ${result.title}\n`;
+                    context += `${result.snippet}\n\n`;
+                    if (result.sourceUrl) {
+                        context += `Reference: ${result.sourceUrl}\n\n`;
+                    }
+                });
+            }
+            return {
+                messages: [
+                    {
+                        role: 'user',
+                        content: {
+                            type: 'text',
+                            text: `Create a step-by-step integration guide for ${operation} using ONLY the documentation provided below.
+
+CRITICAL RULES:
+- Use ONLY information from the documentation context
+- Do NOT invent steps or details
+- If the documentation is incomplete, note what is missing
+
+Documentation Context:
+${context}
+
+Please create a clear, step-by-step guide based on the documentation.`,
+                        },
+                    },
+                ],
+            };
+        }
+        throw new Error(`Unknown prompt: ${request.params.name}`);
+    });
+    // Register MCP tool handlers
+    server.setRequestHandler(ListToolsRequestSchema, async () => {
+        return {
+            tools: [
+                {
+                    name: 'scrape_grubtech_docs',
+                    description: 'PREREQUISITE TOOL - RUN FIRST: Downloads and caches all 210+ documentation pages from https://docs.grubtech.io covering Authentication, Ordering Platforms, POS Applications, Logistics, Data API, gData, and more. This tool MUST be run before using any other tools or prompts. Takes 2-3 minutes to complete. Documentation is cached for 24 hours. WHEN TO USE: (1) At the start of any new conversation/session, (2) When search returns "No documentation is cached", (3) Once per day to refresh documentation. Set force=true to refresh even if cache exists.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            force: {
+                                type: 'boolean',
+                                description: 'Set to true to force re-scrape even if fresh cache exists (less than 24 hours old). Default: false. Use force=true when you need the latest documentation or if cached data seems outdated.',
+                            },
+                        },
+                    },
+                },
+                {
+                    name: 'search_grubtech_docs',
+                    description: 'SEARCH DOCUMENTATION: Searches through 14,000+ cached documentation chunks using keyword matching and returns the top 5 most relevant results with source URLs and relevance scores. Use this tool to find specific information before answering questions. WHEN TO USE: (1) Before answering any technical question about Grubtech API, (2) To find endpoint details, payload structures, or authentication methods, (3) To gather context for code generation. RETURNS: Formatted search results with titles, URLs, relevance percentages, and content snippets. Each result includes anti-hallucination warnings. PREREQUISITE: Must run scrape_grubtech_docs first.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            query: {
+                                type: 'string',
+                                description: 'Search keywords or natural language question (e.g., "API authentication", "menu upload endpoint", "order webhook payload structure"). The search uses keyword matching across titles, headings, and content.',
+                            },
+                            integrationPhase: {
+                                type: 'string',
+                                enum: ['AUTH', 'MENU_SYNC', 'ORDER_CREATION', 'ORDER_STATUS', 'GO_LIVE'],
+                                description: 'OPTIONAL: Filter results by integration phase. Use AUTH for authentication topics, MENU_SYNC for menu operations, ORDER_CREATION for receiving orders, ORDER_STATUS for order updates, GO_LIVE for production deployment. Leave empty to search all documentation.',
+                            },
+                        },
+                        required: ['query'],
+                    },
+                },
+                {
+                    name: 'generate_integration_code',
+                    description: 'CODE GENERATOR: Generates production-ready code snippets with complete error handling, type definitions, and inline documentation. Available in 4 languages: TypeScript (Node.js with fetch), Python (requests library), Java (HttpClient), and cURL (shell scripts). Each template includes placeholder markers like {{API_KEY}} that need to be replaced with actual values. WHEN TO USE: After finding documentation via search, when user requests code examples, or when implementing specific integration operations. Returns fully working code with comments and example usage. NOTE: Code is template-based, not generated from documentation - always verify against current API docs.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            operation: {
+                                type: 'string',
+                                enum: [
+                                    'authenticate',
+                                    'create_menu',
+                                    'receive_order',
+                                    'update_order_status',
+                                    'update_item_availability',
+                                ],
+                                description: 'The integration operation: "authenticate" (get API access token), "create_menu" (upload menu data to Grubtech), "receive_order" (handle incoming order webhooks), "update_order_status" (update order lifecycle), "update_item_availability" (toggle menu item stock)',
+                            },
+                            language: {
+                                type: 'string',
+                                enum: ['typescript', 'python', 'java', 'curl'],
+                                description: 'Programming language: "typescript" (Node.js 18+ with fetch API), "python" (Python 3.8+ with requests), "java" (Java 11+ with HttpClient), "curl" (command-line for testing)',
+                            },
+                        },
+                        required: ['operation', 'language'],
+                    },
+                },
+            ],
+        };
+    });
+    server.setRequestHandler(CallToolRequestSchema, async (request) => {
+        const args = request.params.arguments;
+        if (request.params.name === 'scrape_grubtech_docs') {
+            const force = args?.force || false;
+            // Check if already scraping
+            if (isScrapingInProgress) {
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: 'Documentation scraping is already in progress. Please wait...',
+                        },
+                    ],
+                };
+            }
+            // Check if recent cache exists
+            if (!force) {
+                const docs = await cache.getAllCachedDocs();
+                if (docs.length > 0 && lastScrapedAt) {
+                    const hoursSinceLastScrape = (Date.now() - lastScrapedAt.getTime()) / (1000 * 60 * 60);
+                    if (hoursSinceLastScrape < 24) {
+                        return {
+                            content: [
+                                {
+                                    type: 'text',
+                                    text: `Documentation cache is fresh (${Math.round(hoursSinceLastScrape * 10) / 10} hours old, ${docs.length} documents). Use force:true to re-scrape.`,
+                                },
+                            ],
+                        };
+                    }
+                }
+            }
+            // Start scraping
+            isScrapingInProgress = true;
+            try {
+                console.error('Starting documentation scrape...');
+                const chunks = await docsScraper.scrapeDocumentation();
+                lastScrapedAt = new Date();
+                isScrapingInProgress = false;
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Successfully scraped ${chunks.length} documentation chunks from Grubtech portal. Documentation is now available for search and queries.`,
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                isScrapingInProgress = false;
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Failed to scrape documentation: ${error.message}`,
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+        }
+        if (request.params.name === 'search_grubtech_docs') {
+            const response = await searchTool.search(args?.query || '', args?.integrationPhase);
+            // Check if cache is empty
+            if (response.results.length === 0 && response.metadata.totalResults === 0) {
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: 'No documentation is cached. Please run the "scrape_grubtech_docs" tool first to download documentation from the Grubtech portal.',
+                        },
+                    ],
+                };
+            }
+            // Format response with clear instructions
+            let resultText = `# Search Results for: "${response.metadata.query}"\n\n`;
+            if (response.results.length === 0) {
+                resultText += `No relevant documentation found for your query.\n\n`;
+                resultText += `Searched ${response.metadata.totalResults} cached documents.\n\n`;
+                resultText += `IMPORTANT: Answer "I could not find information about this in the Grubtech documentation" - do not make up information.`;
+            }
+            else {
+                resultText += `Found ${response.results.length} relevant results:\n\n`;
+                response.results.forEach((result, idx) => {
+                    resultText += `## Result ${idx + 1}: ${result.title}\n`;
+                    resultText += `**Source:** ${result.sourceUrl}\n`;
+                    resultText += `**Relevance:** ${(result.relevanceScore * 100).toFixed(1)}%\n\n`;
+                    resultText += `${result.snippet}\n\n`;
+                    resultText += `---\n\n`;
+                });
+                resultText += `\nIMPORTANT: Base your answer ONLY on the documentation above. If the answer is not in these results, say "This specific information is not available in the Grubtech documentation I have access to."`;
+            }
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: resultText,
+                    },
+                ],
+            };
+        }
+        if (request.params.name === 'generate_integration_code') {
+            const result = await codeGenerationTool.generate(args?.operation, args?.language);
+            if ('error' in result) {
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: JSON.stringify(result, null, 2),
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: `Generated ${result.operation} code in ${result.language}:\n\n\`\`\`${result.language}\n${result.code}\n\`\`\`\n\nPlaceholders to replace: ${result.metadata.placeholders.join(', ')}`,
+                    },
+                ],
+            };
+        }
+        throw new Error(`Unknown tool: ${request.params.name}`);
+    });
+    // Graceful shutdown handlers
+    process.on('SIGINT', async () => {
+        console.error('Shutting down MCP server (SIGINT)...');
+        process.exit(0);
+    });
+    process.on('SIGTERM', async () => {
+        console.error('Shutting down MCP server (SIGTERM)...');
+        process.exit(0);
+    });
+    // Start MCP server with STDIO transport
+    async function main() {
+        try {
+            const transport = new StdioServerTransport();
+            await server.connect(transport);
+            console.error('🚀 Grubtech Integration Support MCP server started');
+            console.error('');
+            console.error('📖 Capabilities: Tools & Prompts');
+            console.error('');
+            console.error('💡 Available Prompts:');
+            console.error('  - get_started: Learn how to use this agent (START HERE!)');
+            console.error('  - grubtech_api_question: Ask questions about Grubtech API');
+            console.error('  - integration_guide: Get step-by-step integration guides');
+            console.error('');
+            console.error('🔧 Available Tools:');
+            console.error('  - scrape_grubtech_docs: Download documentation (run this first)');
+            console.error('  - search_grubtech_docs: Search cached documentation');
+            console.error('  - generate_integration_code: Generate code templates');
+            console.error('');
+            console.error('✨ Quick Start:');
+            console.error('  1. Use "get_started" prompt to see the workflow guide');
+            console.error('  2. Run "scrape_grubtech_docs" tool to download 210+ pages');
+            console.error('  3. Search and ask questions about Grubtech API');
+            console.error('');
+        }
+        catch (error) {
+            console.error('Failed to start MCP server:', error);
+            process.exit(1);
+        }
+    }
+    main().catch((error) => {
+        console.error('Unhandled error in main:', error);
+        process.exit(1);
+    });
+}
+else {
+    console.error(`Unknown command: ${args[0]}`);
+    console.log('Usage: npx @grubtech/integration-agent [init|mcp-server]');
+    process.exit(1);
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EACL,sBAAsB,EACtB,qBAAqB,EACrB,wBAAwB,EACxB,sBAAsB,GACvB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAEzD,yBAAyB;AACzB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;IACvB,WAAW,EAAE;SACV,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC;KAAM,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,YAAY,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IACzD,mBAAmB;IACnB,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB;QACE,IAAI,EAAE,8BAA8B;QACpC,OAAO,EAAE,OAAO;KACjB,EACD;QACE,YAAY,EAAE;YACZ,KAAK,EAAE,EAAE;YACT,OAAO,EAAE,EAAE;SACZ;KACF,CACF,CAAC;IAEF,6BAA6B;IAC7B,MAAM,KAAK,GAAG,IAAI,SAAS,EAAE,CAAC;IAC9B,MAAM,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC;IACpC,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,EAAE,CAAC;IACpD,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC;IAE3C,uBAAuB;IACvB,IAAI,oBAAoB,GAAG,KAAK,CAAC;IACjC,IAAI,aAAa,GAAgB,IAAI,CAAC;IAEtC,+BAA+B;IAC/B,MAAM,CAAC,iBAAiB,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;QAC5D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,aAAa;oBACnB,WAAW,EAAE,4UAA4U;oBACzV,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,uBAAuB;oBAC7B,WAAW,EAAE,0iBAA0iB;oBACvjB,SAAS,EAAE;wBACT;4BACE,IAAI,EAAE,UAAU;4BAChB,WAAW,EAAE,0IAA0I;4BACvJ,QAAQ,EAAE,IAAI;yBACf;qBACF;iBACF;gBACD;oBACE,IAAI,EAAE,mBAAmB;oBACzB,WAAW,EAAE,uaAAua;oBACpb,SAAS,EAAE;wBACT;4BACE,IAAI,EAAE,WAAW;4BACjB,WAAW,EAAE,qNAAqN;4BAClO,QAAQ,EAAE,IAAI;yBACf;qBACF;iBACF;aACF;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACjE,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YAC1C,OAAO;gBACL,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uDA6EmC;yBAC1C;qBACF;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,uBAAuB,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAI,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,QAAmB,IAAI,EAAE,CAAC;YAEtE,oCAAoC;YACpC,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAExD,IAAI,OAAO,GAAG,EAAE,CAAC;YACjB,IAAI,aAAa,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvC,OAAO,GAAG,uEAAuE,CAAC;YACpF,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,2CAA2C,CAAC;gBACtD,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;oBAC5C,OAAO,IAAI,cAAc,GAAG,GAAG,CAAC,KAAK,MAAM,CAAC,KAAK,IAAI,CAAC;oBACtD,OAAO,IAAI,QAAQ,MAAM,CAAC,SAAS,IAAI,CAAC;oBACxC,OAAO,IAAI,GAAG,MAAM,CAAC,OAAO,MAAM,CAAC;gBACrC,CAAC,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE;;;;;;;;YAQR,QAAQ;;;EAGlB,OAAO;;iFAEwE;yBACpE;qBACF;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;YAChD,MAAM,SAAS,GAAI,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,SAAoB,IAAI,EAAE,CAAC;YAExE,gCAAgC;YAChC,MAAM,QAAQ,GAA2B;gBACvC,gBAAgB,EAAE,8BAA8B;gBAChD,WAAW,EAAE,6BAA6B;gBAC1C,gBAAgB,EAAE,uBAAuB;gBACzC,mBAAmB,EAAE,0BAA0B;aAChD,CAAC;YAEF,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;YAC/C,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAErD,IAAI,OAAO,GAAG,EAAE,CAAC;YACjB,IAAI,aAAa,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvC,OAAO,GAAG,8BAA8B,SAAS,wCAAwC,CAAC;YAC5F,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,qDAAqD,CAAC;gBAChE,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;oBAC5C,OAAO,IAAI,YAAY,GAAG,GAAG,CAAC,KAAK,MAAM,CAAC,KAAK,IAAI,CAAC;oBACpD,OAAO,IAAI,GAAG,MAAM,CAAC,OAAO,MAAM,CAAC;oBACnC,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;wBACrB,OAAO,IAAI,cAAc,MAAM,CAAC,SAAS,MAAM,CAAC;oBAClD,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,+CAA+C,SAAS;;;;;;;;EAQ1E,OAAO;;sEAE6D;yBACzD;qBACF;iBACF;aACF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,6BAA6B;IAC7B,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QAC1D,OAAO;YACL,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,sBAAsB;oBAC5B,WAAW,EAAE,yiBAAyiB;oBACtjB,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,KAAK,EAAE;gCACL,IAAI,EAAE,SAAS;gCACf,WAAW,EAAE,6LAA6L;6BAC3M;yBACF;qBACF;iBACF;gBACD;oBACE,IAAI,EAAE,sBAAsB;oBAC5B,WAAW,EAAE,yoBAAyoB;oBACtpB,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,KAAK,EAAE;gCACL,IAAI,EAAE,QAAQ;gCACd,WAAW,EAAE,8MAA8M;6BAC5N;4BACD,gBAAgB,EAAE;gCAChB,IAAI,EAAE,QAAQ;gCACd,IAAI,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,gBAAgB,EAAE,cAAc,EAAE,SAAS,CAAC;gCACxE,WAAW,EAAE,oQAAoQ;6BAClR;yBACF;wBACD,QAAQ,EAAE,CAAC,OAAO,CAAC;qBACpB;iBACF;gBACD;oBACE,IAAI,EAAE,2BAA2B;oBACjC,WAAW,EAAE,kqBAAkqB;oBAC/qB,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,SAAS,EAAE;gCACT,IAAI,EAAE,QAAQ;gCACd,IAAI,EAAE;oCACJ,cAAc;oCACd,aAAa;oCACb,eAAe;oCACf,qBAAqB;oCACrB,0BAA0B;iCAC3B;gCACD,WAAW,EAAE,uQAAuQ;6BACrR;4BACD,QAAQ,EAAE;gCACR,IAAI,EAAE,QAAQ;gCACd,IAAI,EAAE,CAAC,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;gCAC9C,WAAW,EAAE,6KAA6K;6BAC3L;yBACF;wBACD,QAAQ,EAAE,CAAC,WAAW,EAAE,UAAU,CAAC;qBACpC;iBACF;aACF;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,SAAgC,CAAC;QAE7D,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,KAAK,CAAC;YAEnC,4BAA4B;YAC5B,IAAI,oBAAoB,EAAE,CAAC;gBACzB,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,+DAA+D;yBACtE;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,+BAA+B;YAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,gBAAgB,EAAE,CAAC;gBAC5C,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,EAAE,CAAC;oBACrC,MAAM,oBAAoB,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;oBACvF,IAAI,oBAAoB,GAAG,EAAE,EAAE,CAAC;wBAC9B,OAAO;4BACL,OAAO,EAAE;gCACP;oCACE,IAAI,EAAE,MAAM;oCACZ,IAAI,EAAE,iCAAiC,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,GAAG,EAAE,eAAe,IAAI,CAAC,MAAM,2CAA2C;iCACvJ;6BACF;yBACF,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAED,iBAAiB;YACjB,oBAAoB,GAAG,IAAI,CAAC;YAC5B,IAAI,CAAC;gBACH,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;gBAClD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,mBAAmB,EAAE,CAAC;gBACvD,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC;gBAC3B,oBAAoB,GAAG,KAAK,CAAC;gBAE7B,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,wBAAwB,MAAM,CAAC,MAAM,oGAAoG;yBAChJ;qBACF;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,oBAAoB,GAAG,KAAK,CAAC;gBAC7B,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,mCAAmC,KAAK,CAAC,OAAO,EAAE;yBACzD;qBACF;oBACD,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YACnD,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,MAAM,CACtC,IAAI,EAAE,KAAK,IAAI,EAAE,EACjB,IAAI,EAAE,gBAAgB,CACvB,CAAC;YAEF,0BAA0B;YAC1B,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;gBAC1E,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,kIAAkI;yBACzI;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,0CAA0C;YAC1C,IAAI,UAAU,GAAG,0BAA0B,QAAQ,CAAC,QAAQ,CAAC,KAAK,OAAO,CAAC;YAE1E,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClC,UAAU,IAAI,qDAAqD,CAAC;gBACpE,UAAU,IAAI,YAAY,QAAQ,CAAC,QAAQ,CAAC,YAAY,wBAAwB,CAAC;gBACjF,UAAU,IAAI,yHAAyH,CAAC;YAC1I,CAAC;iBAAM,CAAC;gBACN,UAAU,IAAI,SAAS,QAAQ,CAAC,OAAO,CAAC,MAAM,wBAAwB,CAAC;gBAEvE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;oBACvC,UAAU,IAAI,aAAa,GAAG,GAAG,CAAC,KAAK,MAAM,CAAC,KAAK,IAAI,CAAC;oBACxD,UAAU,IAAI,eAAe,MAAM,CAAC,SAAS,IAAI,CAAC;oBAClD,UAAU,IAAI,kBAAkB,CAAC,MAAM,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;oBAChF,UAAU,IAAI,GAAG,MAAM,CAAC,OAAO,MAAM,CAAC;oBACtC,UAAU,IAAI,SAAS,CAAC;gBAC1B,CAAC,CAAC,CAAC;gBAEH,UAAU,IAAI,wMAAwM,CAAC;YACzN,CAAC;YAED,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,2BAA2B,EAAE,CAAC;YACxD,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,QAAQ,CAC9C,IAAI,EAAE,SAAS,EACf,IAAI,EAAE,QAAQ,CACf,CAAC;YAEF,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;gBACtB,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;yBACtC;qBACF;oBACD,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,aAAa,MAAM,CAAC,SAAS,YAAY,MAAM,CAAC,QAAQ,cAAc,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,IAAI,wCAAwC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;qBAC7L;iBACF;aACF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,iBAAiB,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,6BAA6B;IAC7B,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;QAC9B,OAAO,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACtD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;QAC/B,OAAO,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;QACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,wCAAwC;IACxC,KAAK,UAAU,IAAI;QACjB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;YAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAChC,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;YACpE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;YAClD,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;YACvC,OAAO,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;YAC5E,OAAO,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;YAC7E,OAAO,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;YAC5E,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YACrC,OAAO,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC;YACnF,OAAO,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;YACvE,OAAO,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;YACxE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;YAChC,OAAO,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;YACzE,OAAO,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;YAC7E,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACrB,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC"}

package/dist/scrapers/docs-scraper.d.ts

@@ -0,0 +1,28 @@
+import { FileCache } from '../cache/file-cache.js';
+import type { DocumentChunk } from '@grubtech/agent-core';
+export declare class DocsScraper {
+    private axios;
+    private cache;
+    private visited;
+    private lastRequestTime;
+    private readonly RATE_LIMIT_MS;
+    private readonly BASE_URL;
+    private readonly START_URL;
+    constructor(cache: FileCache);
+    private rateLimit;
+    private urlToKey;
+    private shouldFollowLink;
+    /**
+     * Extract all navigation links from the sidebar
+     */
+    private getAllDocumentationLinks;
+    /**
+     * Scrape a single page and extract content chunks
+     */
+    scrapePage(url: string): Promise<DocumentChunk[]>;
+    /**
+     * Main scraping method - extracts all documentation links and scrapes each page
+     */
+    scrapeDocumentation(): Promise<DocumentChunk[]>;
+}
+//# sourceMappingURL=docs-scraper.d.ts.map

package/dist/scrapers/docs-scraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"docs-scraper.d.ts","sourceRoot":"","sources":["../../src/scrapers/docs-scraper.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAG1D,qBAAa,WAAW;IACtB,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,KAAK,CAAY;IACzB,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,eAAe,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAO;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA8B;IACvD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmD;gBAEjE,KAAK,EAAE,SAAS;YAYd,SAAS;IAWvB,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,gBAAgB;IAgBxB;;OAEG;YACW,wBAAwB;IAmCtC;;OAEG;IACG,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAiGvD;;OAEG;IACG,mBAAmB,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;CA0BtD"}