@deventerprisesoftware/scrapi-mcp 0.1.1 → 0.2.0

package/README.md

@@ -11,22 +11,73 @@ MCP server for using [ScrAPI](https://scrapi.tech) to scrape web pages.
 
 ScrAPI is your ultimate web scraping solution, offering powerful, reliable, and easy-to-use features to extract data from any website effortlessly.
 
+<a href="https://glama.ai/mcp/servers/@DevEnterpriseSoftware/scrapi-mcp">
+  <img width="380" height="200" src="https://glama.ai/mcp/servers/@DevEnterpriseSoftware/scrapi-mcp/badge" alt="ScrAPI Server MCP server" />
+</a>
+
 ## Tools
 
 1. `scrape_url_html`
    - Use a URL to scrape a website using the ScrAPI service and retrieve the result as HTML.
      Use this for scraping website content that is difficult to access because of bot detection, captchas or even geolocation restrictions.
      The result will be in HTML which is preferable if advanced parsing is required.
-   - Input: `url` (string)
+   - Inputs:
+     - `url` (string, required): The URL to scrape
+     - `browserCommands` (string, optional): JSON array of browser commands to execute before scraping
    - Returns: HTML content of the URL
 
 2. `scrape_url_markdown`
    - Use a URL to scrape a website using the ScrAPI service and retrieve the result as Markdown.
      Use this for scraping website content that is difficult to access because of bot detection, captchas or even geolocation restrictions.
      The result will be in Markdown which is preferable if the text content of the webpage is important and not the structural information of the page.
-   - Input: `url` (string)
+   - Inputs:
+     - `url` (string, required): The URL to scrape
+     - `browserCommands` (string, optional): JSON array of browser commands to execute before scraping
    - Returns: Markdown content of the URL
 
+## Browser Commands
+
+Both tools support optional browser commands that allow you to interact with the page before scraping. This is useful for:
+- Clicking buttons (e.g., "Accept Cookies", "Load More")
+- Filling out forms
+- Selecting dropdown options
+- Scrolling to load dynamic content
+- Waiting for elements to appear
+- Executing custom JavaScript
+
+### Available Commands
+
+Commands are provided as a JSON array string. All commands are executed with human-like behavior (random mouse movements, variable typing speed, etc.):
+
+| Command | Format | Description |
+|---------|--------|-------------|
+| **Click** | `{"click": "#buttonId"}` | Click an element using CSS selector |
+| **Input** | `{"input": {"input[name='email']": "value"}}` | Fill an input field |
+| **Select** | `{"select": {"select[name='country']": "USA"}}` | Select from dropdown (by value or text) |
+| **Scroll** | `{"scroll": 1000}` | Scroll down by pixels (negative values scroll up) |
+| **Wait** | `{"wait": 5000}` | Wait for milliseconds (max 15000) |
+| **WaitFor** | `{"waitfor": "#elementId"}` | Wait for element to appear in DOM |
+| **JavaScript** | `{"javascript": "console.log('test')"}` | Execute custom JavaScript code |
+
+### Example Usage
+
+```json
+[
+  {"click": "#accept-cookies"},
+  {"wait": 2000},
+  {"input": {"input[name='search']": "web scraping"}},
+  {"click": "button[type='submit']"},
+  {"waitfor": "#results"},
+  {"scroll": 500}
+]
+```
+
+### Finding CSS Selectors
+
+Need help finding CSS selectors? Try the [Rayrun browser extension](https://chromewebstore.google.com/detail/rayrun/olljocejdgeipcaompahmnfebhkfmnma) to easily select elements and generate selectors.
+
+For more details, see the [Browser Commands documentation](https://scrapi.tech/docs/api_details/v1_scrape/browser_commands).
+
 ## Setup
 
 ### API Key (optional)
@@ -101,4 +152,4 @@ docker build -t deventerprisesoftware/scrapi-mcp -f Dockerfile .
 
 ## License
 
-This MCP server is licensed under the MIT License. This means you are free to use, modify, and distribute the software, subject to the terms and conditions of the MIT License. For more details, please see the LICENSE file in the project repository.
+This MCP server is licensed under the MIT License. This means you are free to use, modify, and distribute the software, subject to the terms and conditions of the MIT License. For more details, please see the LICENSE file in the project repository.

package/dist/index.js

@@ -8,12 +8,19 @@ import { z } from "zod";
 const PORT = process.env.PORT || 5000;
 const SCRAPI_API_KEY = process.env.SCRAPI_API_KEY || "00000000-0000-0000-0000-000000000000";
 const SCRAPI_SERVER_NAME = "ScrAPI MCP Server";
-const SCRAPI_SERVER_VERSION = "0.1.0";
+const SCRAPI_SERVER_VERSION = "0.2.0";
 const app = express();
 app.use(cors({
     origin: "*",
-    exposedHeaders: ["Mcp-Session-Id", "mcp-protocol-version"],
-    allowedHeaders: ["Content-Type", "mcp-session-id"],
+    exposedHeaders: ["mcp-session-id", "mcp-protocol-version"],
+    allowedHeaders: [
+        "Content-Type",
+        "mcp-session-id",
+        "mcp-protocol-version",
+        "Authorization",
+    ],
+    methods: ["GET", "POST", "OPTIONS"],
+    preflightContinue: false
 }));
 app.use(express.json());
 // Define session configuration schema
@@ -38,28 +45,56 @@ export default function createServer({ config, }) {
         title: "Scrape URL and respond with HTML",
         description: "Use a URL to scrape a website using the ScrAPI service and retrieve the result as HTML. " +
             "Use this for scraping website content that is difficult to access because of bot detection, captchas or even geolocation restrictions. " +
-            "The result will be in HTML which is preferable if advanced parsing is required.",
+            "The result will be in HTML which is preferable if advanced parsing is required.\n\n" +
+            "BROWSER COMMANDS: You can optionally provide browser commands to interact with the page before scraping (e.g., clicking buttons, filling forms, scrolling). " +
+            "Provide commands as a JSON array string. Available commands:\n" +
+            "- Click: {\"click\": \"#buttonId\"} - Click an element using CSS selector\n" +
+            "- Input: {\"input\": {\"input[name='email']\": \"value\"}} - Fill an input field\n" +
+            "- Select: {\"select\": {\"select[name='country']\": \"USA\"}} - Select from dropdown\n" +
+            "- Scroll: {\"scroll\": 1000} - Scroll down (negative values scroll up)\n" +
+            "- Wait: {\"wait\": 5000} - Wait milliseconds (max 15000)\n" +
+            "- WaitFor: {\"waitfor\": \"#elementId\"} - Wait for element to appear\n" +
+            "- JavaScript: {\"javascript\": \"console.log('test')\"} - Execute custom JS\n" +
+            "Example: [{\"click\": \"#accept-cookies\"}, {\"wait\": 2000}, {\"input\": {\"input[name='search']\": \"query\"}}]",
         inputSchema: {
             url: z
                 .string()
                 .url({ message: "Invalid URL" })
                 .describe("The URL to scrape"),
+            browserCommands: z
+                .string()
+                .optional()
+                .describe("Optional JSON array of browser commands to execute before scraping. See tool description for available commands and format."),
         },
-    }, async ({ url }) => await scrapeUrl(url, "HTML"));
+    }, async ({ url, browserCommands }) => await scrapeUrl(url, "HTML", browserCommands));
     server.registerTool("scrape_url_markdown", {
         title: "Scrape URL and respond with Markdown",
         description: "Use a URL to scrape a website using the ScrAPI service and retrieve the result as Markdown. " +
             "Use this for scraping website content that is difficult to access because of bot detection, captchas or even geolocation restrictions. " +
-            "The result will be in Markdown which is preferable if the text content of the webpage is important and not the structural information of the page.",
+            "The result will be in Markdown which is preferable if the text content of the webpage is important and not the structural information of the page.\n\n" +
+            "BROWSER COMMANDS: You can optionally provide browser commands to interact with the page before scraping (e.g., clicking buttons, filling forms, scrolling). " +
+            "Provide commands as a JSON array string. Available commands:\n" +
+            "- Click: {\"click\": \"#buttonId\"} - Click an element using CSS selector\n" +
+            "- Input: {\"input\": {\"input[name='email']\": \"value\"}} - Fill an input field\n" +
+            "- Select: {\"select\": {\"select[name='country']\": \"USA\"}} - Select from dropdown\n" +
+            "- Scroll: {\"scroll\": 1000} - Scroll down (negative values scroll up)\n" +
+            "- Wait: {\"wait\": 5000} - Wait milliseconds (max 15000)\n" +
+            "- WaitFor: {\"waitfor\": \"#elementId\"} - Wait for element to appear\n" +
+            "- JavaScript: {\"javascript\": \"console.log('test')\"} - Execute custom JS\n" +
+            "Example: [{\"click\": \"#accept-cookies\"}, {\"wait\": 2000}, {\"input\": {\"input[name='search']\": \"query\"}}]",
         inputSchema: {
             url: z
                 .string()
                 .url({ message: "Invalid URL" })
                 .describe("The URL to scrape"),
+            browserCommands: z
+                .string()
+                .optional()
+                .describe("Optional JSON array of browser commands to execute before scraping. See tool description for available commands and format."),
         },
-    }, async ({ url }) => await scrapeUrl(url, "Markdown"));
-    async function scrapeUrl(url, format) {
-        var body = {
+    }, async ({ url, browserCommands }) => await scrapeUrl(url, "Markdown", browserCommands));
+    async function scrapeUrl(url, format, browserCommands) {
+        const body = {
             url: url,
             useBrowser: true,
             solveCaptchas: true,
@@ -67,6 +102,38 @@ export default function createServer({ config, }) {
             proxyType: "Residential",
             responseFormat: format,
         };
+        // Parse and add browser commands if provided
+        if (browserCommands && browserCommands.trim() !== "") {
+            try {
+                const commands = JSON.parse(browserCommands);
+                if (Array.isArray(commands)) {
+                    body.browserCommands = commands;
+                }
+                else {
+                    return {
+                        content: [
+                            {
+                                type: "text",
+                                text: "Error: Browser commands must be a JSON array.",
+                            },
+                        ],
+                        isError: true,
+                    };
+                }
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : "Unknown error";
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Error: Invalid browser commands format. ${errorMessage}`,
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+        }
         try {
             const response = await fetch("https://api.scrapi.tech/v1/scrape", {
                 method: "POST",
@@ -76,7 +143,7 @@ export default function createServer({ config, }) {
                     "X-API-KEY": config.scrapiApiKey || SCRAPI_API_KEY,
                 },
                 body: JSON.stringify(body),
-                signal: AbortSignal.timeout(30000),
+                signal: AbortSignal.timeout(300000),
             });
             const data = await response.text();
             if (response.ok) {
@@ -104,29 +171,17 @@ export default function createServer({ config, }) {
         }
         catch (error) {
             console.error("Error calling API:", error);
-        }
-        const response = await fetch("https://api.scrapi.tech/v1/scrape", {
-            method: "POST",
-            headers: {
-                "User-Agent": `${SCRAPI_SERVER_NAME} - ${SCRAPI_SERVER_VERSION}`,
-                "Content-Type": "application/json",
-                "X-API-KEY": SCRAPI_API_KEY,
-            },
-            body: JSON.stringify(body),
-            signal: AbortSignal.timeout(30000),
-        });
-        const data = await response.text();
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: data,
-                    _meta: {
-                        mimeType: `text/${format.toLowerCase()}`,
+            const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `Error: Failed to scrape URL. ${errorMessage}`,
                     },
-                },
-            ],
-        };
+                ],
+                isError: true,
+            };
+        }
     }
     return server.server;
 }
@@ -161,6 +216,15 @@ app.all("/mcp", async (req, res) => {
         }
     }
 });
+app.options("/*", (req, res) => {
+    const reqHeaders = req.header("access-control-request-headers");
+    if (reqHeaders) {
+        res.setHeader("Access-Control-Allow-Headers", reqHeaders);
+    }
+    res.setHeader("Access-Control-Allow-Methods", "GET,POST,OPTIONS");
+    res.setHeader("Access-Control-Allow-Origin", "*");
+    res.sendStatus(204);
+});
 // Main function to start the server in the appropriate mode
 async function main() {
     const transport = process.env.TRANSPORT || "stdio";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@deventerprisesoftware/scrapi-mcp",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "MCP server for using ScrAPI to scrape web pages.",
   "keywords": [
     "mcp",
@@ -37,7 +37,7 @@
     "docker:release": "npm run build && npm run docker:build && npm run docker:push"
   },
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.24.2",
+    "@modelcontextprotocol/sdk": "^1.25.2",
     "cors": "^2.8.5",
     "express": "^5.2.1",
     "zod": "^3.25.76"
@@ -45,7 +45,7 @@
   "devDependencies": {
     "@types/cors": "^2.8.19",
     "@types/express": "^5.0.6",
-    "@types/node": "^24.10.1",
+    "@types/node": "^25.0.3",
     "shx": "^0.4.0",
     "typescript": "^5.9.3"
   }