@deventerprisesoftware/scrapi-mcp 0.0.3 → 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 DevEnterprise Software
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,7 +1,13 @@
+![ScrAPI logo](https://raw.githubusercontent.com/DevEnterpriseSoftware/scrapi-sdk-dotnet/master/icon_small.png)
 
 # ScrAPI MCP Server
 
-MCP server for using ScrAPI to scrape web pages.
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![NPM Downloads](https://img.shields.io/npm/dm/@deventerprisesoftware/scrapi-mcp)](https://www.npmjs.com/package/@deventerprisesoftware/scrapi-mcp)
+[![Docker Pulls](https://img.shields.io/docker/pulls/deventerprisesoftware/scrapi-mcp)](https://hub.docker.com/r/deventerprisesoftware/scrapi-mcp)
+[![smithery badge](https://smithery.ai/badge/@DevEnterpriseSoftware/scrapi-mcp)](https://smithery.ai/server/@DevEnterpriseSoftware/scrapi-mcp)
+
+MCP server for using [ScrAPI](https://scrapi.tech) to scrape web pages.
 
 ScrAPI is your ultimate web scraping solution, offering powerful, reliable, and easy-to-use features to extract data from any website effortlessly.
 
@@ -29,6 +35,14 @@ Optionally get an API key from the [ScrAPI website](https://scrapi.tech).
 
 Without an API key you will be limited to one concurrent call and twenty free calls per day with minimal queuing capabilities.
 
+### Cloud Server
+
+The ScrAPI MCP Server is also available in the cloud over SSE at https://api.scrapi.tech/mcp/sse and streamable HTTP at https://api.scrapi.tech/mcp
+
+Cloud MCP servers are not widely supported yet but you can access this directly from your own custom clients or use [MCP Inspector](https://github.com/modelcontextprotocol/inspector) to test it. There is currently no facility to pass through your API key when connecting to the cloud MCP server.
+
+![MCP-Inspector](https://raw.githubusercontent.com/DevEnterpriseSoftware/scrapi-mcp/master/img/mcp-inspector.jpg)
+
 ### Usage with Claude Desktop
 
 Add the following to your `claude_desktop_config.json`:
@@ -38,7 +52,7 @@ Add the following to your `claude_desktop_config.json`:
 ```json
 {
   "mcpServers": {
-    "scrapi": {
+    "ScrAPI": {
       "command": "docker",
       "args": [
         "run",
@@ -56,12 +70,12 @@ Add the following to your `claude_desktop_config.json`:
 }
 ```
 
-### NPX
+#### NPX
 
 ```json
 {
   "mcpServers": {
-    "scrapi": {
+    "ScrAPI": {
       "command": "npx",
       "args": [
         "-y",
@@ -75,6 +89,8 @@ Add the following to your `claude_desktop_config.json`:
 }
 ```
 
+![Claude-Desktop](https://raw.githubusercontent.com/DevEnterpriseSoftware/scrapi-mcp/master/img/claude-desktop.jpg)
+
 ## Build
 
 Docker build:

package/dist/index.js

@@ -1,30 +1,110 @@
 #!/usr/bin/env node
+import express from "express";
+import cors from "cors";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
 import { z } from "zod";
+const PORT = process.env.PORT || 5000;
 const SCRAPI_API_KEY = process.env.SCRAPI_API_KEY || "00000000-0000-0000-0000-000000000000";
 const SCRAPI_SERVER_NAME = "ScrAPI MCP Server";
-const SCRAPI_SERVER_VERSION = "0.0.1";
-const server = new McpServer({
-    name: SCRAPI_SERVER_NAME,
-    version: SCRAPI_SERVER_VERSION,
+const SCRAPI_SERVER_VERSION = "0.1.0";
+const app = express();
+app.use(cors({
+    origin: "*",
+    exposedHeaders: ["Mcp-Session-Id", "mcp-protocol-version"],
+    allowedHeaders: ["Content-Type", "mcp-session-id"],
+}));
+app.use(express.json());
+// Define session configuration schema
+export const configSchema = z.object({
+    scrapiApiKey: z.string().optional().describe("ScrAPI API key for scraping. Leave empty for default limited usage."),
 });
-server.tool("scrape_url_html", "Use a URL to scrape a website using the ScrAPI service and retrieve the result as HTML. " +
-    "Use this for scraping website content that is difficult to access because of bot detection, captchas or even geolocation restrictions. " +
-    "The result will be in HTML which is preferable if advanced parsing is required.", { url: z.string().url({ message: "Invalid URL" }) }, async ({ url }) => await scrapeUrl(url, "HTML"));
-server.tool("scrape_url_markdown", "Use a URL to scrape a website using the ScrAPI service and retrieve the result as Markdown. " +
-    "Use this for scraping website content that is difficult to access because of bot detection, captchas or even geolocation restrictions. " +
-    "The result will be in Markdown which is preferable if the text content of the webpage is important and not the structural information of the page.", { url: z.string().url({ message: "Invalid URL" }) }, async ({ url }) => await scrapeUrl(url, "Markdown"));
-async function scrapeUrl(url, format) {
-    var body = {
-        url: url,
-        useBrowser: true,
-        solveCaptchas: true,
-        acceptDialogs: true,
-        proxyType: "Residential",
-        responseFormat: format,
-    };
-    try {
+// Parse configuration from query parameters
+function parseConfig(req) {
+    const configParam = req.query.config;
+    if (configParam) {
+        return JSON.parse(Buffer.from(configParam, "base64").toString());
+    }
+    return {};
+}
+// Create MCP server with your tools
+export default function createServer({ config, }) {
+    const server = new McpServer({
+        name: SCRAPI_SERVER_NAME,
+        version: SCRAPI_SERVER_VERSION,
+    });
+    server.registerTool("scrape_url_html", {
+        title: "Scrape URL and respond with HTML",
+        description: "Use a URL to scrape a website using the ScrAPI service and retrieve the result as HTML. " +
+            "Use this for scraping website content that is difficult to access because of bot detection, captchas or even geolocation restrictions. " +
+            "The result will be in HTML which is preferable if advanced parsing is required.",
+        inputSchema: {
+            url: z
+                .string()
+                .url({ message: "Invalid URL" })
+                .describe("The URL to scrape"),
+        },
+    }, async ({ url }) => await scrapeUrl(url, "HTML"));
+    server.registerTool("scrape_url_markdown", {
+        title: "Scrape URL and respond with Markdown",
+        description: "Use a URL to scrape a website using the ScrAPI service and retrieve the result as Markdown. " +
+            "Use this for scraping website content that is difficult to access because of bot detection, captchas or even geolocation restrictions. " +
+            "The result will be in Markdown which is preferable if the text content of the webpage is important and not the structural information of the page.",
+        inputSchema: {
+            url: z
+                .string()
+                .url({ message: "Invalid URL" })
+                .describe("The URL to scrape"),
+        },
+    }, async ({ url }) => await scrapeUrl(url, "Markdown"));
+    async function scrapeUrl(url, format) {
+        var body = {
+            url: url,
+            useBrowser: true,
+            solveCaptchas: true,
+            acceptDialogs: true,
+            proxyType: "Residential",
+            responseFormat: format,
+        };
+        try {
+            const response = await fetch("https://api.scrapi.tech/v1/scrape", {
+                method: "POST",
+                headers: {
+                    "User-Agent": `${SCRAPI_SERVER_NAME} - ${SCRAPI_SERVER_VERSION}`,
+                    "Content-Type": "application/json",
+                    "X-API-KEY": config.scrapiApiKey || SCRAPI_API_KEY,
+                },
+                body: JSON.stringify(body),
+                signal: AbortSignal.timeout(30000),
+            });
+            const data = await response.text();
+            if (response.ok) {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: data,
+                            _meta: {
+                                mimeType: `text/${format.toLowerCase()}`,
+                            },
+                        },
+                    ],
+                };
+            }
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: data,
+                    },
+                ],
+                isError: true,
+            };
+        }
+        catch (error) {
+            console.error("Error calling API:", error);
+        }
         const response = await fetch("https://api.scrapi.tech/v1/scrape", {
             method: "POST",
             headers: {
@@ -36,57 +116,76 @@ async function scrapeUrl(url, format) {
             signal: AbortSignal.timeout(30000),
         });
         const data = await response.text();
-        if (response.ok) {
-            return {
-                content: [
-                    {
-                        type: "text",
-                        mimeType: `text/${format.toLowerCase()}`,
-                        text: data,
-                    },
-                ],
-            };
-        }
         return {
             content: [
                 {
                     type: "text",
                     text: data,
+                    _meta: {
+                        mimeType: `text/${format.toLowerCase()}`,
+                    },
                 },
             ],
-            isError: true,
         };
     }
+    return server.server;
+}
+app.all("/mcp", async (req, res) => {
+    try {
+        // Parse configuration
+        const rawConfig = parseConfig ? parseConfig(req) : {};
+        // Validate and parse configuration
+        const config = configSchema
+            ? configSchema.parse({ scrapiApiKey: rawConfig.scrapiApiKey || SCRAPI_API_KEY })
+            : {};
+        const server = createServer({ config });
+        const transport = new StreamableHTTPServerTransport({
+            sessionIdGenerator: undefined,
+        });
+        // Clean up on request close
+        res.on("close", () => {
+            transport.close();
+            server.close();
+        });
+        await server.connect(transport);
+        await transport.handleRequest(req, res, req.body);
+    }
     catch (error) {
-        console.error("Error calling API:", error);
+        console.error("Error handling MCP request:", error);
+        if (!res.headersSent) {
+            res.status(500).json({
+                jsonrpc: "2.0",
+                error: { code: -32603, message: "Internal server error" },
+                id: null,
+            });
+        }
     }
-    const response = await fetch("https://api.scrapi.tech/v1/scrape", {
-        method: "POST",
-        headers: {
-            "User-Agent": `${SCRAPI_SERVER_NAME} - ${SCRAPI_SERVER_VERSION}`,
-            "Content-Type": "application/json",
-            "X-API-KEY": SCRAPI_API_KEY,
-        },
-        body: JSON.stringify(body),
-        signal: AbortSignal.timeout(30000),
-    });
-    const data = await response.text();
-    return {
-        content: [
-            {
-                type: "text",
-                mimeType: `text/${format.toLowerCase()}`,
-                text: data,
+});
+// Main function to start the server in the appropriate mode
+async function main() {
+    const transport = process.env.TRANSPORT || "stdio";
+    if (transport === "http") {
+        // Run in HTTP mode
+        app.listen(PORT, () => {
+            console.log(`MCP HTTP Server listening on port ${PORT}`);
+        });
+    }
+    else {
+        const scrapiApiKey = SCRAPI_API_KEY;
+        // Create server with configuration
+        const server = createServer({
+            config: {
+                scrapiApiKey,
             },
-        ],
-    };
-}
-try {
-    const transport = new StdioServerTransport();
-    await server.connect(transport);
-    console.error(`${SCRAPI_SERVER_NAME} running on stdio.`);
+        });
+        // Start receiving messages on stdin and sending messages on stdout
+        const stdioTransport = new StdioServerTransport();
+        await server.connect(stdioTransport);
+        console.error("MCP Server running in stdio mode");
+    }
 }
-catch (error) {
-    console.error("Fatal error running server:", error);
+// Start the server
+main().catch((error) => {
+    console.error("Server error:", error);
     process.exit(1);
-}
+});

package/package.json

@@ -1,42 +1,52 @@
-{
-  "name": "@deventerprisesoftware/scrapi-mcp",
-  "version": "0.0.3",
-  "description": "MCP server for using ScrAPI to scrape web pages.",
-  "keywords": [
-    "mcp",
-    "web scraper",
-    "web scraping",
-    "web data extractor",
-    "claude",
-    "ai"
-  ],
-  "homepage": "https://scrapi.tech",
-  "bugs": {
-    "url": "https://github.com/DevEnterpriseSoftware/scrapi-mcp/issues"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/DevEnterpriseSoftware/scrapi-mcp.git"
-  },
-  "license": "MIT",
-  "author": "DevEnterprise Software (https://deventerprise.com)",
-  "type": "module",
-  "bin": "dist/index.js",
-  "files": [
-    "dist"
-  ],
-  "scripts": {
-    "build": "tsc && shx chmod +x dist/*.js",
-    "prepare": "npm run build",
-    "watch": "tsc --watch"
-  },
-  "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.9.0",
-    "zod": "^3.24.2"
-  },
-  "devDependencies": {
-    "@types/node": "^22.14.0",
-    "shx": "^0.4.0",
-    "typescript": "^5.8.3"
-  }
-}
+{
+  "name": "@deventerprisesoftware/scrapi-mcp",
+  "version": "0.1.1",
+  "description": "MCP server for using ScrAPI to scrape web pages.",
+  "keywords": [
+    "mcp",
+    "web scraper",
+    "web scraping",
+    "web data extractor",
+    "claude",
+    "ai"
+  ],
+  "homepage": "https://scrapi.tech",
+  "bugs": {
+    "url": "https://github.com/DevEnterpriseSoftware/scrapi-mcp/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/DevEnterpriseSoftware/scrapi-mcp.git"
+  },
+  "license": "MIT",
+  "author": "DevEnterprise Software (https://deventerprise.com)",
+  "type": "module",
+  "bin": {
+    "scrapi-mcp": "dist/index.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc && shx chmod +x dist/*.js",
+    "publish": "npm run build && npm publish",
+    "watch": "tsc --watch",
+    "ncu": "npx npm-check-updates --interactive",
+    "docker:build": "docker build -f Dockerfile -t deventerprisesoftware/scrapi-mcp:v%npm_package_version% -t deventerprisesoftware/scrapi-mcp:latest .",
+    "docker:push": "docker push deventerprisesoftware/scrapi-mcp --all-tags",
+    "docker:release": "npm run build && npm run docker:build && npm run docker:push"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.24.2",
+    "cors": "^2.8.5",
+    "express": "^5.2.1",
+    "zod": "^3.25.76"
+  },
+  "devDependencies": {
+    "@types/cors": "^2.8.19",
+    "@types/express": "^5.0.6",
+    "@types/node": "^24.10.1",
+    "shx": "^0.4.0",
+    "typescript": "^5.9.3"
+  }
+}