@amplify-studio/open-mcp 0.8.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 IS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,244 @@
+# Open MCP
+
+> An open-source MCP (Model Context Protocol) server solution for AI agent integration.
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## Overview
+
+Open MCP is a comprehensive server solution that enables AI assistants (like Claude) to interact with external services through the Model Context Protocol. Currently focused on web search and content extraction, with plans to expand into a full-featured AI agent platform.
+
+## Current Features
+
+### 🌐 Web Search
+- General web queries with pagination
+- Time-based filtering (day, month, year)
+- Language selection
+- Safe search levels
+
+### 📄 URL Content Reading
+- Extract web page content as text/markdown
+- Intelligent caching with TTL
+- Section extraction and pagination options
+
+## Roadmap
+
+### Phase 1: Current (Search & Content)
+- ✅ Web search via Gateway API
+- ✅ URL content reading
+- ✅ Intelligent caching
+
+### Phase 2: Knowledge Base (RAG)
+- 🔄 Document indexing
+- 🔄 Semantic search
+- 🔄 Vector storage integration
+
+### Phase 3: Data Integration
+- ⏳ Database connectors
+- ⏳ API integrations
+- ⏳ File system access
+
+### Phase 4: Agent Framework
+- ⏳ Tool composition
+- ⏳ Workflow orchestration
+- ⏳ Multi-agent coordination
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                        Open MCP                             │
+├─────────────────────────────────────────────────────────────┤
+│                                                               │
+│  ┌────────────┐  ┌────────────┐  ┌────────────┐            │
+│  │   Search   │  │  Content   │  │  Future    │            │
+│  │   Module   │  │   Reader   │  │  Modules   │            │
+│  └─────┬──────┘  └─────┬──────┘  └─────┬──────┘            │
+│        │                │                │                   │
+│        └────────────────┴────────────────┘                   │
+│                         │                                    │
+│                   ┌─────▼─────┐                              │
+│                   │   Core    │                              │
+│                   │  Layer    │                              │
+│                   └─────┬─────┘                              │
+│                         │                                    │
+│  ┌──────────────────────┼──────────────────────┐            │
+│  │                      │                      │            │
+│ ▼▼                     ▼▼                    ▼▼           │
+┌─────────┐         ┌─────────┐          ┌─────────┐         │
+│  Cache  │         │ Gateway │          │ Plugins │         │
+└─────────┘         └─────────┘          └─────────┘         │
+│                                                               │
+└─────────────────────────────────────────────────────────────┘
+                               │
+                               ▼
+                    ┌─────────────────┐
+                    │  Gateway API    │
+                    │  (External)     │
+                    └─────────────────┘
+```
+
+## Installation
+
+### Option 1: From npm (Recommended for Users)
+
+Install from npm registry:
+
+**Using Claude CLI:**
+```bash
+claude mcp add @amplify-studio/open-mcp
+```
+
+**Or with npx:**
+```json
+{
+  "mcpServers": {
+    "open-mcp": {
+      "command": "npx",
+      "args": ["-y", "@amplify-studio/open-mcp"]
+    }
+  }
+}
+```
+
+### Option 2: From GitHub
+
+No installation needed - runs directly from GitHub:
+
+**Claude Desktop Config**:
+```json
+{
+  "mcpServers": {
+    "open-mcp": {
+      "command": "npx",
+      "args": ["-y", "github:amplify-studio/open-mcp"]
+    }
+  }
+}
+```
+
+### Option 3: Custom Gateway
+
+If you have your own gateway instance:
+
+```json
+{
+  "mcpServers": {
+    "open-mcp": {
+      "command": "npx",
+      "args": ["-y", "github:amplify-studio/open-mcp"],
+      "env": {
+        "GATEWAY_URL": "http://your-gateway.com:80"
+      }
+    }
+  }
+}
+```
+
+### Option 4: Local Development (For Developers)
+
+```bash
+# Clone the repository
+git clone https://github.com/amplify-studio/open-mcp.git
+cd open-mcp
+
+# Install dependencies
+npm install
+
+# Build the project
+npm run build
+
+# Run directly
+node dist/index.js
+```
+
+**Claude Desktop Config** (local development):
+```json
+{
+  "mcpServers": {
+    "open-mcp": {
+      "command": "node",
+      "args": ["/path/to/open-mcp/dist/index.js"]
+    }
+  }
+}
+```
+
+### Option 5: Test with MCP Inspector
+
+```bash
+cd open-mcp
+npm run inspector
+# Visit http://localhost:6274 to test
+```
+
+### Option 6: Global Install (Advanced)
+
+```bash
+cd open-mcp
+npm link
+# Then use: open-mcp
+```
+
+## Configuration
+
+| Variable | Required | Default | Description |
+|----------|----------|---------|-------------|
+| `GATEWAY_URL` | No | `http://115.190.91.253:80` | Gateway API base URL |
+| `AUTH_USERNAME` | No | - | Basic auth username |
+| `AUTH_PASSWORD` | No | - | Basic auth password |
+| `HTTP_PROXY` | No | - | Proxy URL for HTTP requests |
+| `HTTPS_PROXY` | No | - | Proxy URL for HTTPS requests |
+| `NO_PROXY` | No | - | Comma-separated bypass list |
+| `MCP_HTTP_PORT` | No | - | Enable HTTP transport on specified port |
+
+## Gateway API
+
+The server connects to a Gateway API that provides:
+
+| API | Method | Endpoint | Description |
+|-----|--------|----------|-------------|
+| Search | GET | `/api/search/` | Web search |
+| Read | GET | `/api/read/{url}` | Extract web content |
+| Health | GET | `/health` | Health check |
+| Status | GET | `/api/status` | Service status |
+
+## Development
+
+```bash
+# Install dependencies
+npm install
+
+# Development mode with file watching
+npm run watch
+
+# Run tests
+npm test
+
+# Test with MCP Inspector
+npm run inspector
+
+# Build for production
+npm run build
+```
+
+## Contributing
+
+We're building an open-source community for AI agent infrastructure. Contributions welcome!
+
+- Fork the repository
+- Create a feature branch
+- Make your changes
+- Submit a pull request
+
+## License
+
+MIT
+
+## Credits
+
+Based on [mcp-searxng](https://github.com/ihor-sokoliuk/mcp-searxng) by Ihor Sokoliuk
+
+---
+
+**Made with ❤️ by [Amplify Studio](https://github.com/amplify-studio)**

package/dist/cache.d.ts

@@ -0,0 +1,26 @@
+interface CacheEntry {
+    htmlContent: string;
+    markdownContent: string;
+    timestamp: number;
+}
+declare class SimpleCache {
+    private cache;
+    private readonly ttlMs;
+    private cleanupInterval;
+    constructor(ttlMs?: number);
+    private startCleanup;
+    private cleanupExpired;
+    get(url: string): CacheEntry | null;
+    set(url: string, htmlContent: string, markdownContent: string): void;
+    clear(): void;
+    destroy(): void;
+    getStats(): {
+        size: number;
+        entries: Array<{
+            url: string;
+            age: number;
+        }>;
+    };
+}
+export declare const urlCache: SimpleCache;
+export { SimpleCache };

package/dist/cache.js

@@ -0,0 +1,68 @@
+class SimpleCache {
+    cache = new Map();
+    ttlMs;
+    cleanupInterval = null;
+    constructor(ttlMs = 60000) {
+        this.ttlMs = ttlMs;
+        this.startCleanup();
+    }
+    startCleanup() {
+        // Clean up expired entries every 30 seconds
+        this.cleanupInterval = setInterval(() => {
+            this.cleanupExpired();
+        }, 30000);
+    }
+    cleanupExpired() {
+        const now = Date.now();
+        for (const [key, entry] of this.cache.entries()) {
+            if (now - entry.timestamp > this.ttlMs) {
+                this.cache.delete(key);
+            }
+        }
+    }
+    get(url) {
+        const entry = this.cache.get(url);
+        if (!entry) {
+            return null;
+        }
+        // Check if expired
+        if (Date.now() - entry.timestamp > this.ttlMs) {
+            this.cache.delete(url);
+            return null;
+        }
+        return entry;
+    }
+    set(url, htmlContent, markdownContent) {
+        this.cache.set(url, {
+            htmlContent,
+            markdownContent,
+            timestamp: Date.now()
+        });
+    }
+    clear() {
+        this.cache.clear();
+    }
+    destroy() {
+        if (this.cleanupInterval) {
+            clearInterval(this.cleanupInterval);
+            this.cleanupInterval = null;
+        }
+        this.clear();
+    }
+    // Get cache statistics for debugging
+    getStats() {
+        const now = Date.now();
+        const entries = Array.from(this.cache.entries()).map(([url, entry]) => ({
+            url,
+            age: now - entry.timestamp
+        }));
+        return {
+            size: this.cache.size,
+            entries
+        };
+    }
+}
+// Global cache instance
+export const urlCache = new SimpleCache();
+// Export for testing and cleanup
+export { SimpleCache };

package/dist/error-handler.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Concise error handling for MCP SearXNG server
+ * Provides clear, focused error messages that identify the root cause
+ */
+export interface ErrorContext {
+    url?: string;
+    searxngUrl?: string;
+    gatewayUrl?: string;
+    proxyAgent?: boolean;
+    username?: string;
+    timeout?: number;
+    query?: string;
+}
+export declare class MCPSearXNGError extends Error {
+    constructor(message: string);
+}
+export declare function createConfigurationError(message: string): MCPSearXNGError;
+export declare function createNetworkError(error: any, context: ErrorContext): MCPSearXNGError;
+export declare function createServerError(status: number, statusText: string, responseBody: string, context: ErrorContext): MCPSearXNGError;
+export declare function createJSONError(responseText: string, context: ErrorContext): MCPSearXNGError;
+export declare function createDataError(data: any, context: ErrorContext): MCPSearXNGError;
+export declare function createNoResultsMessage(query: string): string;
+export declare function createURLFormatError(url: string): MCPSearXNGError;
+export declare function createContentError(message: string, url: string): MCPSearXNGError;
+export declare function createConversionError(error: any, url: string, htmlContent: string): MCPSearXNGError;
+export declare function createTimeoutError(timeout: number, url: string): MCPSearXNGError;
+export declare function createEmptyContentWarning(url: string, htmlLength: number, htmlPreview: string): string;
+export declare function createUnexpectedError(error: any, context: ErrorContext): MCPSearXNGError;
+export declare function validateEnvironment(): string | null;

package/dist/error-handler.js

@@ -0,0 +1,114 @@
+/**
+ * Concise error handling for MCP SearXNG server
+ * Provides clear, focused error messages that identify the root cause
+ */
+export class MCPSearXNGError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'MCPSearXNGError';
+    }
+}
+export function createConfigurationError(message) {
+    return new MCPSearXNGError(`🔧 Configuration Error: ${message}`);
+}
+export function createNetworkError(error, context) {
+    const target = context.searxngUrl ? 'SearXNG server' : (context.gatewayUrl ? 'Gateway server' : 'website');
+    if (error.code === 'ECONNREFUSED') {
+        return new MCPSearXNGError(`🌐 Connection Error: ${target} is not responding (${context.url})`);
+    }
+    if (error.code === 'ENOTFOUND' || error.code === 'EAI_NONAME') {
+        const hostname = context.url ? new URL(context.url).hostname : 'unknown';
+        return new MCPSearXNGError(`🌐 DNS Error: Cannot resolve hostname "${hostname}"`);
+    }
+    if (error.code === 'ETIMEDOUT') {
+        return new MCPSearXNGError(`🌐 Timeout Error: ${target} is too slow to respond`);
+    }
+    if (error.message?.includes('certificate')) {
+        return new MCPSearXNGError(`🌐 SSL Error: Certificate problem with ${target}`);
+    }
+    // For generic fetch failures, provide root cause guidance
+    const errorMsg = error.message || error.code || 'Connection failed';
+    if (errorMsg === 'fetch failed' || errorMsg === 'Connection failed') {
+        const guidance = context.searxngUrl
+            ? 'Check if the SEARXNG_URL is correct and the SearXNG server is available'
+            : (context.gatewayUrl
+                ? 'Check if the GATEWAY_URL is correct and the Gateway server is available'
+                : 'Check if the website URL is accessible');
+        return new MCPSearXNGError(`🌐 Network Error: ${errorMsg}. ${guidance}`);
+    }
+    return new MCPSearXNGError(`🌐 Network Error: ${errorMsg}`);
+}
+export function createServerError(status, statusText, responseBody, context) {
+    const target = context.searxngUrl ? 'SearXNG server' : (context.gatewayUrl ? 'Gateway server' : 'Website');
+    if (status === 403) {
+        const reason = context.searxngUrl ? 'Authentication required or IP blocked' : 'Access blocked (bot detection or geo-restriction)';
+        return new MCPSearXNGError(`🚫 ${target} Error (${status}): ${reason}`);
+    }
+    if (status === 404) {
+        const reason = context.searxngUrl ? 'Search endpoint not found' : 'Page not found';
+        return new MCPSearXNGError(`🚫 ${target} Error (${status}): ${reason}`);
+    }
+    if (status === 429) {
+        return new MCPSearXNGError(`🚫 ${target} Error (${status}): Rate limit exceeded`);
+    }
+    if (status >= 500) {
+        return new MCPSearXNGError(`🚫 ${target} Error (${status}): Internal server error`);
+    }
+    return new MCPSearXNGError(`🚫 ${target} Error (${status}): ${statusText}`);
+}
+export function createJSONError(responseText, context) {
+    const preview = responseText.substring(0, 100).replace(/\n/g, ' ');
+    return new MCPSearXNGError(`🔍 SearXNG Response Error: Invalid JSON format. Response: "${preview}..."`);
+}
+export function createDataError(data, context) {
+    return new MCPSearXNGError(`🔍 SearXNG Data Error: Missing results array in response`);
+}
+export function createNoResultsMessage(query) {
+    return `🔍 No results found for "${query}". Try different search terms or check if SearXNG search engines are working.`;
+}
+export function createURLFormatError(url) {
+    return new MCPSearXNGError(`🔧 URL Format Error: Invalid URL "${url}"`);
+}
+export function createContentError(message, url) {
+    return new MCPSearXNGError(`📄 Content Error: ${message} (${url})`);
+}
+export function createConversionError(error, url, htmlContent) {
+    return new MCPSearXNGError(`🔄 Conversion Error: Cannot convert HTML to Markdown (${url})`);
+}
+export function createTimeoutError(timeout, url) {
+    const hostname = new URL(url).hostname;
+    return new MCPSearXNGError(`⏱️ Timeout Error: ${hostname} took longer than ${timeout}ms to respond`);
+}
+export function createEmptyContentWarning(url, htmlLength, htmlPreview) {
+    return `📄 Content Warning: Page fetched but appears empty after conversion (${url}). May contain only media or require JavaScript.`;
+}
+export function createUnexpectedError(error, context) {
+    return new MCPSearXNGError(`❓ Unexpected Error: ${error.message || String(error)}`);
+}
+export function validateEnvironment() {
+    const issues = [];
+    const gatewayUrl = process.env.GATEWAY_URL;
+    if (gatewayUrl) {
+        try {
+            const url = new URL(gatewayUrl);
+            if (!['http:', 'https:'].includes(url.protocol)) {
+                issues.push(`GATEWAY_URL invalid protocol: ${url.protocol}`);
+            }
+        }
+        catch (error) {
+            issues.push(`GATEWAY_URL invalid format: ${gatewayUrl}`);
+        }
+    }
+    const authUsername = process.env.AUTH_USERNAME;
+    const authPassword = process.env.AUTH_PASSWORD;
+    if (authUsername && !authPassword) {
+        issues.push("AUTH_USERNAME set but AUTH_PASSWORD missing");
+    }
+    else if (!authUsername && authPassword) {
+        issues.push("AUTH_PASSWORD set but AUTH_USERNAME missing");
+    }
+    if (issues.length === 0) {
+        return null;
+    }
+    return `⚠️ Configuration Issues: ${issues.join(', ')}. GATEWAY_URL is optional (defaults to http://115.190.91.253:80)`;
+}

package/dist/http-server.d.ts

@@ -0,0 +1,3 @@
+import express from "express";
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+export declare function createHttpServer(server: Server): Promise<express.Application>;

package/dist/http-server.js

@@ -0,0 +1,150 @@
+import express from "express";
+import cors from "cors";
+import { randomUUID } from "crypto";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
+import { logMessage } from "./logging.js";
+import { packageVersion } from "./index.js";
+export async function createHttpServer(server) {
+    const app = express();
+    app.use(express.json());
+    // Add CORS support for web clients
+    app.use(cors({
+        origin: '*', // Configure appropriately for production
+        exposedHeaders: ['Mcp-Session-Id'],
+        allowedHeaders: ['Content-Type', 'mcp-session-id'],
+    }));
+    // Map to store transports by session ID  
+    const transports = {};
+    // Handle POST requests for client-to-server communication
+    app.post('/mcp', async (req, res) => {
+        const sessionId = req.headers['mcp-session-id'];
+        let transport;
+        if (sessionId && transports[sessionId]) {
+            // Reuse existing transport
+            transport = transports[sessionId];
+            logMessage(server, "debug", `Reusing session: ${sessionId}`);
+        }
+        else if (!sessionId && isInitializeRequest(req.body)) {
+            // New initialization request
+            logMessage(server, "info", "Creating new HTTP session");
+            transport = new StreamableHTTPServerTransport({
+                sessionIdGenerator: () => randomUUID(),
+                onsessioninitialized: (sessionId) => {
+                    transports[sessionId] = transport;
+                    logMessage(server, "debug", `Session initialized: ${sessionId}`);
+                },
+                // DNS rebinding protection disabled by default for backwards compatibility
+                // For production, consider enabling:
+                // enableDnsRebindingProtection: true,
+                // allowedHosts: ['127.0.0.1', 'localhost'],
+            });
+            // Clean up transport when closed
+            transport.onclose = () => {
+                if (transport.sessionId) {
+                    logMessage(server, "debug", `Session closed: ${transport.sessionId}`);
+                    delete transports[transport.sessionId];
+                }
+            };
+            // Connect the existing server to the new transport
+            await server.connect(transport);
+        }
+        else {
+            // Invalid request
+            console.warn(`⚠️  POST request rejected - invalid request:`, {
+                clientIP: req.ip || req.connection.remoteAddress,
+                sessionId: sessionId || 'undefined',
+                hasInitializeRequest: isInitializeRequest(req.body),
+                userAgent: req.headers['user-agent'],
+                contentType: req.headers['content-type'],
+                accept: req.headers['accept']
+            });
+            res.status(400).json({
+                jsonrpc: '2.0',
+                error: {
+                    code: -32000,
+                    message: 'Bad Request: No valid session ID provided',
+                },
+                id: null,
+            });
+            return;
+        }
+        // Handle the request
+        try {
+            await transport.handleRequest(req, res, req.body);
+        }
+        catch (error) {
+            // Log header-related rejections for debugging
+            if (error instanceof Error && error.message.includes('accept')) {
+                console.warn(`⚠️  Connection rejected due to missing headers:`, {
+                    clientIP: req.ip || req.connection.remoteAddress,
+                    userAgent: req.headers['user-agent'],
+                    contentType: req.headers['content-type'],
+                    accept: req.headers['accept'],
+                    error: error.message
+                });
+            }
+            throw error;
+        }
+    });
+    // Handle GET requests for server-to-client notifications via SSE
+    app.get('/mcp', async (req, res) => {
+        const sessionId = req.headers['mcp-session-id'];
+        if (!sessionId || !transports[sessionId]) {
+            console.warn(`⚠️  GET request rejected - missing or invalid session ID:`, {
+                clientIP: req.ip || req.connection.remoteAddress,
+                sessionId: sessionId || 'undefined',
+                userAgent: req.headers['user-agent']
+            });
+            res.status(400).send('Invalid or missing session ID');
+            return;
+        }
+        const transport = transports[sessionId];
+        try {
+            await transport.handleRequest(req, res);
+        }
+        catch (error) {
+            console.warn(`⚠️  GET request failed:`, {
+                clientIP: req.ip || req.connection.remoteAddress,
+                sessionId,
+                error: error instanceof Error ? error.message : String(error)
+            });
+            throw error;
+        }
+    });
+    // Handle DELETE requests for session termination
+    app.delete('/mcp', async (req, res) => {
+        const sessionId = req.headers['mcp-session-id'];
+        if (!sessionId || !transports[sessionId]) {
+            console.warn(`⚠️  DELETE request rejected - missing or invalid session ID:`, {
+                clientIP: req.ip || req.connection.remoteAddress,
+                sessionId: sessionId || 'undefined',
+                userAgent: req.headers['user-agent']
+            });
+            res.status(400).send('Invalid or missing session ID');
+            return;
+        }
+        const transport = transports[sessionId];
+        try {
+            await transport.handleRequest(req, res);
+        }
+        catch (error) {
+            console.warn(`⚠️  DELETE request failed:`, {
+                clientIP: req.ip || req.connection.remoteAddress,
+                sessionId,
+                error: error instanceof Error ? error.message : String(error)
+            });
+            throw error;
+        }
+    });
+    // Health check endpoint
+    app.get('/health', (_req, res) => {
+        res.json({
+            status: 'healthy',
+            server: 'ihor-sokoliuk/mcp-searxng',
+            version: packageVersion,
+            transport: 'http'
+        });
+    });
+    return app;
+}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+declare const packageVersion = "0.8.0";
+export { packageVersion };
+export declare function isWebUrlReadArgs(args: unknown): args is {
+    url: string;
+    startChar?: number;
+    maxLength?: number;
+    section?: string;
+    paragraphRange?: string;
+    readHeadings?: boolean;
+};