mcp-web-engine 1.0.5 → 1.2.0

package/mcp_protocol.py

@@ -1,210 +0,0 @@
-"""
-MCP Protocol Handler (Spec 2026-07-28 Pure Stateless Core with Header Validation & Per-User Telemetry)
-"""
-import json
-import time
-import secrets
-from typing import Optional
-from fastapi import status, HTTPException
-from mcp_tools import MCP_TOOL_DEFINITIONS, handle_mcp_tool_call
-from logging_obs import logger, metrics
-
-MCP_PROTOCOL_VERSION_2026 = "2026-07-28"
-MCP_PROTOCOL_VERSION_LEGACY = "2025-11-25"
-
-REGISTERED_TOOL_NAMES = {t["name"] for t in MCP_TOOL_DEFINITIONS}
-VALID_2026_METHODS = {"server/discover", "tools/list", "tools/call"}
-
-legacy_sessions = {}
-
-# JSON-RPC 2.0 Error Codes
-PARSE_ERROR = -32700
-INVALID_REQUEST = -32600
-METHOD_NOT_FOUND = -32601
-INVALID_PARAMS = -32602
-INTERNAL_ERROR = -32603
-
-def make_jsonrpc_response(request_id, result):
-    return {
-        "jsonrpc": "2.0",
-        "id": request_id,
-        "result": result
-    }
-
-def make_jsonrpc_error(request_id, code, message, data=None):
-    err_obj = {"code": code, "message": message}
-    if data:
-        err_obj["data"] = data
-    return {
-        "jsonrpc": "2.0",
-        "id": request_id,
-        "error": err_obj
-    }
-
-def validate_2026_mcp_headers(
-    header_mcp_version: Optional[str],
-    header_mcp_method: Optional[str],
-    header_mcp_name: Optional[str],
-    payload: dict
-):
-    if header_mcp_version and header_mcp_version != MCP_PROTOCOL_VERSION_2026:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail={"error": "INVALID_PROTOCOL_VERSION", "message": f"Expected 'MCP-Protocol-Version: {MCP_PROTOCOL_VERSION_2026}'."}
-        )
-
-    if not header_mcp_method:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail={"error": "MISSING_MCP_METHOD", "message": "Header 'Mcp-Method' is required."}
-        )
-
-    clean_method = header_mcp_method.strip()
-    if clean_method not in VALID_2026_METHODS:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail={"error": "INVALID_MCP_METHOD", "message": f"Method '{clean_method}' is not a valid MCP 2026-07-28 method."}
-        )
-
-    body_method = payload.get("method")
-    if clean_method != body_method:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail={"error": "MISMATCHED_MCP_METHOD", "message": f"Header Mcp-Method '{clean_method}' does not match body method '{body_method}'."}
-        )
-
-    if clean_method == "tools/call":
-        if not header_mcp_name:
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail={"error": "MISSING_MCP_NAME", "message": "Header 'Mcp-Name' is required for method 'tools/call'."}
-            )
-
-        clean_name = header_mcp_name.strip()
-        if clean_name not in REGISTERED_TOOL_NAMES:
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail={"error": "INVALID_MCP_NAME", "message": f"Tool '{clean_name}' is not registered."}
-            )
-
-        body_tool_name = payload.get("params", {}).get("name")
-        if clean_name != body_tool_name:
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail={"error": "MISMATCHED_MCP_NAME", "message": f"Header Mcp-Name '{clean_name}' does not match body params.name '{body_tool_name}'."}
-            )
-
-async def process_mcp_2026_stateless(payload: dict, api_key: Optional[str] = None) -> tuple[dict, int]:
-    if not isinstance(payload, dict) or payload.get("jsonrpc") != "2.0":
-        return make_jsonrpc_error(payload.get("id") if isinstance(payload, dict) else None, INVALID_REQUEST, "Invalid JSON-RPC 2.0 request."), status.HTTP_400_BAD_REQUEST
-
-    request_id = payload.get("id")
-    method = payload.get("method")
-    params = payload.get("params", {})
-
-    if method == "server/discover":
-        res_data = {
-            "protocolVersion": MCP_PROTOCOL_VERSION_2026,
-            "server": {
-                "name": "mcp-web-engine",
-                "version": "1.0.0"
-            },
-            "capabilities": {
-                "tools": True
-            }
-        }
-        return make_jsonrpc_response(request_id, res_data), status.HTTP_200_OK
-
-    elif method == "tools/list":
-        res_data = {
-            "tools": MCP_TOOL_DEFINITIONS,
-            "cacheScope": "global",
-            "ttlMs": 3600000,
-            "listChanged": False
-        }
-        return make_jsonrpc_response(request_id, res_data), status.HTTP_200_OK
-
-    elif method == "tools/call":
-        tool_name = params.get("name")
-        arguments = params.get("arguments", {})
-
-        if not tool_name:
-            return make_jsonrpc_error(request_id, INVALID_PARAMS, "Missing tool 'name' in params."), status.HTTP_400_BAD_REQUEST
-
-        start_t = time.time()
-        try:
-            raw_result = await handle_mcp_tool_call(tool_name, arguments)
-            lat = round((time.time() - start_t) * 1000, 2)
-            metrics.record(tool_name, lat, success=True, api_key=api_key)
-
-            content_text = json.dumps(raw_result, indent=2, ensure_ascii=False)
-            mcp_result = {
-                "content": [
-                    {
-                        "type": "text",
-                        "text": content_text
-                    }
-                ],
-                "isError": False
-            }
-            return make_jsonrpc_response(request_id, mcp_result), status.HTTP_200_OK
-
-        except Exception as e:
-            lat = round((time.time() - start_t) * 1000, 2)
-            metrics.record(tool_name, lat, success=False, api_key=api_key)
-            logger.error(f"Error executing MCP tool '{tool_name}': {str(e)}")
-
-            error_content = {
-                "content": [
-                    {
-                        "type": "text",
-                        "text": f"Error executing tool '{tool_name}': {str(e)}"
-                    }
-                ],
-                "isError": True
-            }
-            return make_jsonrpc_response(request_id, error_content), status.HTTP_200_OK
-
-    else:
-        return make_jsonrpc_error(request_id, METHOD_NOT_FOUND, f"Method '{method}' not supported in MCP 2026-07-28 stateless core."), status.HTTP_404_NOT_FOUND
-
-async def process_mcp_2025_legacy_stateful(payload: dict, session_id: Optional[str] = None) -> tuple[dict, int, str]:
-    if session_id and session_id in legacy_sessions:
-        current_sess_id = session_id
-    else:
-        current_sess_id = f"mcp_sess_legacy_{secrets.token_hex(8)}"
-        legacy_sessions[current_sess_id] = {"created": time.time()}
-
-    if not isinstance(payload, dict) or payload.get("jsonrpc") != "2.0":
-        return make_jsonrpc_error(payload.get("id") if isinstance(payload, dict) else None, INVALID_REQUEST, "Invalid JSON-RPC 2.0 request."), status.HTTP_400_BAD_REQUEST, current_sess_id
-
-    request_id = payload.get("id")
-    method = payload.get("method")
-    params = payload.get("params", {})
-
-    if method == "initialize":
-        res_data = {
-            "protocolVersion": MCP_PROTOCOL_VERSION_LEGACY,
-            "capabilities": {"tools": {"listChanged": False}},
-            "serverInfo": {"name": "mcp-web-engine-legacy", "version": "1.0.0"}
-        }
-        return make_jsonrpc_response(request_id, res_data), status.HTTP_200_OK, current_sess_id
-
-    elif method == "notifications/initialized":
-        return {}, status.HTTP_202_ACCEPTED, current_sess_id
-
-    elif method == "tools/list":
-        return make_jsonrpc_response(request_id, {"tools": MCP_TOOL_DEFINITIONS}), status.HTTP_200_OK, current_sess_id
-
-    elif method == "tools/call":
-        tool_name = params.get("name")
-        arguments = params.get("arguments", {})
-        try:
-            raw_result = await handle_mcp_tool_call(tool_name, arguments)
-            mcp_result = {"content": [{"type": "text", "text": json.dumps(raw_result)}], "isError": False}
-            return make_jsonrpc_response(request_id, mcp_result), status.HTTP_200_OK, current_sess_id
-        except Exception as e:
-            return make_jsonrpc_response(request_id, {"content": [{"type": "text", "text": str(e)}], "isError": True}), status.HTTP_200_OK, current_sess_id
-
-    else:
-        return make_jsonrpc_error(request_id, METHOD_NOT_FOUND, f"Method '{method}' not found."), status.HTTP_404_NOT_FOUND, current_sess_id

package/security.py

@@ -1,169 +0,0 @@
-"""
-Hardened Security Module: SSRF Protection, IP/DNS Validation, Multi-Key Auth (Master & Beta Keys) & Rate Limiting
-"""
-import ipaddress
-import socket
-import time
-import os
-import json
-import urllib.parse
-from fastapi import HTTPException, Security, status
-from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
-from config import settings
-
-security_bearer = HTTPBearer(auto_error=False)
-
-rate_limit_records = {}
-BETA_KEYS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "beta_keys.json")
-
-def is_beta_key_valid(token: str) -> bool:
-    """
-    Checks if a token is an active Beta Key in beta_keys.json.
-    """
-    if not os.path.exists(BETA_KEYS_FILE):
-        return False
-    try:
-        with open(BETA_KEYS_FILE, "r", encoding="utf-8") as f:
-            keys = json.load(f)
-            if token in keys and keys[token].get("status") == "active":
-                return True
-    except Exception:
-        pass
-    return False
-
-# Reserved Subnets for SSRF Protection
-PRIVATE_NETWORKS = [
-    ipaddress.ip_network("0.0.0.0/8"),
-    ipaddress.ip_network("10.0.0.0/8"),
-    ipaddress.ip_network("100.64.0.0/10"),
-    ipaddress.ip_network("127.0.0.0/8"),
-    ipaddress.ip_network("169.254.0.0/16"),
-    ipaddress.ip_network("172.16.0.0/12"),
-    ipaddress.ip_network("192.0.0.0/24"),
-    ipaddress.ip_network("192.0.2.0/24"),
-    ipaddress.ip_network("192.88.99.0/24"),
-    ipaddress.ip_network("192.168.0.0/16"),
-    ipaddress.ip_network("198.18.0.0/15"),
-    ipaddress.ip_network("198.51.100.0/24"),
-    ipaddress.ip_network("203.0.113.0/24"),
-    ipaddress.ip_network("224.0.0.0/4"),
-    ipaddress.ip_network("240.0.0.0/4"),
-    ipaddress.ip_network("255.255.255.255/32"),
-    # IPv6
-    ipaddress.ip_network("::/128"),
-    ipaddress.ip_network("::1/128"),
-    ipaddress.ip_network("fc00::/7"),
-    ipaddress.ip_network("fe80::/10")
-]
-
-FORBIDDEN_HOSTNAMES = {"localhost", "loopback", "broadcasthost", "local", "0.0.0.0", "127.0.0.1", "::1"}
-
-def is_ip_private(ip_str: str) -> bool:
-    try:
-        if ip_str.isdigit():
-            ip_obj = ipaddress.ip_address(int(ip_str))
-        else:
-            ip_obj = ipaddress.ip_address(ip_str)
-
-        for net in PRIVATE_NETWORKS:
-            if ip_obj in net:
-                return True
-        return False
-    except ValueError:
-        return False
-
-def validate_ssrf_url(url_str: str) -> str:
-    if not url_str or not isinstance(url_str, str):
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail={"error": "INVALID_URL", "message": "URL must be a non-empty string."}
-        )
-
-    unquoted_url = urllib.parse.unquote(url_str)
-    
-    try:
-        parsed = urllib.parse.urlparse(unquoted_url)
-    except Exception:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail={"error": "INVALID_URL", "message": f"URL '{url_str}' is unparseable."}
-        )
-
-    if parsed.scheme not in ["http", "https"]:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail={"error": "INVALID_SCHEME", "message": f"Scheme '{parsed.scheme}' not allowed. Only http/https supported."}
-        )
-
-    hostname = parsed.hostname
-    if not hostname:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail={"error": "MISSING_HOSTNAME", "message": "URL must contain a valid hostname."}
-        )
-
-    clean_host = hostname.lower().strip(".")
-
-    if clean_host in FORBIDDEN_HOSTNAMES or clean_host.endswith(".local"):
-        raise HTTPException(
-            status_code=status.HTTP_403_FORBIDDEN,
-            detail={"error": "SSRF_BLOCKED", "message": f"Access to hostname '{clean_host}' is blocked (SSRF protection)."}
-        )
-
-    if is_ip_private(clean_host):
-        raise HTTPException(
-            status_code=status.HTTP_403_FORBIDDEN,
-            detail={"error": "SSRF_BLOCKED", "message": f"Access to private IP '{clean_host}' is blocked."}
-        )
-
-    try:
-        ip_addresses = socket.getaddrinfo(clean_host, None)
-    except socket.gaierror:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail={"error": "DNS_RESOLUTION_FAILED", "message": f"Could not resolve hostname '{clean_host}'."}
-        )
-
-    for item in ip_addresses:
-        resolved_ip = item[4][0]
-        if is_ip_private(resolved_ip):
-            raise HTTPException(
-                status_code=status.HTTP_403_FORBIDDEN,
-                detail={"error": "SSRF_BLOCKED", "message": f"URL resolves to private/loopback IP address '{resolved_ip}' which is blocked."}
-            )
-
-    return url_str
-
-def verify_api_key(auth: HTTPAuthorizationCredentials = Security(security_bearer)):
-    if not auth or not auth.credentials:
-        raise HTTPException(
-            status_code=status.HTTP_401_UNAUTHORIZED,
-            detail={"error": "UNAUTHORIZED", "message": "Invalid or missing API key."}
-        )
-
-    token = auth.credentials
-    # Valid if matches Master API_KEY or any active Beta Key
-    if token == settings.API_KEY or is_beta_key_valid(token):
-        return token
-
-    raise HTTPException(
-        status_code=status.HTTP_401_UNAUTHORIZED,
-        detail={"error": "UNAUTHORIZED", "message": "Invalid or revoked API key."}
-    )
-
-def check_rate_limit(api_key: str):
-    now = time.time()
-    window_start = now - 60.0
-
-    if api_key not in rate_limit_records:
-        rate_limit_records[api_key] = []
-
-    rate_limit_records[api_key] = [t for t in rate_limit_records[api_key] if t > window_start]
-
-    if len(rate_limit_records[api_key]) >= settings.RATE_LIMIT_PER_MINUTE:
-        raise HTTPException(
-            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
-            detail={"error": "RATE_LIMIT_EXCEEDED", "message": f"Rate limit of {settings.RATE_LIMIT_PER_MINUTE} req/min exceeded."}
-        )
-
-    rate_limit_records[api_key].append(now)

package/web_engine.py

@@ -1,181 +0,0 @@
-"""
-Core Web Engine Module: SearXNG Connector with DuckDuckGo Direct Fallback, SSRF-Safe HTTP Fetcher & HTML/Markdown Parser
-"""
-import time
-import urllib.parse
-import re
-from curl_cffi.requests import AsyncSession
-from bs4 import BeautifulSoup
-import html2text
-from fastapi import HTTPException, status
-from config import settings
-from security import validate_ssrf_url
-
-USER_AGENTS = [
-    "Mozilla/5.0 (X11; Linux aarch64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
-]
-
-def get_headers():
-    return {
-        "User-Agent": USER_AGENTS[0],
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-        "Accept-Language": "es-ES,es;q=0.9,en-US;q=0.8,en;q=0.7",
-        "Sec-Ch-Ua": '"Not A(Brand";v="99", "Google Chrome";v="120"',
-        "Sec-Fetch-Dest": "document"
-    }
-
-async def execute_ddg_fallback(query: str, limit: int = 10):
-    """
-    Direct DuckDuckGo HTML Search Fallback if SearXNG is unavailable.
-    """
-    start_t = time.time()
-    ddg_url = f"https://html.duckduckgo.com/html/?q={urllib.parse.quote(query)}"
-    headers = get_headers()
-
-    async with AsyncSession() as session:
-        try:
-            r = await session.get(ddg_url, headers=headers, impersonate="chrome120", timeout=8)
-            if r.status_code == 200:
-                soup = BeautifulSoup(r.text, "html.parser")
-                results = []
-                for idx, a in enumerate(soup.find_all("a", class_="result__a")[:limit]):
-                    title = a.get_text(strip=True)
-                    raw_href = a.get("href", "")
-                    # Extract target URL from DDG redirect
-                    match = re.search(r"uddg=(https?%3A%2F%2F[^&]+)", raw_href)
-                    final_url = urllib.parse.unquote(match.group(1)) if match else raw_href
-                    results.append({
-                        "rank": idx + 1,
-                        "title": title,
-                        "url": final_url,
-                        "snippet": f"Search result for '{query}'",
-                        "engine": "duckduckgo_fallback"
-                    })
-                return {
-                    "query": query,
-                    "count": len(results),
-                    "latency_ms": round((time.time() - start_t) * 1000, 2),
-                    "results": results
-                }
-        except Exception:
-            pass
-
-    return {"query": query, "count": 0, "latency_ms": 0, "results": []}
-
-async def execute_web_search(query: str, limit: int = 10):
-    start_t = time.time()
-    url = f"{settings.SEARXNG_URL}?q={query}&format=json"
-    
-    async with AsyncSession() as session:
-        try:
-            r = await session.get(url, timeout=4.0)
-            if r.status_code == 200:
-                data = r.json()
-                results = data.get("results", [])[:limit]
-                items = []
-                for idx, item in enumerate(results):
-                    items.append({
-                        "rank": idx + 1,
-                        "title": item.get("title"),
-                        "url": item.get("url"),
-                        "snippet": item.get("content"),
-                        "engine": item.get("engine")
-                    })
-                return {
-                    "query": query,
-                    "count": len(items),
-                    "latency_ms": round((time.time() - start_t) * 1000, 2),
-                    "results": items
-                }
-        except Exception:
-            pass
-
-    # SearXNG failed or timed out -> Fallback to direct DuckDuckGo HTML parser
-    return await execute_ddg_fallback(query, limit)
-
-async def execute_fetch_url(url: str, max_bytes: int = None):
-    current_url = validate_ssrf_url(url)
-    byte_limit = max_bytes or settings.MAX_PAYLOAD_BYTES
-
-    start_t = time.time()
-    headers = get_headers()
-    max_redirects = 5
-    redirect_count = 0
-
-    async with AsyncSession() as session:
-        while redirect_count < max_redirects:
-            try:
-                r = await session.get(
-                    current_url,
-                    headers=headers,
-                    impersonate="chrome120",
-                    allow_redirects=False,
-                    timeout=settings.DEFAULT_TIMEOUT_SEC
-                )
-            except Exception as e:
-                raise HTTPException(
-                    status_code=status.HTTP_504_GATEWAY_TIMEOUT if "timeout" in str(e).lower() else status.HTTP_502_BAD_GATEWAY,
-                    detail={"error": "FETCH_ERROR", "message": f"Failed to fetch target URL: {str(e)}"}
-                )
-
-            if r.status_code in [301, 302, 303, 307, 308]:
-                location = r.headers.get("Location") or r.headers.get("location")
-                if not location:
-                    break
-                next_url = urllib.parse.urljoin(current_url, location)
-                current_url = validate_ssrf_url(next_url)
-                redirect_count += 1
-                continue
-            else:
-                break
-
-    if r.status_code != 200:
-        raise HTTPException(
-            status_code=status.HTTP_502_BAD_GATEWAY,
-            detail={"error": "HTTP_FETCH_FAILED", "message": f"Target server returned HTTP status {r.status_code}."}
-        )
-
-    content = r.text[:byte_limit]
-    if not content or len(content.strip()) < 10:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail={"error": "EMPTY_RESPONSE", "message": "Target URL returned an empty or truncated payload."}
-        )
-
-    return {
-        "url": current_url,
-        "status_code": r.status_code,
-        "content_length": len(content),
-        "latency_ms": round((time.time() - start_t) * 1000, 2),
-        "content": content
-    }
-
-async def execute_extract_markdown(url: str, max_bytes: int = None):
-    fetch_res = await execute_fetch_url(url, max_bytes)
-    html_content = fetch_res["content"]
-
-    try:
-        soup = BeautifulSoup(html_content, "html.parser")
-        for tag in soup(["script", "style", "nav", "footer", "iframe"]):
-            tag.decompose()
-
-        h = html2text.HTML2Text()
-        h.ignore_links = False
-        h.ignore_images = True
-        h.body_width = 0
-        
-        markdown_text = h.handle(str(soup))
-        
-        return {
-            "url": fetch_res["url"],
-            "status_code": 200,
-            "markdown_length": len(markdown_text),
-            "latency_ms": fetch_res["latency_ms"],
-            "markdown": markdown_text
-        }
-    except Exception as e:
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail={"error": "EXTRACTION_FAILED", "message": f"Failed to convert HTML to Markdown: {str(e)}"}
-        )