mcp-web-engine 1.0.0 → 1.0.2

package/bin/cli.js

@@ -1,19 +1,55 @@
 #!/usr/bin/env node
-const { spawn } = require('child_process');
+
+/**
+ * MCP Web Engine CLI Entry Point (npm package executable)
+ * Spawns the MCP Web Engine Python server (stdio/SSE)
+ * Respects SEARXNG_URL, PORT, ALLOWED_NETWORKS, and environment settings.
+ */
+
+const { spawn, execSync } = require('child_process');
 const path = require('path');
 
 const mainPy = path.join(__dirname, '..', 'main.py');
+const reqTxt = path.join(__dirname, '..', 'requirements.txt');
+
+function findPython() {
+  const candidates = ['python3', 'python'];
+  for (const cmd of candidates) {
+    try {
+      execSync(`${cmd} --version`, { stdio: 'ignore' });
+      return cmd;
+    } catch (e) {}
+  }
+  return 'python3';
+}
+
+const pythonCmd = findPython();
+
+// Auto-check and install python dependencies if missing (pip logs sent to stderr to preserve stdio JSON-RPC)
+try {
+  execSync(`${pythonCmd} -c "import fastapi, uvicorn, requests, bs4"`, { stdio: 'ignore' });
+} catch (e) {
+  try {
+    execSync(`${pythonCmd} -m pip install -r "${reqTxt}" --break-system-packages`, { stdio: [0, 2, 2] });
+  } catch (err) {
+    // Fallback gracefully
+  }
+}
 
-const child = spawn('python3', [mainPy, ...process.argv.slice(2)], {
+// Spawn MCP Web Engine
+const child = spawn(pythonCmd, [mainPy, ...process.argv.slice(2)], {
   stdio: 'inherit',
   env: process.env
 });
 
 child.on('error', (err) => {
-  console.error('Failed to start mcp-web-engine python process:', err);
+  console.error('[mcp-web-engine] Failed to start Python process:', err.message);
   process.exit(1);
 });
 
-child.on('exit', (code) => {
+child.on('exit', (code, signal) => {
+  if (signal) {
+    process.exit(128 + (signal === 'SIGINT' ? 2 : 15));
+  }
   process.exit(code ?? 0);
 });

package/main.py

@@ -1,9 +1,11 @@
 """
-Main FastAPI Application Gateway & MCP Protocol Handler
+Main FastAPI Application Gateway & MCP Protocol Handler (SSE & STDIO Modes)
 """
 import time
 import json
 import os
+import sys
+import asyncio
 from fastapi import FastAPI, Depends, HTTPException, status, Response, Request, Header
 from typing import Optional
 from sse_starlette.sse import EventSourceResponse
@@ -49,7 +51,6 @@ async def get_beta_telemetry(api_key: str = Depends(verify_api_key)):
         with open(BETA_KEYS_FILE, "r", encoding="utf-8") as f:
             keys = json.load(f)
         
-        # Redact raw key strings, expose only public beta user telemetry
         summary = {}
         for k, v in keys.items():
             beta_id = v.get("id", "Unknown")
@@ -193,6 +194,41 @@ async def extract_endpoint(body: ExtractMarkdownInput, api_key: str = Depends(ve
     check_rate_limit(api_key)
     return await execute_extract_markdown(body.url, body.max_bytes)
 
+async def run_mcp_stdio_server():
+    """
+    Stdio MCP protocol handler for CLI executions (e.g. npx -y mcp-web-engine)
+    """
+    loop = asyncio.get_event_loop()
+    reader = asyncio.StreamReader()
+    protocol = asyncio.StreamReaderProtocol(reader)
+    await loop.connect_read_pipe(lambda: protocol, sys.stdin)
+
+    while True:
+        line = await reader.readline()
+        if not line:
+            break
+        line_str = line.decode("utf-8").strip()
+        if not line_str:
+            continue
+        try:
+            payload = json.loads(line_str)
+            res_body, _ = await process_mcp_2026_stateless(payload, api_key="local-stdio")
+            if res_body:
+                sys.stdout.write(json.dumps(res_body, ensure_ascii=False) + "\n")
+                sys.stdout.flush()
+        except Exception as e:
+            err_res = {
+                "jsonrpc": "2.0",
+                "id": None,
+                "error": {"code": -32603, "message": str(e)}
+            }
+            sys.stdout.write(json.dumps(err_res) + "\n")
+            sys.stdout.flush()
+
 if __name__ == "__main__":
-    import uvicorn
-    uvicorn.run("main:app", host=settings.HOST, port=settings.PORT, reload=False)
+    is_stdio = "--stdio" in sys.argv or not sys.stdin.isatty()
+    if is_stdio:
+        asyncio.run(run_mcp_stdio_server())
+    else:
+        import uvicorn
+        uvicorn.run("main:app", host=settings.HOST, port=settings.PORT, reload=False)

package/package.json

@@ -1,11 +1,14 @@
 {
   "name": "mcp-web-engine",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Privacy-First, Self-Hostable & SSRF-Hardened MCP Server & Web Engine for AI Agents",
   "main": "bin/cli.js",
   "bin": {
     "mcp-web-engine": "bin/cli.js"
   },
+  "engines": {
+    "node": ">=18.0.0"
+  },
   "files": [
     "bin",
     "*.py",
@@ -23,7 +26,9 @@
     "searxng",
     "web-search",
     "ai-agent",
-    "turbollm"
+    "turbollm",
+    "web-scraper",
+    "ssrf-protection"
   ],
   "author": "Arbolencio",
   "license": "MIT"

package/requirements.txt

@@ -5,5 +5,5 @@ pydantic-settings>=2.1.0
 curl-cffi>=0.6.0
 beautifulsoup4>=4.12.0
 html2text>=2024.1.14
+sse-starlette>=2.0.0
 pytest>=8.0.0
-httpx>=0.26.0