litescrape-mcp-server 0.2.0 → 0.3.0

package/README.md

@@ -67,7 +67,7 @@ One call runs at a time per network, and failed calls never count. When a limit
 
 ## Add an API key
 
-A key unlocks `google_ai_mode`, `google_ai_overview`, `google_shopping` and `google_reviews`, removes the daily limits and allows concurrent calls. Get one at [litescrape.com](https://litescrape.com), then set `LITESCRAPE_API_KEY` in the server's environment:
+A key unlocks `google_ai_mode`, `google_ai_overview`, `google_shopping`, `google_reviews` and `web_fetch`, removes the daily limits and allows concurrent calls. Get one at [litescrape.com](https://litescrape.com), then set `LITESCRAPE_API_KEY` in the server's environment:
 
 ```json
 {
@@ -100,6 +100,7 @@ With the hosted endpoint: `claude mcp add --transport http litescrape https://mc
 | `google_ai_mode`     | Yes        | Google AI Mode's generated answer with its cited sources; `continuable` returns a follow-up token, `image_url` adds a picture to the question                                                                                                                          |
 | `google_shopping`    | Yes        | The Google Shopping product grid, category blocks, sponsored listings and refinement chips; price, sale, shipping and small-business refinements, `sort_by`, pagination                                                                                                |
 | `google_reviews`     | Yes        | The Google Maps reviews of one place by `place_id` or `data_id` (from a `google_maps` result) with `place_info`; `sort_by`, `topic_id` or `query` filters, `num`, and `next_page_token` for the next page                                                              |
+| `web_fetch`          | Yes        | One public web page rendered in a fresh browser, as Markdown (default), HTML, text or a base64 PNG screenshot, with the site's `status_code`; `target_selector`, `remove_selector`, `wait_until`, `wait_for_selector`, `page_timeout`, link and image styles. Alpha    |
 
 Parameter names and accepted values follow the [Litescrape API reference](https://litescrape.com/docs).
 
@@ -141,7 +142,7 @@ Requests time out after 120 seconds by default (the API's own deadline is 90 sec
 
 `litescrape-mcp-server --http` (or `LITESCRAPE_MCP_TRANSPORT=http`) serves MCP over HTTP instead of stdio: stateless `POST /mcp`, `GET /healthz`, listening on `PORT` (default 8080) and `HOST` (default `0.0.0.0`), or `--port` and `--host`. Each request's API key comes from its `Authorization: Bearer` header or `?api_key=` query parameter, so one process serves keyed and keyless callers; `LITESCRAPE_API_KEY` is ignored in this mode. This is what runs at `https://mcp.litescrape.com/mcp`.
 
-Behind a shared address, keyless callers would all count against one allowance. `LITESCRAPE_KEYLESS_PROXY_SECRET` is the secret the Litescrape API shares with its own hosted endpoint for forwarding each caller's address; it is not needed for a private deployment, where the server's own address is metered.
+Behind a shared address, keyless callers would all count against one allowance. `LITESCRAPE_KEYLESS_PROXY_SECRET` is the secret the Litescrape API shares with its own hosted endpoint for forwarding each caller's address, and `LITESCRAPE_EDGE_SECRET` is the value Cloudflare adds in front of that endpoint so the caller can be read from `CF-Connecting-IP`. Neither is needed for a private deployment, where the server's own address is metered.
 
 ```
 docker build -t litescrape-mcp-server . && docker run -p 8080:8080 litescrape-mcp-server --http

package/dist/http.d.ts

@@ -4,6 +4,8 @@ export declare const MCP_PATH = "/mcp";
 export declare const HEALTH_PATH = "/healthz";
 export declare const FORWARDED_IP_HEADER = "X-Litescrape-Keyless-Ip";
 export declare const PROXY_SECRET_HEADER = "X-Litescrape-Keyless-Secret";
+export declare const EDGE_SECRET_HEADER = "x-litescrape-edge-secret";
+export declare const EDGE_CLIENT_HEADER = "cf-connecting-ip";
 export interface HttpOptions {
     /**
      * Secret shared with the Litescrape API. When set, every upstream request
@@ -11,6 +13,12 @@ export interface HttpOptions {
      * caller rather than to this server's own address.
      */
     proxySecret?: string;
+    /**
+     * Secret Cloudflare adds to every request it relays. When set, the caller is
+     * read from CF-Connecting-IP on requests that carry it, and nothing is
+     * forwarded for requests that bypassed Cloudflare.
+     */
+    edgeSecret?: string;
     apiUrl?: string;
     timeoutMs?: number;
     fetch?: ClientOptions['fetch'];
@@ -19,11 +27,14 @@ export interface HttpOptions {
 }
 export declare function bearerToken(req: IncomingMessage): string | undefined;
 /**
- * The caller as the ingress reports it. X-Forwarded-For is passed through
- * unchanged: the API meters it only when it holds exactly one address, so a
- * crafted multi-hop value is refused there instead of being trusted here.
+ * The caller as the ingress reports it. With an edge secret configured, only a
+ * request carrying it is known to have come through Cloudflare, and its caller
+ * is CF-Connecting-IP; a request that bypassed Cloudflare gets no caller at all.
+ * Without one, X-Forwarded-For is passed through unchanged: the API meters it
+ * only when it holds exactly one address, so a crafted multi-hop value is
+ * refused there instead of being trusted here.
  */
-export declare function callerAddress(req: IncomingMessage): string | undefined;
+export declare function callerAddress(req: IncomingMessage, options?: Pick<HttpOptions, 'edgeSecret'>): string | undefined;
 export declare function clientForRequest(req: IncomingMessage, options: HttpOptions): LitescrapeClient;
 export declare function handleRequest(req: IncomingMessage, res: ServerResponse, options: HttpOptions): Promise<void>;
 export declare function createHttpServer(options?: HttpOptions): Server;

package/dist/http.js

@@ -1,3 +1,4 @@
+import { createHash, timingSafeEqual } from 'node:crypto';
 import { createServer as createNodeServer, } from 'node:http';
 import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
 import { LitescrapeClient } from './client.js';
@@ -9,6 +10,12 @@ export const HEALTH_PATH = '/healthz';
 // shared secret, and only when it holds exactly one address.
 export const FORWARDED_IP_HEADER = 'X-Litescrape-Keyless-Ip';
 export const PROXY_SECRET_HEADER = 'X-Litescrape-Keyless-Secret';
+// Behind Cloudflare the socket peer is one of its egress addresses and
+// X-Forwarded-For is a chain, so the caller is CF-Connecting-IP; a transform
+// rule on the zone adds the edge secret, which is how a request proves it came
+// through Cloudflare rather than straight to the origin with forged headers.
+export const EDGE_SECRET_HEADER = 'x-litescrape-edge-secret';
+export const EDGE_CLIENT_HEADER = 'cf-connecting-ip';
 function requestUrl(req) {
     return new URL(req.url ?? '/', 'http://localhost');
 }
@@ -20,21 +27,37 @@ export function bearerToken(req) {
     const fromQuery = requestUrl(req).searchParams.get('api_key')?.trim();
     return fromQuery || undefined;
 }
+function headerValue(req, name) {
+    const raw = req.headers[name];
+    return (Array.isArray(raw) ? raw.join(', ') : raw)?.trim() ?? '';
+}
+function sameSecret(presented, expected) {
+    const digest = (value) => createHash('sha256').update(value).digest();
+    return timingSafeEqual(digest(presented), digest(expected));
+}
 /**
- * The caller as the ingress reports it. X-Forwarded-For is passed through
- * unchanged: the API meters it only when it holds exactly one address, so a
- * crafted multi-hop value is refused there instead of being trusted here.
+ * The caller as the ingress reports it. With an edge secret configured, only a
+ * request carrying it is known to have come through Cloudflare, and its caller
+ * is CF-Connecting-IP; a request that bypassed Cloudflare gets no caller at all.
+ * Without one, X-Forwarded-For is passed through unchanged: the API meters it
+ * only when it holds exactly one address, so a crafted multi-hop value is
+ * refused there instead of being trusted here.
  */
-export function callerAddress(req) {
-    const forwarded = req.headers['x-forwarded-for'];
-    const value = (Array.isArray(forwarded) ? forwarded.join(', ') : forwarded)?.trim();
-    if (value)
-        return value;
+export function callerAddress(req, options = {}) {
+    if (options.edgeSecret) {
+        const presented = headerValue(req, EDGE_SECRET_HEADER);
+        if (!presented || !sameSecret(presented, options.edgeSecret))
+            return undefined;
+        return headerValue(req, EDGE_CLIENT_HEADER) || undefined;
+    }
+    const forwarded = headerValue(req, 'x-forwarded-for');
+    if (forwarded)
+        return forwarded;
     return req.socket?.remoteAddress || undefined;
 }
 export function clientForRequest(req, options) {
     const headers = {};
-    const caller = options.proxySecret ? callerAddress(req) : undefined;
+    const caller = options.proxySecret ? callerAddress(req, options) : undefined;
     if (options.proxySecret && caller) {
         headers[FORWARDED_IP_HEADER] = caller;
         headers[PROXY_SECRET_HEADER] = options.proxySecret;

package/dist/index.js

@@ -19,12 +19,20 @@ function http(args) {
     const port = Number(flag(args, '--port') ?? process.env.PORT ?? 8080);
     const host = flag(args, '--host') ?? process.env.HOST ?? '0.0.0.0';
     const proxySecret = process.env.LITESCRAPE_KEYLESS_PROXY_SECRET?.trim() || undefined;
+    const edgeSecret = process.env.LITESCRAPE_EDGE_SECRET?.trim() || undefined;
     const apiUrl = process.env.LITESCRAPE_API_URL?.trim() || undefined;
     const timeoutMs = Number(process.env.LITESCRAPE_TIMEOUT_MS) || undefined;
-    const server = createHttpServer({ proxySecret, apiUrl, timeoutMs, log: console.error });
+    const server = createHttpServer({
+        proxySecret,
+        edgeSecret,
+        apiUrl,
+        timeoutMs,
+        log: console.error,
+    });
     server.listen(port, host, () => {
+        const forwarding = proxySecret ? (edgeSecret ? 'on, Cloudflare edge' : 'on') : 'off';
         console.error(`${PACKAGE_NAME} ${VERSION} listening on http://${host}:${port}${MCP_PATH} ` +
-            `(API key per request, caller forwarding ${proxySecret ? 'on' : 'off'})`);
+            `(API key per request, caller forwarding ${forwarding})`);
     });
     const stop = () => server.close(() => process.exit(0));
     process.on('SIGTERM', stop);

package/dist/messages.js

@@ -19,14 +19,15 @@ export function instructions(keyed) {
             'each successful call is billed to that key. search is google_search in fast mode ' +
             '(organic results only). google_ai_mode generates an answer per request and is the ' +
             'slowest tool. google_reviews takes the place_id or data_id of a google_maps result ' +
-            'and pages with next_page_token. ' +
+            'and pages with next_page_token. web_fetch returns one web page as Markdown, HTML, ' +
+            'text or a screenshot. ' +
             shared);
     }
     return ('This session has no Litescrape API key. google_search (25 calls), bing_search (50), ' +
         'google_maps (50) and duckduckgo_search (50) are free per network per UTC day, one call ' +
         'at a time; failed calls do not count. search is google_search in fast mode (organic ' +
         'results only) and shares its allowance. google_ai_mode, google_ai_overview, ' +
-        'google_shopping and google_reviews need an API key, which also lifts every limit: get one at ' +
+        'google_shopping, google_reviews and web_fetch need an API key, which also lifts every limit: get one at ' +
         `${KEY_URL} and set LITESCRAPE_API_KEY in this server's environment. ` +
         shared);
 }

package/dist/surfaces.js

@@ -86,9 +86,9 @@ const googleSearchBase = {
         .number()
         .int()
         .min(1)
-        .max(100)
+        .max(10)
         .optional()
-        .describe('Requested result count, 1-100; Google may return fewer'),
+        .describe('Requested result count, 1-10; Google may return fewer'),
     device,
 };
 function count(payload, key) {
@@ -127,9 +127,9 @@ export const SURFACES = [
                 .number()
                 .int()
                 .min(1)
-                .max(100)
+                .max(10)
                 .optional()
-                .describe('Requested result count, 1-100; Google may return fewer'),
+                .describe('Requested result count, 1-10; Google may return fewer'),
             start: z.number().int().min(0).optional().describe('Result offset for pagination'),
             result_groups: resultGroups,
         },
@@ -438,6 +438,80 @@ export const SURFACES = [
             return `Google Reviews${name}: ${reviews} reviews${more}.`;
         },
     },
+    {
+        name: 'web_fetch',
+        title: 'Fetch a web page',
+        description: 'Render one public web page in a fresh browser and return it as Markdown (default), ' +
+            'HTML, plain text or a full-page PNG screenshot encoded as base64, with url, title, ' +
+            'content and the status_code the site returned. Narrow the content with ' +
+            'target_selector or remove_selector. Alpha. Requires an API key.',
+        path: '/api/web/fetch',
+        keyless: false,
+        inputSchema: {
+            url: z
+                .string()
+                .min(1)
+                .max(8192)
+                .describe('Public http:// or https:// URL on the standard port, without credentials'),
+            respond_with: z
+                .enum(['markdown', 'html', 'text', 'screenshot'])
+                .optional()
+                .describe('Output format; default markdown. screenshot returns a base64 PNG'),
+            target_selector: z
+                .string()
+                .min(1)
+                .max(2048)
+                .optional()
+                .describe('CSS selector to extract; falls back to the full page when nothing matches'),
+            remove_selector: z
+                .string()
+                .min(1)
+                .max(2048)
+                .optional()
+                .describe('CSS selector removed before extraction, e.g. "nav, footer"'),
+            wait_for_selector: z
+                .string()
+                .min(1)
+                .max(2048)
+                .optional()
+                .describe('CSS selector to wait for after navigation'),
+            wait_until: z
+                .enum(['commit', 'domcontentloaded', 'load', 'networkidle'])
+                .optional()
+                .describe('Navigation event to wait for; default domcontentloaded'),
+            page_timeout: z
+                .number()
+                .int()
+                .min(1)
+                .max(180)
+                .optional()
+                .describe('Seconds for browser operations; default 30'),
+            locale: z.string().min(2).max(64).optional().describe('Browser locale; default en-US'),
+            user_agent: z.string().min(1).max(1024).optional().describe('User-Agent to send'),
+            with_links: z
+                .enum(['inlined', 'referenced', 'collapsed', 'shortcut', 'discarded'])
+                .optional()
+                .describe('How Markdown writes links; default inlined'),
+            with_images: z
+                .enum(['all', 'alt', 'none'])
+                .optional()
+                .describe('How Markdown writes images; default all'),
+            with_iframe: z
+                .enum(['true', 'false', 'quoted'])
+                .optional()
+                .describe('Include direct child frames before extraction; default false'),
+            with_shadow_dom: z
+                .boolean()
+                .optional()
+                .describe('Expand open shadow roots before extraction; default false'),
+        },
+        summarize: (payload) => {
+            const title = typeof payload.title === 'string' && payload.title ? ` "${payload.title}"` : '';
+            const status = typeof payload.status_code === 'number' ? `, site status ${payload.status_code}` : '';
+            const size = typeof payload.content === 'string' ? `, ${payload.content.length} characters` : '';
+            return `Fetched${title}${status}${size}.`;
+        },
+    },
 ];
 export function findSurface(name) {
     return SURFACES.find((surface) => surface.name === name);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "litescrape-mcp-server",
-	"version": "0.2.0",
+	"version": "0.3.0",
 	"description": "MCP server for the Litescrape API: Google Search, Bing, DuckDuckGo and Google Maps free without an API key, plus Google AI Mode, AI Overview, Shopping and Reviews with one. Stdio or Streamable HTTP.",
 	"license": "MIT",
 	"homepage": "https://litescrape.com",

package/server.json

@@ -8,19 +8,19 @@
 		"source": "github"
 	},
 	"websiteUrl": "https://litescrape.com",
-	"version": "0.2.0",
+	"version": "0.3.0",
 	"packages": [
 		{
 			"registryType": "npm",
 			"identifier": "litescrape-mcp-server",
-			"version": "0.2.0",
+			"version": "0.3.0",
 			"transport": {
 				"type": "stdio"
 			},
 			"environmentVariables": [
 				{
 					"name": "LITESCRAPE_API_KEY",
-					"description": "Optional Litescrape API key. Without it, search, google_search, bing_search, google_maps and duckduckgo_search work from a free daily allowance; with it google_ai_mode, google_ai_overview, google_shopping and google_reviews are available too and the limits are lifted.",
+					"description": "Optional Litescrape API key. Without it, search, google_search, bing_search, google_maps and duckduckgo_search work from a free daily allowance; with it google_ai_mode, google_ai_overview, google_shopping, google_reviews and web_fetch are available too and the limits are lifted.",
 					"isRequired": false,
 					"isSecret": true,
 					"format": "string"