mcp-scraper 0.34.0 → 0.35.1

package/docs/mcp-tool-manifest.generated.json

@@ -1,12 +1,12 @@
 {
-  "generatedAt": "2026-07-25T03:25:17.950Z",
+  "generatedAt": "2026-07-27T21:59:31.780Z",
   "generatedFrom": "dist/bin/mcp-stdio-server.js",
   "serverInfo": {
     "name": "mcp-scraper",
-    "version": "0.33.7"
+    "version": "0.35.1"
   },
   "counts": {
-    "unified_stdio": 167
+    "unified_stdio": 168
   },
   "surfaces": {
     "unified_stdio": [
@@ -105,6 +105,7 @@
       "list-vaults",
       "list-webhooks",
       "map_site_urls",
+      "map_wayback_snapshots",
       "maps_place_intel",
       "maps_search",
       "memory-backlinks",
@@ -7296,20 +7297,57 @@
     {
       "name": "extract_site",
       "title": "Multi-Page Site Content Crawl",
-      "description": "Crawl a public website and return page CONTENT (Markdown) across multiple pages. Bulk crawls over 25 pages are saved as per-page Markdown files in a local folder instead of inlined. Content only — for a technical SEO audit use audit_site instead.",
+      "description": "Crawl a public website and return page CONTENT (Markdown) across multiple pages. A Wayback replay URL produces one archived site snapshot. The optional wayback plan produces whole-site, single-page, or selected-page timelines across explicit months or a month range, all in one export with a capture matrix. Bulk crawls over 25 pages are saved as per-page Markdown files in a local folder instead of inlined. Content only — for a technical SEO audit use audit_site instead.",
       "inputSchema": {
         "type": "object",
         "properties": {
           "url": {
             "type": "string",
             "format": "uri",
-            "description": "Public website URL or domain to crawl for page CONTENT (map + scrape). For a technical SEO audit use audit_site instead — this returns content only, not analysis."
+            "description": "Public website URL or web.archive.org replay URL. Without wayback, this crawls live content or one archived site snapshot. With wayback, it creates a multi-month archive timeline."
           },
           "maxPages": {
             "type": "integer",
             "minimum": 1,
             "maximum": 10000,
-            "description": "Maximum pages to extract. Bulk crawls (over 25 pages) switch to folder mode: each page saved as its own Markdown file, with a summary plus folder path returned instead of inlining content."
+            "description": "Maximum pages per Wayback month, or maximum total pages for a normal crawl. Multi-month jobs remain capped at 10,000 total captures and 500 pages per month."
+          },
+          "wayback": {
+            "type": "object",
+            "properties": {
+              "months": {
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "pattern": "^\\d{4}-(?:0[1-9]|1[0-2])$"
+                },
+                "minItems": 1,
+                "maxItems": 60
+              },
+              "from": {
+                "$ref": "#/properties/wayback/properties/months/items"
+              },
+              "to": {
+                "$ref": "#/properties/wayback/properties/months/items"
+              },
+              "intervalMonths": {
+                "type": "integer",
+                "minimum": 1,
+                "maximum": 12,
+                "default": 1
+              },
+              "urls": {
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "format": "uri"
+                },
+                "minItems": 1,
+                "maxItems": 100
+              }
+            },
+            "additionalProperties": false,
+            "description": "Optional temporal archive plan. Provide explicit YYYY-MM months or a from/to range plus intervalMonths. Omit urls for whole-site monthly snapshots, provide one URL for a single-page timeline, or several URLs for selected-page timelines. All results share one durable export."
           },
           "rotateProxies": {
             "type": "boolean",
@@ -7470,7 +7508,7 @@
           "url": {
             "type": "string",
             "format": "uri",
-            "description": "Public http/https URL to extract."
+            "description": "Public http/https URL or web.archive.org replay URL to extract."
           },
           "screenshot": {
             "type": "boolean",
@@ -7491,6 +7529,11 @@
             "default": false,
             "description": "Extract brand colors, fonts, logo, and favicon via a rendered browser session."
           },
+          "includeFeaturedImage": {
+            "type": "boolean",
+            "default": false,
+            "description": "Return the best featured image from Open Graph, Twitter, JSON-LD, or page content. For Wayback replay URLs, also returns the timestamp-matched archived image URL when available."
+          },
           "downloadMedia": {
             "type": "boolean",
             "default": false,
@@ -7595,6 +7638,70 @@
           "screenshotSaved": {
             "$ref": "#/properties/title"
           },
+          "archive": {
+            "anyOf": [
+              {
+                "type": "object",
+                "properties": {
+                  "timestamp": {
+                    "type": "string"
+                  },
+                  "originalUrl": {
+                    "type": "string"
+                  },
+                  "replayUrl": {
+                    "type": "string"
+                  },
+                  "rawReplayUrl": {
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "timestamp",
+                  "originalUrl",
+                  "replayUrl",
+                  "rawReplayUrl"
+                ],
+                "additionalProperties": false
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
+          "featuredImage": {
+            "anyOf": [
+              {
+                "type": "object",
+                "properties": {
+                  "url": {
+                    "type": "string"
+                  },
+                  "archivedUrl": {
+                    "$ref": "#/properties/title"
+                  },
+                  "source": {
+                    "type": "string",
+                    "enum": [
+                      "og:image",
+                      "twitter:image",
+                      "json-ld",
+                      "content-image"
+                    ]
+                  }
+                },
+                "required": [
+                  "url",
+                  "archivedUrl",
+                  "source"
+                ],
+                "additionalProperties": false
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
           "memory": {
             "type": "object",
             "properties": {
@@ -7638,7 +7745,9 @@
           "entityTypes",
           "napScore",
           "missingSchemaFields",
-          "screenshotSaved"
+          "screenshotSaved",
+          "archive",
+          "featuredImage"
         ],
         "additionalProperties": false,
         "$schema": "http://json-schema.org/draft-07/schema#"
@@ -12277,6 +12386,388 @@
         "taskSupport": "forbidden"
       }
     },
+    {
+      "name": "map_wayback_snapshots",
+      "title": "Wayback Snapshot Inventory",
+      "description": "Inventory Wayback Machine captures without scraping their page content. Counts captures, unique archived URLs, unique content digests, first/last captures, monthly/yearly coverage, missing months, and per-URL history across an inclusive date range. Use exact for one page, prefix for one path tree, host for one hostname, domain for subdomains, or urls for selected pages. Counts are exact unless maxCaptures is reached, in which case countType is lower_bound. Set includeCaptures true only when individual timestamps are needed; use extract_site.wayback afterward to download selected copy.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "url": {
+            "type": "string",
+            "format": "uri",
+            "description": "Original public page/site URL or a web.archive.org replay URL to inventory."
+          },
+          "scope": {
+            "type": "string",
+            "enum": [
+              "exact",
+              "prefix",
+              "host",
+              "domain"
+            ],
+            "default": "exact",
+            "description": "exact = one page; prefix = one path tree; host = one hostname; domain = the domain plus subdomains. Ignored when urls is provided."
+          },
+          "urls": {
+            "type": "array",
+            "items": {
+              "type": "string",
+              "format": "uri"
+            },
+            "minItems": 1,
+            "maxItems": 100,
+            "description": "Optional selected page URLs to inventory together using exact matching. Every URL must belong to the same site as url."
+          },
+          "from": {
+            "type": "string",
+            "pattern": "^(?:\\d{4}|\\d{4}-(?:0[1-9]|1[0-2])|\\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\\d|3[01])|\\d{14})$",
+            "description": "Inclusive beginning of the archive range: YYYY, YYYY-MM, YYYY-MM-DD, or a 14-digit Wayback timestamp."
+          },
+          "to": {
+            "$ref": "#/properties/from",
+            "description": "Inclusive end of the archive range: YYYY, YYYY-MM, YYYY-MM-DD, or a 14-digit Wayback timestamp."
+          },
+          "successfulHtmlOnly": {
+            "type": "boolean",
+            "default": true,
+            "description": "Count only HTTP 200 text/html captures. Set false to include redirects, errors, and archived assets."
+          },
+          "maxCaptures": {
+            "type": "integer",
+            "minimum": 1,
+            "maximum": 100000,
+            "default": 10000,
+            "description": "Maximum CDX capture rows to scan. If reached, countType is lower_bound instead of exact. Narrow the range or raise this cap for an exact large inventory."
+          },
+          "includeCaptures": {
+            "type": "boolean",
+            "default": false,
+            "description": "Return individual timestamp rows in addition to aggregate counts. Leave false for a compact count-only inventory."
+          },
+          "maxCaptureRows": {
+            "type": "integer",
+            "minimum": 0,
+            "maximum": 1000,
+            "default": 500,
+            "description": "Maximum individual capture rows returned when includeCaptures is true. Aggregated counts still use every scanned capture."
+          }
+        },
+        "required": [
+          "url"
+        ],
+        "additionalProperties": false,
+        "$schema": "http://json-schema.org/draft-07/schema#"
+      },
+      "outputSchema": {
+        "type": "object",
+        "properties": {
+          "url": {
+            "type": "string"
+          },
+          "scope": {
+            "type": "string",
+            "enum": [
+              "exact",
+              "prefix",
+              "host",
+              "domain"
+            ]
+          },
+          "selectedUrls": {
+            "anyOf": [
+              {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
+          "from": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "to": {
+            "$ref": "#/properties/from"
+          },
+          "successfulHtmlOnly": {
+            "type": "boolean"
+          },
+          "totalCaptures": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "countType": {
+            "type": "string",
+            "enum": [
+              "exact",
+              "lower_bound"
+            ]
+          },
+          "complete": {
+            "type": "boolean"
+          },
+          "truncated": {
+            "type": "boolean"
+          },
+          "maxCaptures": {
+            "type": "integer",
+            "minimum": 1
+          },
+          "queryPages": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "uniqueUrls": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "uniqueDigests": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "firstCapture": {
+            "anyOf": [
+              {
+                "type": "object",
+                "properties": {
+                  "timestamp": {
+                    "type": "string"
+                  },
+                  "originalUrl": {
+                    "type": "string"
+                  },
+                  "digest": {
+                    "$ref": "#/properties/from"
+                  },
+                  "rawReplayUrl": {
+                    "type": "string"
+                  },
+                  "replayUrl": {
+                    "$ref": "#/properties/from"
+                  },
+                  "statusCode": {
+                    "anyOf": [
+                      {
+                        "type": "integer"
+                      },
+                      {
+                        "type": "null"
+                      }
+                    ]
+                  },
+                  "mimeType": {
+                    "$ref": "#/properties/from"
+                  },
+                  "length": {
+                    "anyOf": [
+                      {
+                        "type": "integer",
+                        "minimum": 0
+                      },
+                      {
+                        "type": "null"
+                      }
+                    ]
+                  }
+                },
+                "required": [
+                  "timestamp",
+                  "originalUrl",
+                  "digest",
+                  "rawReplayUrl",
+                  "replayUrl"
+                ],
+                "additionalProperties": false
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
+          "lastCapture": {
+            "anyOf": [
+              {
+                "$ref": "#/properties/firstCapture/anyOf/0"
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
+          "monthlyCounts": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "month": {
+                  "type": "string"
+                },
+                "captures": {
+                  "type": "integer",
+                  "minimum": 0
+                }
+              },
+              "required": [
+                "month",
+                "captures"
+              ],
+              "additionalProperties": false
+            }
+          },
+          "yearlyCounts": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "year": {
+                  "type": "string"
+                },
+                "captures": {
+                  "type": "integer",
+                  "minimum": 0
+                }
+              },
+              "required": [
+                "year",
+                "captures"
+              ],
+              "additionalProperties": false
+            }
+          },
+          "missingMonths": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "perUrl": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "url": {
+                  "type": "string"
+                },
+                "captures": {
+                  "type": "integer",
+                  "minimum": 0
+                },
+                "uniqueDigests": {
+                  "type": "integer",
+                  "minimum": 0
+                },
+                "firstTimestamp": {
+                  "type": "string"
+                },
+                "lastTimestamp": {
+                  "type": "string"
+                }
+              },
+              "required": [
+                "url",
+                "captures",
+                "uniqueDigests",
+                "firstTimestamp",
+                "lastTimestamp"
+              ],
+              "additionalProperties": false
+            }
+          },
+          "perUrlTruncatedCount": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "captures": {
+            "type": "array",
+            "items": {
+              "$ref": "#/properties/firstCapture/anyOf/0"
+            }
+          },
+          "captureRowsTruncatedCount": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "durationMs": {
+            "type": "number",
+            "minimum": 0
+          },
+          "truncatedCount": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "artifact": {
+            "type": "object",
+            "properties": {
+              "artifactId": {
+                "type": "string"
+              },
+              "bytes": {
+                "type": "integer",
+                "minimum": 0
+              },
+              "expiresAt": {
+                "type": "string"
+              },
+              "preview": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "artifactId",
+              "bytes",
+              "expiresAt",
+              "preview"
+            ],
+            "additionalProperties": false
+          }
+        },
+        "required": [
+          "url",
+          "scope",
+          "selectedUrls",
+          "from",
+          "to",
+          "successfulHtmlOnly",
+          "totalCaptures",
+          "countType",
+          "complete",
+          "truncated",
+          "maxCaptures",
+          "queryPages",
+          "uniqueUrls",
+          "uniqueDigests",
+          "firstCapture",
+          "lastCapture",
+          "monthlyCounts",
+          "yearlyCounts",
+          "missingMonths",
+          "perUrl",
+          "perUrlTruncatedCount",
+          "captures",
+          "captureRowsTruncatedCount",
+          "durationMs"
+        ],
+        "additionalProperties": false,
+        "$schema": "http://json-schema.org/draft-07/schema#"
+      },
+      "annotations": {
+        "title": "Wayback Snapshot Inventory",
+        "readOnlyHint": true,
+        "destructiveHint": false,
+        "idempotentHint": false,
+        "openWorldHint": true
+      },
+      "execution": {
+        "taskSupport": "forbidden"
+      }
+    },
     {
       "name": "maps_place_intel",
       "title": "Google Maps Business Profile Details",
@@ -15606,7 +16097,7 @@
     {
       "name": "query_fanout_workflow",
       "title": "Capture AI Search Fan-Out",
-      "description": "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites. Returns raw structured data for you to classify and analyze. Set export=true for JSON/CSV/TSV/HTML artifacts. WRITE NOTE: passing prompt submits a real message in the user's logged-in account — only send when the user wants that; omit it to capture a prompt the user just ran. The session must already be open on chatgpt.com or claude.ai (see browser_profile_connect) while the prompt streams. NOT for Google AI Overview — use harvest_paa for that.",
+      "description": "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites. Complete structured data is always returned inline for analysis. export=true additionally writes JSON/CSV/TSV/HTML only from an installed local MCP server; hosted OAuth/HTTP clients receive exports=null and use the inline data. A local export failure does not discard a successful capture. WRITE NOTE: passing prompt submits a real message in the user's logged-in account — only send when the user wants that; omit it to capture a prompt the user just ran. The session must already be open on chatgpt.com or claude.ai (see browser_profile_connect) while the prompt streams. NOT for Google AI Overview — use harvest_paa for that.",
       "inputSchema": {
         "type": "object",
         "properties": {
@@ -15636,7 +16127,7 @@
           "export": {
             "type": "boolean",
             "default": false,
-            "description": "Write JSON/CSV/TSV/HTML exports to MCP_SCRAPER_OUTPUT_DIR/fanout, returning relative paths."
+            "description": "When using the installed local MCP server, write JSON/CSV/TSV/HTML exports to MCP_SCRAPER_OUTPUT_DIR/fanout. Hosted clients such as ChatGPT always receive the complete structured result inline and leave exports null."
           }
         },
         "required": [
@@ -15914,6 +16405,13 @@
               "null"
             ]
           },
+          "export_error": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "description": "Non-fatal local export failure. The inline capture remains complete when this is present."
+          },
           "exports": {
             "anyOf": [
               {

package/docs/specs/query-fanout-transport-contract-fix.md

@@ -0,0 +1,45 @@
+# Query fan-out transport contract fix
+
+- **Status:** Released and verified in production
+- **Baseline:** `origin/main` at `e2c90f8db7c22fa2aa48fc62cef10e2e0c799e15` (2026-07-25)
+- **Decision:** [Keep hosted fan-out results inline](../decisions/2026-07-25-hosted-fanout-results-stay-inline.md)
+- **Owner:** MCP Scraper browser-agent surface
+
+## Problem
+
+`/mcp` correctly registers browser-agent tools with `savesReportsLocally: false`, but `query_fanout_workflow` previously forwarded `input.export` to the browser service and treated remote export paths as usable. An OAuth call with `export: true` could therefore fail while writing an inaccessible sandbox path.
+
+## Contract
+
+| Transport | Capture data | `export: true` behavior | Filesystem contract |
+| --- | --- | --- | --- |
+| Local stdio or MCPB | Always inline structured content | Client writes optional JSON/CSV/TSV/HTML after capture | Relative paths under local `MCP_SCRAPER_OUTPUT_DIR/fanout` |
+| Hosted HTTP/OAuth `/mcp` | Always inline structured content | Accepted for compatibility but does not create files | `exports: null`; no remote or sandbox path is returned |
+
+The inline result is canonical: queries, researched/cited URLs, snippets, counts, aggregates, and capture metadata must remain immediately usable by an AI.
+
+## Implementation requirements
+
+1. Send `export: false` to `/agent/sessions/:id/capture-fanout` for every transport.
+2. Return `res.data.result ?? res.data` inline on success and never return `res.data.exports`.
+3. Export only when `opts.savesReportsLocally !== false`, the caller asked for it, and the inline result is an enriched fan-out capture.
+4. Keep `exports` nullable. A local write failure returns the successful inline capture, `exports: null`, and optional `export_error`.
+5. Keep `export` accepted for compatibility and make docs/manifest distinguish local export from hosted inline delivery.
+
+## Verification
+
+Focused tests must prove hosted calls force upstream `export: false` and return `exports: null`; local calls force the same upstream value, then write relative local files; and a local write failure is non-fatal. Preserve the 180,000 ms capture timeout allowance (at least 210,000 ms upstream).
+
+Live release acceptance requires an explicitly authorized new web-search prompt: hosted OAuth with `export: true` must return a populated inline result with `exports: null`; stdio with the same option must additionally produce readable local exports.
+
+## Release evidence
+
+- PR [#64](https://github.com/VilovietaSEO/mcp-scraper/pull/64) merged as `34a5eefa69b1fcaa2dec1dd645ac0dfc55cf4035`.
+- npm, annotated tag, and GitHub/MCPB release published as `0.34.1` / `v0.34.1`.
+- Vercel production deployment `dpl_CFCSkpAywT6PyqRe8LPDaKvicGrp` reached `READY` and serves `mcpscraper.dev`.
+- Hosted Streamable HTTP with OAuth exposed 167 tools and returned a successful fan-out with 21 researched sources, complete inline data, `exports: null`, and no export error.
+- The exact published npm `0.34.1` stdio package exposed 167 tools, returned a successful fan-out with 30 researched sources, and produced nine readable local export files with no export error.
+
+## Out of scope
+
+Public/shared download URLs, capture-parser changes, profile authentication, billing, timeout changes, and recovery of fan-out from pre-capture prompts.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.34.0",
+  "version": "0.35.1",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",