mcp-scraper 0.33.7 → 0.34.1

package/dist/{site-extract-repository-JHMVHENZ.js → site-extract-repository-OLVWMOU2.js}

@@ -10,8 +10,8 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-SUOHUXQS.js";
-import "./chunk-JBSGBSGT.js";
+} from "./chunk-5PZ6N2QM.js";
+import "./chunk-BWXLTWF7.js";
 import "./chunk-M2S27J6Z.js";
 import "./chunk-62DQAWPF.js";
 export {
@@ -27,4 +27,4 @@ export {
   setExtractJobTotal,
   settleExtractJob
 };
-//# sourceMappingURL=site-extract-repository-JHMVHENZ.js.map
+//# sourceMappingURL=site-extract-repository-OLVWMOU2.js.map

package/dist/{worker-3LYNVGLF.js → worker-645BZPEK.js}

@@ -6,17 +6,17 @@ import {
 } from "./chunk-3HBPKR5G.js";
 import {
   harvest
-} from "./chunk-XVCETVYY.js";
+} from "./chunk-XVVNKASZ.js";
 import {
   browserServiceApiKey,
   runWithCostContext
-} from "./chunk-V36LS5YV.js";
+} from "./chunk-4ZB3X6BQ.js";
 import "./chunk-K443GQY5.js";
-import "./chunk-V73MPRU6.js";
+import "./chunk-CB5C3BPB.js";
 import {
   MC_COSTS,
   serpActualCostMc
-} from "./chunk-JBSGBSGT.js";
+} from "./chunk-BWXLTWF7.js";
 import "./chunk-M2S27J6Z.js";
 import {
   claimPendingJob,
@@ -139,4 +139,4 @@ export {
   startWorker,
   tickOnce
 };
-//# sourceMappingURL=worker-3LYNVGLF.js.map
+//# sourceMappingURL=worker-645BZPEK.js.map

package/docs/mcp-tool-craft-lint.generated.md

@@ -1,56 +1,178 @@
 # MCP Tool Craft Lint
 
-Generated: 2026-06-23T22:55:40.518Z
+Generated: 2026-07-26T13:48:25.492Z
+Manifest: `/Users/vilovieta/.codex/worktrees/mcp-scraper-query-fanout-transport-fix/docs/mcp-tool-manifest.generated.json`
 
-Tools: 46
-Checks per tool: 10
+Tools: 167
+Checks per tool: 7
 Failing tools: 0
 
 | Tool | Score | Missing |
 |---|---:|---|
-| `harvest_paa` | 10/10 | none |
-| `search_serp` | 10/10 | none |
-| `extract_url` | 10/10 | none |
-| `map_site_urls` | 10/10 | none |
-| `extract_site` | 10/10 | none |
-| `youtube_harvest` | 10/10 | none |
-| `youtube_transcribe` | 10/10 | none |
-| `facebook_page_intel` | 10/10 | none |
-| `facebook_ad_search` | 10/10 | none |
-| `facebook_ad_transcribe` | 10/10 | none |
-| `facebook_video_transcribe` | 10/10 | none |
-| `instagram_profile_content` | 10/10 | none |
-| `instagram_media_download` | 10/10 | none |
-| `maps_place_intel` | 10/10 | none |
-| `maps_search` | 10/10 | none |
-| `directory_workflow` | 10/10 | none |
-| `workflow_list` | 10/10 | none |
-| `workflow_suggest` | 10/10 | none |
-| `workflow_run` | 10/10 | none |
-| `workflow_step` | 10/10 | none |
-| `workflow_status` | 10/10 | none |
-| `workflow_artifact_read` | 10/10 | none |
-| `rank_tracker_blueprint` | 10/10 | none |
-| `credits_info` | 10/10 | none |
-| `browser_open` | 10/10 | none |
-| `browser_profile_onboard` | 10/10 | none |
-| `browser_profile_status` | 10/10 | none |
-| `browser_screenshot` | 10/10 | none |
-| `browser_read` | 10/10 | none |
-| `browser_locate` | 10/10 | none |
-| `browser_goto` | 10/10 | none |
-| `browser_click` | 10/10 | none |
-| `browser_type` | 10/10 | none |
-| `browser_scroll` | 10/10 | none |
-| `browser_press` | 10/10 | none |
-| `browser_replay_start` | 10/10 | none |
-| `browser_replay_stop` | 10/10 | none |
-| `browser_list_replays` | 10/10 | none |
-| `browser_replay_download` | 10/10 | none |
-| `browser_replay_mark` | 10/10 | none |
-| `browser_replay_annotate` | 10/10 | none |
-| `browser_close` | 10/10 | none |
-| `browser_list_sessions` | 10/10 | none |
-| `browser_capture_fanout` | 10/10 | none |
-| `capture_serp_snapshot` | 10/10 | none |
-| `capture_serp_page_snapshots` | 10/10 | none |
+| `access-accept-share` | 7/7 | none |
+| `access-approve-sender` | 7/7 | none |
+| `access-decline-share` | 7/7 | none |
+| `access-inbox-settings` | 7/7 | none |
+| `access-invite-account` | 7/7 | none |
+| `access-issue-key` | 7/7 | none |
+| `access-list-approved-senders` | 7/7 | none |
+| `access-list-keys` | 7/7 | none |
+| `access-note-inbox` | 7/7 | none |
+| `access-remove-approved-sender` | 7/7 | none |
+| `access-revoke-key` | 7/7 | none |
+| `access-revoke-share` | 7/7 | none |
+| `access-set-scope` | 7/7 | none |
+| `access-share-note` | 7/7 | none |
+| `access-share-vault` | 7/7 | none |
+| `access-swap-vault` | 7/7 | none |
+| `access-switch-account` | 7/7 | none |
+| `access-unlink-share` | 7/7 | none |
+| `add-vault` | 7/7 | none |
+| `audit_site` | 7/7 | none |
+| `browser_click` | 7/7 | none |
+| `browser_close` | 7/7 | none |
+| `browser_extension_delete` | 7/7 | none |
+| `browser_extension_import` | 7/7 | none |
+| `browser_extension_list` | 7/7 | none |
+| `browser_goto` | 7/7 | none |
+| `browser_list_replays` | 7/7 | none |
+| `browser_list_sessions` | 7/7 | none |
+| `browser_locate` | 7/7 | none |
+| `browser_open` | 7/7 | none |
+| `browser_press` | 7/7 | none |
+| `browser_profile_connect` | 7/7 | none |
+| `browser_profile_list` | 7/7 | none |
+| `browser_read` | 7/7 | none |
+| `browser_replay_annotate` | 7/7 | none |
+| `browser_replay_download` | 7/7 | none |
+| `browser_replay_mark` | 7/7 | none |
+| `browser_replay_start` | 7/7 | none |
+| `browser_replay_stop` | 7/7 | none |
+| `browser_screenshot` | 7/7 | none |
+| `browser_scroll` | 7/7 | none |
+| `browser_type` | 7/7 | none |
+| `bulk-delete-notes` | 7/7 | none |
+| `call_service_connection_action` | 7/7 | none |
+| `capture_serp_page_snapshots` | 7/7 | none |
+| `capture_serp_snapshot` | 7/7 | none |
+| `check_site_export` | 7/7 | none |
+| `cost-usage` | 7/7 | none |
+| `create-channel` | 7/7 | none |
+| `create-scheduled-action` | 7/7 | none |
+| `create-secure-vault` | 7/7 | none |
+| `create-webhook` | 7/7 | none |
+| `credits_info` | 7/7 | none |
+| `delete-note` | 7/7 | none |
+| `delete-scheduled-action` | 7/7 | none |
+| `delete-vault` | 7/7 | none |
+| `describe_service_connection_tool` | 7/7 | none |
+| `diff_page` | 7/7 | none |
+| `directory_workflow` | 7/7 | none |
+| `export_connected_service_data` | 7/7 | none |
+| `export_search_console_table_data` | 7/7 | none |
+| `extract_site` | 7/7 | none |
+| `extract_url` | 7/7 | none |
+| `facebook_ad_search` | 7/7 | none |
+| `facebook_ad_transcribe` | 7/7 | none |
+| `facebook_page_intel` | 7/7 | none |
+| `facebook_video_transcribe` | 7/7 | none |
+| `fact-history` | 7/7 | none |
+| `g2_reviews` | 7/7 | none |
+| `get-chat-link` | 7/7 | none |
+| `get-message-note` | 7/7 | none |
+| `get-schedule-link` | 7/7 | none |
+| `get-schedule-status` | 7/7 | none |
+| `get-vault-app-link` | 7/7 | none |
+| `get-vault-contract` | 7/7 | none |
+| `gmail_search_contacts` | 7/7 | none |
+| `gmail_send_message` | 7/7 | none |
+| `google_ads_page_intel` | 7/7 | none |
+| `google_ads_search` | 7/7 | none |
+| `google_ads_transcribe` | 7/7 | none |
+| `google_calendar_create_event` | 7/7 | none |
+| `harvest_paa` | 7/7 | none |
+| `import_service_connection_to_memory` | 7/7 | none |
+| `instagram_media_download` | 7/7 | none |
+| `instagram_profile_content` | 7/7 | none |
+| `library-ingest` | 7/7 | none |
+| `list_service_connections` | 7/7 | none |
+| `list-channel-members` | 7/7 | none |
+| `list-channel-messages` | 7/7 | none |
+| `list-memory-tags` | 7/7 | none |
+| `list-scheduled-actions` | 7/7 | none |
+| `list-shared-with-me` | 7/7 | none |
+| `list-vaults` | 7/7 | none |
+| `list-webhooks` | 7/7 | none |
+| `map_site_urls` | 7/7 | none |
+| `maps_place_intel` | 7/7 | none |
+| `maps_search` | 7/7 | none |
+| `memory-backlinks` | 7/7 | none |
+| `memory-capture` | 7/7 | none |
+| `memory-export` | 7/7 | none |
+| `memory-get` | 7/7 | none |
+| `memory-graph-path` | 7/7 | none |
+| `memory-graph-universe` | 7/7 | none |
+| `memory-list` | 7/7 | none |
+| `memory-put` | 7/7 | none |
+| `memory-questions` | 7/7 | none |
+| `memory-search` | 7/7 | none |
+| `memory-suggest` | 7/7 | none |
+| `memory-upload` | 7/7 | none |
+| `meta_ad_creative_media` | 7/7 | none |
+| `my-mentions` | 7/7 | none |
+| `pause-scheduled-action` | 7/7 | none |
+| `poll-channel` | 7/7 | none |
+| `post-message` | 7/7 | none |
+| `prepare-memory-write` | 7/7 | none |
+| `propose-scheduled-action` | 7/7 | none |
+| `provision-defaults` | 7/7 | none |
+| `query_fanout_workflow` | 7/7 | none |
+| `rank_tracker_workflow` | 7/7 | none |
+| `react-message` | 7/7 | none |
+| `read_service_connection` | 7/7 | none |
+| `record-fact` | 7/7 | none |
+| `reddit_thread` | 7/7 | none |
+| `reddit_trending` | 7/7 | none |
+| `remove-channel-member` | 7/7 | none |
+| `renew_connected_data_download` | 7/7 | none |
+| `reply-message` | 7/7 | none |
+| `report_artifact_read` | 7/7 | none |
+| `resolve-memory-tags` | 7/7 | none |
+| `resume-scheduled-action` | 7/7 | none |
+| `revoke-chat-link` | 7/7 | none |
+| `revoke-schedule-link` | 7/7 | none |
+| `revoke-vault-app-link` | 7/7 | none |
+| `revoke-webhook` | 7/7 | none |
+| `route-memory` | 7/7 | none |
+| `search_serp` | 7/7 | none |
+| `set_scheduled_action_connections` | 7/7 | none |
+| `set-agent-identity` | 7/7 | none |
+| `set-schedule-defaults` | 7/7 | none |
+| `set-schedule-entitlement` | 7/7 | none |
+| `slack_send_message` | 7/7 | none |
+| `storage-usage` | 7/7 | none |
+| `table-create` | 7/7 | none |
+| `table-delete-rows` | 7/7 | none |
+| `table-describe` | 7/7 | none |
+| `table-drop` | 7/7 | none |
+| `table-insert-rows` | 7/7 | none |
+| `table-list` | 7/7 | none |
+| `table-query` | 7/7 | none |
+| `temporal-recall` | 7/7 | none |
+| `test_service_connection` | 7/7 | none |
+| `trustpilot_reviews` | 7/7 | none |
+| `upsert-memory-tag` | 7/7 | none |
+| `validate-memory-write` | 7/7 | none |
+| `video_frame_analysis` | 7/7 | none |
+| `video_frame_analysis_status` | 7/7 | none |
+| `video-analyze-start` | 7/7 | none |
+| `video-analyze-status` | 7/7 | none |
+| `workflow_artifact_read` | 7/7 | none |
+| `workflow_list` | 7/7 | none |
+| `workflow_run` | 7/7 | none |
+| `workflow_status` | 7/7 | none |
+| `workflow_step` | 7/7 | none |
+| `workflow_suggest` | 7/7 | none |
+| `youtube_harvest` | 7/7 | none |
+| `youtube_transcribe` | 7/7 | none |
+| `zoom_create_meeting` | 7/7 | none |

package/docs/mcp-tool-manifest.generated.json

@@ -1,9 +1,9 @@
 {
-  "generatedAt": "2026-07-24T22:48:53.625Z",
+  "generatedAt": "2026-07-26T13:48:25.203Z",
   "generatedFrom": "dist/bin/mcp-stdio-server.js",
   "serverInfo": {
     "name": "mcp-scraper",
-    "version": "0.33.7"
+    "version": "0.34.1"
   },
   "counts": {
     "unified_stdio": 167
@@ -15606,7 +15606,7 @@
     {
       "name": "query_fanout_workflow",
       "title": "Capture AI Search Fan-Out",
-      "description": "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites. Returns raw structured data for you to classify and analyze. Set export=true for JSON/CSV/TSV/HTML artifacts. WRITE NOTE: passing prompt submits a real message in the user's logged-in account — only send when the user wants that; omit it to capture a prompt the user just ran. The session must already be open on chatgpt.com or claude.ai (see browser_profile_connect) while the prompt streams. NOT for Google AI Overview — use harvest_paa for that.",
+      "description": "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites. Complete structured data is always returned inline for analysis. export=true additionally writes JSON/CSV/TSV/HTML only from an installed local MCP server; hosted OAuth/HTTP clients receive exports=null and use the inline data. A local export failure does not discard a successful capture. WRITE NOTE: passing prompt submits a real message in the user's logged-in account — only send when the user wants that; omit it to capture a prompt the user just ran. The session must already be open on chatgpt.com or claude.ai (see browser_profile_connect) while the prompt streams. NOT for Google AI Overview — use harvest_paa for that.",
       "inputSchema": {
         "type": "object",
         "properties": {
@@ -15636,7 +15636,7 @@
           "export": {
             "type": "boolean",
             "default": false,
-            "description": "Write JSON/CSV/TSV/HTML exports to MCP_SCRAPER_OUTPUT_DIR/fanout, returning relative paths."
+            "description": "When using the installed local MCP server, write JSON/CSV/TSV/HTML exports to MCP_SCRAPER_OUTPUT_DIR/fanout. Hosted clients such as ChatGPT always receive the complete structured result inline and leave exports null."
           }
         },
         "required": [
@@ -15914,6 +15914,13 @@
               "null"
             ]
           },
+          "export_error": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "description": "Non-fatal local export failure. The inline capture remains complete when this is present."
+          },
           "exports": {
             "anyOf": [
               {
@@ -16727,7 +16734,7 @@
     {
       "name": "reddit_trending",
       "title": "Reddit Trending",
-      "description": "Discover the top Reddit conversations about a topic from the last 30 days: searches Reddit (optionally one subreddit), ranks threads by engagement (upvotes + 2x comments), scrapes the top ones, and extracts the real questions people asked. Each scraped thread takes ~30s, so keep maxThreads small; for a wide scan set includeComments:false to rank cheaply first, then read the winners with reddit_thread. Not for reading one known thread URL — use reddit_thread for that.",
+      "description": "Discover the top Reddit conversations about a topic from the last week or month: finds relevant recent threads via a Google site:reddit.com search (optionally scoped to one subreddit), scrapes them for real upvotes, comments, and the questions people asked, and ranks by engagement (upvotes + 2x comments). Scraping runs in parallel across the discovered threads; set includeComments:false for a fast, cheap discovery-only sweep (relevant thread list, no engagement stats, no per-thread billing) and then read the ones you want with reddit_thread. Not for reading one known thread URL — use reddit_thread for that.",
       "inputSchema": {
         "type": "object",
         "properties": {
@@ -16741,17 +16748,26 @@
             "minLength": 1,
             "description": "Bare subreddit name to scope the scan to one community, e.g. \"SEO\" (no r/ prefix, no URL). Omit to scan all of Reddit."
           },
+          "window": {
+            "type": "string",
+            "enum": [
+              "week",
+              "month"
+            ],
+            "default": "month",
+            "description": "How recent the threads must be: \"week\" or \"month\" (default). Applied via a Google time filter over reddit.com, so it reflects genuine recency."
+          },
           "maxThreads": {
             "type": "integer",
             "minimum": 1,
-            "maximum": 25,
-            "default": 5,
-            "description": "Top-ranked threads to keep. Default 5 — each scraped thread adds ~30s and the request has a hard ~300s ceiling, so raise this above 5 only with includeComments:false."
+            "maximum": 40,
+            "default": 20,
+            "description": "How many discovered threads to scrape and rank. Default 20 (scrape-all). Each scraped thread is billed like reddit_thread + its comments, so lower this to cap cost; raise toward 40 for a wider sweep. Scraping runs in parallel and stops early if it nears the request time limit (partial:true in the response)."
           },
           "includeComments": {
             "type": "boolean",
             "default": true,
-            "description": "Scrape each ranked thread for its body and comments to extract questions. Set false for a fast, cheap discovery-only sweep (ranking + metadata only), then call reddit_thread on the winners."
+            "description": "Scrape each discovered thread for real upvotes, comments, and the questions people asked, then rank by engagement. Set false for a fast, cheap discovery-only sweep — returns the discovered threads (title + url) in relevance order with NO engagement stats and NO per-thread billing, so you can then call reddit_thread on the ones you want."
           },
           "maxCommentsPerThread": {
             "type": "integer",
@@ -16877,7 +16893,14 @@
             "type": "integer",
             "minimum": 0
           },
-          "searchUrl": {
+          "candidatesFound": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "partial": {
+            "type": "boolean"
+          },
+          "searchQuery": {
             "type": "string"
           }
         },
@@ -16889,7 +16912,9 @@
           "rankedThreads",
           "questions",
           "threadsScraped",
-          "searchUrl"
+          "candidatesFound",
+          "partial",
+          "searchQuery"
         ],
         "additionalProperties": false,
         "$schema": "http://json-schema.org/draft-07/schema#"
@@ -17679,6 +17704,16 @@
             "maximum": 2,
             "default": 1,
             "description": "Number of result pages to fetch (1–2)."
+          },
+          "recency": {
+            "type": "string",
+            "enum": [
+              "day",
+              "week",
+              "month",
+              "year"
+            ],
+            "description": "Restrict results to a recent time window (Google \"past day/week/month/year\" filter). Omit for all-time. Useful for \"what is being said this week\" style queries; pairs well with a site: operator in the query."
           }
         },
         "required": [

package/docs/specs/query-fanout-transport-contract-fix.md

@@ -0,0 +1,37 @@
+# Query fan-out transport contract fix
+
+- **Status:** Implemented; pending deployed live proof
+- **Baseline:** `origin/main` at `e2c90f8db7c22fa2aa48fc62cef10e2e0c799e15` (2026-07-25)
+- **Decision:** [Keep hosted fan-out results inline](../decisions/2026-07-25-hosted-fanout-results-stay-inline.md)
+- **Owner:** MCP Scraper browser-agent surface
+
+## Problem
+
+`/mcp` correctly registers browser-agent tools with `savesReportsLocally: false`, but `query_fanout_workflow` previously forwarded `input.export` to the browser service and treated remote export paths as usable. An OAuth call with `export: true` could therefore fail while writing an inaccessible sandbox path.
+
+## Contract
+
+| Transport | Capture data | `export: true` behavior | Filesystem contract |
+| --- | --- | --- | --- |
+| Local stdio or MCPB | Always inline structured content | Client writes optional JSON/CSV/TSV/HTML after capture | Relative paths under local `MCP_SCRAPER_OUTPUT_DIR/fanout` |
+| Hosted HTTP/OAuth `/mcp` | Always inline structured content | Accepted for compatibility but does not create files | `exports: null`; no remote or sandbox path is returned |
+
+The inline result is canonical: queries, researched/cited URLs, snippets, counts, aggregates, and capture metadata must remain immediately usable by an AI.
+
+## Implementation requirements
+
+1. Send `export: false` to `/agent/sessions/:id/capture-fanout` for every transport.
+2. Return `res.data.result ?? res.data` inline on success and never return `res.data.exports`.
+3. Export only when `opts.savesReportsLocally !== false`, the caller asked for it, and the inline result is an enriched fan-out capture.
+4. Keep `exports` nullable. A local write failure returns the successful inline capture, `exports: null`, and optional `export_error`.
+5. Keep `export` accepted for compatibility and make docs/manifest distinguish local export from hosted inline delivery.
+
+## Verification
+
+Focused tests must prove hosted calls force upstream `export: false` and return `exports: null`; local calls force the same upstream value, then write relative local files; and a local write failure is non-fatal. Preserve the 180,000 ms capture timeout allowance (at least 210,000 ms upstream).
+
+Live release acceptance requires an explicitly authorized new web-search prompt: hosted OAuth with `export: true` must return a populated inline result with `exports: null`; stdio with the same option must additionally produce readable local exports.
+
+## Out of scope
+
+Public/shared download URLs, capture-parser changes, profile authentication, billing, timeout changes, and recovery of fan-out from pre-capture prompts.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.33.7",
+  "version": "0.34.1",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",