mcp-scraper 0.34.0 → 0.34.1

package/dist/{site-extract-repository-JHMVHENZ.js → site-extract-repository-OLVWMOU2.js}

@@ -10,8 +10,8 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-SUOHUXQS.js";
-import "./chunk-JBSGBSGT.js";
+} from "./chunk-5PZ6N2QM.js";
+import "./chunk-BWXLTWF7.js";
 import "./chunk-M2S27J6Z.js";
 import "./chunk-62DQAWPF.js";
 export {
@@ -27,4 +27,4 @@ export {
   setExtractJobTotal,
   settleExtractJob
 };
-//# sourceMappingURL=site-extract-repository-JHMVHENZ.js.map
+//# sourceMappingURL=site-extract-repository-OLVWMOU2.js.map

package/dist/{worker-ZZHSYI3Y.js → worker-645BZPEK.js}

@@ -6,17 +6,17 @@ import {
 } from "./chunk-3HBPKR5G.js";
 import {
   harvest
-} from "./chunk-4KQVIYIH.js";
+} from "./chunk-XVVNKASZ.js";
 import {
   browserServiceApiKey,
   runWithCostContext
-} from "./chunk-V36LS5YV.js";
+} from "./chunk-4ZB3X6BQ.js";
 import "./chunk-K443GQY5.js";
 import "./chunk-CB5C3BPB.js";
 import {
   MC_COSTS,
   serpActualCostMc
-} from "./chunk-JBSGBSGT.js";
+} from "./chunk-BWXLTWF7.js";
 import "./chunk-M2S27J6Z.js";
 import {
   claimPendingJob,
@@ -139,4 +139,4 @@ export {
   startWorker,
   tickOnce
 };
-//# sourceMappingURL=worker-ZZHSYI3Y.js.map
+//# sourceMappingURL=worker-645BZPEK.js.map

package/docs/mcp-tool-craft-lint.generated.md

@@ -1,56 +1,178 @@
 # MCP Tool Craft Lint
 
-Generated: 2026-06-23T22:55:40.518Z
+Generated: 2026-07-26T13:48:25.492Z
+Manifest: `/Users/vilovieta/.codex/worktrees/mcp-scraper-query-fanout-transport-fix/docs/mcp-tool-manifest.generated.json`
 
-Tools: 46
-Checks per tool: 10
+Tools: 167
+Checks per tool: 7
 Failing tools: 0
 
 | Tool | Score | Missing |
 |---|---:|---|
-| `harvest_paa` | 10/10 | none |
-| `search_serp` | 10/10 | none |
-| `extract_url` | 10/10 | none |
-| `map_site_urls` | 10/10 | none |
-| `extract_site` | 10/10 | none |
-| `youtube_harvest` | 10/10 | none |
-| `youtube_transcribe` | 10/10 | none |
-| `facebook_page_intel` | 10/10 | none |
-| `facebook_ad_search` | 10/10 | none |
-| `facebook_ad_transcribe` | 10/10 | none |
-| `facebook_video_transcribe` | 10/10 | none |
-| `instagram_profile_content` | 10/10 | none |
-| `instagram_media_download` | 10/10 | none |
-| `maps_place_intel` | 10/10 | none |
-| `maps_search` | 10/10 | none |
-| `directory_workflow` | 10/10 | none |
-| `workflow_list` | 10/10 | none |
-| `workflow_suggest` | 10/10 | none |
-| `workflow_run` | 10/10 | none |
-| `workflow_step` | 10/10 | none |
-| `workflow_status` | 10/10 | none |
-| `workflow_artifact_read` | 10/10 | none |
-| `rank_tracker_blueprint` | 10/10 | none |
-| `credits_info` | 10/10 | none |
-| `browser_open` | 10/10 | none |
-| `browser_profile_onboard` | 10/10 | none |
-| `browser_profile_status` | 10/10 | none |
-| `browser_screenshot` | 10/10 | none |
-| `browser_read` | 10/10 | none |
-| `browser_locate` | 10/10 | none |
-| `browser_goto` | 10/10 | none |
-| `browser_click` | 10/10 | none |
-| `browser_type` | 10/10 | none |
-| `browser_scroll` | 10/10 | none |
-| `browser_press` | 10/10 | none |
-| `browser_replay_start` | 10/10 | none |
-| `browser_replay_stop` | 10/10 | none |
-| `browser_list_replays` | 10/10 | none |
-| `browser_replay_download` | 10/10 | none |
-| `browser_replay_mark` | 10/10 | none |
-| `browser_replay_annotate` | 10/10 | none |
-| `browser_close` | 10/10 | none |
-| `browser_list_sessions` | 10/10 | none |
-| `browser_capture_fanout` | 10/10 | none |
-| `capture_serp_snapshot` | 10/10 | none |
-| `capture_serp_page_snapshots` | 10/10 | none |
+| `access-accept-share` | 7/7 | none |
+| `access-approve-sender` | 7/7 | none |
+| `access-decline-share` | 7/7 | none |
+| `access-inbox-settings` | 7/7 | none |
+| `access-invite-account` | 7/7 | none |
+| `access-issue-key` | 7/7 | none |
+| `access-list-approved-senders` | 7/7 | none |
+| `access-list-keys` | 7/7 | none |
+| `access-note-inbox` | 7/7 | none |
+| `access-remove-approved-sender` | 7/7 | none |
+| `access-revoke-key` | 7/7 | none |
+| `access-revoke-share` | 7/7 | none |
+| `access-set-scope` | 7/7 | none |
+| `access-share-note` | 7/7 | none |
+| `access-share-vault` | 7/7 | none |
+| `access-swap-vault` | 7/7 | none |
+| `access-switch-account` | 7/7 | none |
+| `access-unlink-share` | 7/7 | none |
+| `add-vault` | 7/7 | none |
+| `audit_site` | 7/7 | none |
+| `browser_click` | 7/7 | none |
+| `browser_close` | 7/7 | none |
+| `browser_extension_delete` | 7/7 | none |
+| `browser_extension_import` | 7/7 | none |
+| `browser_extension_list` | 7/7 | none |
+| `browser_goto` | 7/7 | none |
+| `browser_list_replays` | 7/7 | none |
+| `browser_list_sessions` | 7/7 | none |
+| `browser_locate` | 7/7 | none |
+| `browser_open` | 7/7 | none |
+| `browser_press` | 7/7 | none |
+| `browser_profile_connect` | 7/7 | none |
+| `browser_profile_list` | 7/7 | none |
+| `browser_read` | 7/7 | none |
+| `browser_replay_annotate` | 7/7 | none |
+| `browser_replay_download` | 7/7 | none |
+| `browser_replay_mark` | 7/7 | none |
+| `browser_replay_start` | 7/7 | none |
+| `browser_replay_stop` | 7/7 | none |
+| `browser_screenshot` | 7/7 | none |
+| `browser_scroll` | 7/7 | none |
+| `browser_type` | 7/7 | none |
+| `bulk-delete-notes` | 7/7 | none |
+| `call_service_connection_action` | 7/7 | none |
+| `capture_serp_page_snapshots` | 7/7 | none |
+| `capture_serp_snapshot` | 7/7 | none |
+| `check_site_export` | 7/7 | none |
+| `cost-usage` | 7/7 | none |
+| `create-channel` | 7/7 | none |
+| `create-scheduled-action` | 7/7 | none |
+| `create-secure-vault` | 7/7 | none |
+| `create-webhook` | 7/7 | none |
+| `credits_info` | 7/7 | none |
+| `delete-note` | 7/7 | none |
+| `delete-scheduled-action` | 7/7 | none |
+| `delete-vault` | 7/7 | none |
+| `describe_service_connection_tool` | 7/7 | none |
+| `diff_page` | 7/7 | none |
+| `directory_workflow` | 7/7 | none |
+| `export_connected_service_data` | 7/7 | none |
+| `export_search_console_table_data` | 7/7 | none |
+| `extract_site` | 7/7 | none |
+| `extract_url` | 7/7 | none |
+| `facebook_ad_search` | 7/7 | none |
+| `facebook_ad_transcribe` | 7/7 | none |
+| `facebook_page_intel` | 7/7 | none |
+| `facebook_video_transcribe` | 7/7 | none |
+| `fact-history` | 7/7 | none |
+| `g2_reviews` | 7/7 | none |
+| `get-chat-link` | 7/7 | none |
+| `get-message-note` | 7/7 | none |
+| `get-schedule-link` | 7/7 | none |
+| `get-schedule-status` | 7/7 | none |
+| `get-vault-app-link` | 7/7 | none |
+| `get-vault-contract` | 7/7 | none |
+| `gmail_search_contacts` | 7/7 | none |
+| `gmail_send_message` | 7/7 | none |
+| `google_ads_page_intel` | 7/7 | none |
+| `google_ads_search` | 7/7 | none |
+| `google_ads_transcribe` | 7/7 | none |
+| `google_calendar_create_event` | 7/7 | none |
+| `harvest_paa` | 7/7 | none |
+| `import_service_connection_to_memory` | 7/7 | none |
+| `instagram_media_download` | 7/7 | none |
+| `instagram_profile_content` | 7/7 | none |
+| `library-ingest` | 7/7 | none |
+| `list_service_connections` | 7/7 | none |
+| `list-channel-members` | 7/7 | none |
+| `list-channel-messages` | 7/7 | none |
+| `list-memory-tags` | 7/7 | none |
+| `list-scheduled-actions` | 7/7 | none |
+| `list-shared-with-me` | 7/7 | none |
+| `list-vaults` | 7/7 | none |
+| `list-webhooks` | 7/7 | none |
+| `map_site_urls` | 7/7 | none |
+| `maps_place_intel` | 7/7 | none |
+| `maps_search` | 7/7 | none |
+| `memory-backlinks` | 7/7 | none |
+| `memory-capture` | 7/7 | none |
+| `memory-export` | 7/7 | none |
+| `memory-get` | 7/7 | none |
+| `memory-graph-path` | 7/7 | none |
+| `memory-graph-universe` | 7/7 | none |
+| `memory-list` | 7/7 | none |
+| `memory-put` | 7/7 | none |
+| `memory-questions` | 7/7 | none |
+| `memory-search` | 7/7 | none |
+| `memory-suggest` | 7/7 | none |
+| `memory-upload` | 7/7 | none |
+| `meta_ad_creative_media` | 7/7 | none |
+| `my-mentions` | 7/7 | none |
+| `pause-scheduled-action` | 7/7 | none |
+| `poll-channel` | 7/7 | none |
+| `post-message` | 7/7 | none |
+| `prepare-memory-write` | 7/7 | none |
+| `propose-scheduled-action` | 7/7 | none |
+| `provision-defaults` | 7/7 | none |
+| `query_fanout_workflow` | 7/7 | none |
+| `rank_tracker_workflow` | 7/7 | none |
+| `react-message` | 7/7 | none |
+| `read_service_connection` | 7/7 | none |
+| `record-fact` | 7/7 | none |
+| `reddit_thread` | 7/7 | none |
+| `reddit_trending` | 7/7 | none |
+| `remove-channel-member` | 7/7 | none |
+| `renew_connected_data_download` | 7/7 | none |
+| `reply-message` | 7/7 | none |
+| `report_artifact_read` | 7/7 | none |
+| `resolve-memory-tags` | 7/7 | none |
+| `resume-scheduled-action` | 7/7 | none |
+| `revoke-chat-link` | 7/7 | none |
+| `revoke-schedule-link` | 7/7 | none |
+| `revoke-vault-app-link` | 7/7 | none |
+| `revoke-webhook` | 7/7 | none |
+| `route-memory` | 7/7 | none |
+| `search_serp` | 7/7 | none |
+| `set_scheduled_action_connections` | 7/7 | none |
+| `set-agent-identity` | 7/7 | none |
+| `set-schedule-defaults` | 7/7 | none |
+| `set-schedule-entitlement` | 7/7 | none |
+| `slack_send_message` | 7/7 | none |
+| `storage-usage` | 7/7 | none |
+| `table-create` | 7/7 | none |
+| `table-delete-rows` | 7/7 | none |
+| `table-describe` | 7/7 | none |
+| `table-drop` | 7/7 | none |
+| `table-insert-rows` | 7/7 | none |
+| `table-list` | 7/7 | none |
+| `table-query` | 7/7 | none |
+| `temporal-recall` | 7/7 | none |
+| `test_service_connection` | 7/7 | none |
+| `trustpilot_reviews` | 7/7 | none |
+| `upsert-memory-tag` | 7/7 | none |
+| `validate-memory-write` | 7/7 | none |
+| `video_frame_analysis` | 7/7 | none |
+| `video_frame_analysis_status` | 7/7 | none |
+| `video-analyze-start` | 7/7 | none |
+| `video-analyze-status` | 7/7 | none |
+| `workflow_artifact_read` | 7/7 | none |
+| `workflow_list` | 7/7 | none |
+| `workflow_run` | 7/7 | none |
+| `workflow_status` | 7/7 | none |
+| `workflow_step` | 7/7 | none |
+| `workflow_suggest` | 7/7 | none |
+| `youtube_harvest` | 7/7 | none |
+| `youtube_transcribe` | 7/7 | none |
+| `zoom_create_meeting` | 7/7 | none |

package/docs/mcp-tool-manifest.generated.json

@@ -1,9 +1,9 @@
 {
-  "generatedAt": "2026-07-25T03:25:17.950Z",
+  "generatedAt": "2026-07-26T13:48:25.203Z",
   "generatedFrom": "dist/bin/mcp-stdio-server.js",
   "serverInfo": {
     "name": "mcp-scraper",
-    "version": "0.33.7"
+    "version": "0.34.1"
   },
   "counts": {
     "unified_stdio": 167
@@ -15606,7 +15606,7 @@
     {
       "name": "query_fanout_workflow",
       "title": "Capture AI Search Fan-Out",
-      "description": "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites. Returns raw structured data for you to classify and analyze. Set export=true for JSON/CSV/TSV/HTML artifacts. WRITE NOTE: passing prompt submits a real message in the user's logged-in account — only send when the user wants that; omit it to capture a prompt the user just ran. The session must already be open on chatgpt.com or claude.ai (see browser_profile_connect) while the prompt streams. NOT for Google AI Overview — use harvest_paa for that.",
+      "description": "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites. Complete structured data is always returned inline for analysis. export=true additionally writes JSON/CSV/TSV/HTML only from an installed local MCP server; hosted OAuth/HTTP clients receive exports=null and use the inline data. A local export failure does not discard a successful capture. WRITE NOTE: passing prompt submits a real message in the user's logged-in account — only send when the user wants that; omit it to capture a prompt the user just ran. The session must already be open on chatgpt.com or claude.ai (see browser_profile_connect) while the prompt streams. NOT for Google AI Overview — use harvest_paa for that.",
       "inputSchema": {
         "type": "object",
         "properties": {
@@ -15636,7 +15636,7 @@
           "export": {
             "type": "boolean",
             "default": false,
-            "description": "Write JSON/CSV/TSV/HTML exports to MCP_SCRAPER_OUTPUT_DIR/fanout, returning relative paths."
+            "description": "When using the installed local MCP server, write JSON/CSV/TSV/HTML exports to MCP_SCRAPER_OUTPUT_DIR/fanout. Hosted clients such as ChatGPT always receive the complete structured result inline and leave exports null."
           }
         },
         "required": [
@@ -15914,6 +15914,13 @@
               "null"
             ]
           },
+          "export_error": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "description": "Non-fatal local export failure. The inline capture remains complete when this is present."
+          },
           "exports": {
             "anyOf": [
               {

package/docs/specs/query-fanout-transport-contract-fix.md

@@ -0,0 +1,37 @@
+# Query fan-out transport contract fix
+
+- **Status:** Implemented; pending deployed live proof
+- **Baseline:** `origin/main` at `e2c90f8db7c22fa2aa48fc62cef10e2e0c799e15` (2026-07-25)
+- **Decision:** [Keep hosted fan-out results inline](../decisions/2026-07-25-hosted-fanout-results-stay-inline.md)
+- **Owner:** MCP Scraper browser-agent surface
+
+## Problem
+
+`/mcp` correctly registers browser-agent tools with `savesReportsLocally: false`, but `query_fanout_workflow` previously forwarded `input.export` to the browser service and treated remote export paths as usable. An OAuth call with `export: true` could therefore fail while writing an inaccessible sandbox path.
+
+## Contract
+
+| Transport | Capture data | `export: true` behavior | Filesystem contract |
+| --- | --- | --- | --- |
+| Local stdio or MCPB | Always inline structured content | Client writes optional JSON/CSV/TSV/HTML after capture | Relative paths under local `MCP_SCRAPER_OUTPUT_DIR/fanout` |
+| Hosted HTTP/OAuth `/mcp` | Always inline structured content | Accepted for compatibility but does not create files | `exports: null`; no remote or sandbox path is returned |
+
+The inline result is canonical: queries, researched/cited URLs, snippets, counts, aggregates, and capture metadata must remain immediately usable by an AI.
+
+## Implementation requirements
+
+1. Send `export: false` to `/agent/sessions/:id/capture-fanout` for every transport.
+2. Return `res.data.result ?? res.data` inline on success and never return `res.data.exports`.
+3. Export only when `opts.savesReportsLocally !== false`, the caller asked for it, and the inline result is an enriched fan-out capture.
+4. Keep `exports` nullable. A local write failure returns the successful inline capture, `exports: null`, and optional `export_error`.
+5. Keep `export` accepted for compatibility and make docs/manifest distinguish local export from hosted inline delivery.
+
+## Verification
+
+Focused tests must prove hosted calls force upstream `export: false` and return `exports: null`; local calls force the same upstream value, then write relative local files; and a local write failure is non-fatal. Preserve the 180,000 ms capture timeout allowance (at least 210,000 ms upstream).
+
+Live release acceptance requires an explicitly authorized new web-search prompt: hosted OAuth with `export: true` must return a populated inline result with `exports: null`; stdio with the same option must additionally produce readable local exports.
+
+## Out of scope
+
+Public/shared download URLs, capture-parser changes, profile authentication, billing, timeout changes, and recovery of fan-out from pre-capture prompts.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.34.0",
+  "version": "0.34.1",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",