mcp-scraper 0.16.0 → 0.17.2

package/docs/mcp-tool-manifest.generated.json

@@ -1,12 +1,12 @@
 {
-  "generatedAt": "2026-07-12T18:46:14.399Z",
+  "generatedAt": "2026-07-12T21:11:03.690Z",
   "generatedFrom": "dist/bin/mcp-stdio-server.js",
   "serverInfo": {
     "name": "mcp-scraper",
-    "version": "0.16.0"
+    "version": "0.17.0"
   },
   "counts": {
-    "unified_stdio": 156
+    "unified_stdio": 157
   },
   "surfaces": {
     "unified_stdio": [
@@ -87,6 +87,7 @@
       "google_ads_transcribe",
       "google_calendar_create_event",
       "harvest_paa",
+      "import_service_connection_to_memory",
       "instagram_media_download",
       "instagram_profile_content",
       "library-ingest",
@@ -8950,6 +8951,144 @@
         "taskSupport": "forbidden"
       }
     },
+    {
+      "name": "import_service_connection_to_memory",
+      "title": "Import Connected Service Snapshot to Memory",
+      "description": "Run exactly one bounded, approved read on a tenant-owned connected service and upsert the redacted result into an existing ordinary Memory vault at a server-generated stable path. The saved document is embedded for RAG and marked as untrusted provider data, never instructions. This is a one-result snapshot: it does not paginate, bulk-import an account, continuously sync changes, propagate deletions, or create normalized tables. Use list_service_connections first and supply an exact current readTools entry; action and admin tools are rejected.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "connectionId": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 200,
+            "description": "A tenant-owned connectionId from list_service_connections."
+          },
+          "providerConfigKey": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 200,
+            "description": "The exact providerConfigKey returned with that connection. It is matched together with connectionId against the authenticated caller."
+          },
+          "tool": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 200,
+            "description": "One exact current readTools entry for that connection. Actions, admin tools, and unlisted names are rejected."
+          },
+          "args": {
+            "type": "object",
+            "additionalProperties": {},
+            "description": "JSON arguments for one bounded provider read. The serialized object may be at most 64 KiB."
+          },
+          "vault": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 100,
+            "description": "An existing ordinary Memory vault the caller can write and index. Secure and channel vaults are rejected because this tool creates searchable RAG content."
+          },
+          "title": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 200,
+            "description": "Optional human-readable snapshot title. The server always chooses the stable storage path."
+          }
+        },
+        "required": [
+          "connectionId",
+          "providerConfigKey",
+          "tool",
+          "vault"
+        ],
+        "additionalProperties": false,
+        "$schema": "http://json-schema.org/draft-07/schema#"
+      },
+      "outputSchema": {
+        "type": "object",
+        "properties": {
+          "ok": {
+            "type": "boolean"
+          },
+          "stored": {
+            "type": "boolean"
+          },
+          "status": {
+            "type": "string",
+            "enum": [
+              "search_ready",
+              "stored_not_indexed"
+            ]
+          },
+          "searchReady": {
+            "type": "boolean"
+          },
+          "providerConfigKey": {
+            "type": "string"
+          },
+          "connectionId": {
+            "type": "string"
+          },
+          "tool": {
+            "type": "string"
+          },
+          "vault": {
+            "type": "string"
+          },
+          "path": {
+            "type": "string"
+          },
+          "sourceBytes": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "contentSha256": {
+            "type": "string"
+          },
+          "indexedChunks": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "importedAt": {
+            "type": "string"
+          },
+          "untrustedContent": {
+            "type": "boolean",
+            "const": true
+          },
+          "warning": {
+            "type": "string"
+          },
+          "errorCode": {
+            "type": "string"
+          },
+          "retryable": {
+            "type": "boolean"
+          },
+          "error": {
+            "type": [
+              "string",
+              "null"
+            ]
+          }
+        },
+        "required": [
+          "ok",
+          "error"
+        ],
+        "additionalProperties": false,
+        "$schema": "http://json-schema.org/draft-07/schema#"
+      },
+      "annotations": {
+        "title": "Import Connected Service Snapshot to Memory",
+        "readOnlyHint": false,
+        "destructiveHint": false,
+        "idempotentHint": true,
+        "openWorldHint": true
+      },
+      "execution": {
+        "taskSupport": "forbidden"
+      }
+    },
     {
       "name": "instagram_media_download",
       "title": "Instagram Post/Reel Media Download",

package/docs/specs/google-drive-bulk-access-and-mcp-schema-passthrough-spec.md

@@ -10,7 +10,7 @@
 - Nango production has the exact source-controlled 12-function Drive inventory: ten bounded reads plus the additive `create-drive-folder` and `copy-drive-file` actions.
 - The scheduler exposes the live-policy intersection as 10 reads and 2 actions, injects exact per-schedule grants into transient Mastra toolsets, and audits classified mutations without storing argument values.
 - The current tenant-owned Drive connection is healthy. Its explicitly authorized action switch is on; newly connected accounts still default to actions off.
-- The fixed public MCP remains a 156-tool provider-neutral surface. `describe_service_connection_tool` returned live 64-character schema hashes for both a Drive read and action; `get-about` and a one-item `list-drive-files` page passed through production.
+- The fixed public MCP now exposes a 157-tool provider-neutral surface. `describe_service_connection_tool` returned live 64-character schema hashes for both a Drive read and action; `get-about` and a one-item `list-drive-files` page passed through production.
 - This receipt covers interactive Phase 1 only. Deterministic `connection_sync`, background Proxy/Blob file transfer, native connection-scoped MCP `tools/list`, `upload-document`, broader mutations, and a reversible production write fixture are not shipped.
 
 ## Executive decision

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.16.0",
+  "version": "0.17.2",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -24,7 +24,10 @@
   },
   "files": [
     "dist",
-    "docs",
+    "docs/*.md",
+    "docs/*.json",
+    "docs/adr/*.md",
+    "docs/specs/*.md",
     "README.md"
   ],
   "scripts": {
@@ -50,7 +53,13 @@
     "sync:env": "bash scripts/push-env.sh",
     "sync:env:apply": "bash scripts/push-env.sh --apply",
     "test:webhook": "node scripts/test-webhook.mjs",
-    "test:e2e": "playwright test"
+    "test:e2e": "playwright test",
+    "email:preview": "node scripts/run-ts-sandbox-safe.mjs scripts/email/render-previews.ts",
+    "email:preflight": "node --import tsx scripts/email/preflight.ts",
+    "email:avatar": "node --import tsx scripts/email/generate-sender-avatar.ts",
+    "email:inbound:setup": "node --import tsx scripts/email/setup-resend-inbound.ts",
+    "email:audience": "node scripts/run-ts-sandbox-safe.mjs scripts/email/sync-resend-audience.ts",
+    "email:drafts": "node scripts/run-ts-sandbox-safe.mjs scripts/email/create-resend-drafts.ts"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.96.0",

package/dist/chunk-HQYIP5X3.js

@@ -1,7 +0,0 @@
-// src/version.ts
-var PACKAGE_VERSION = "0.16.0";
-
-export {
-  PACKAGE_VERSION
-};
-//# sourceMappingURL=chunk-HQYIP5X3.js.map

package/dist/chunk-HQYIP5X3.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.16.0'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/chunk-IKABQWYQ.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/install-terminal.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n  color?: boolean\n  version: string\n  apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n  reset: '\\x1b[0m',\n  cyan: '\\x1b[36m',\n  lime: '\\x1b[32m',\n  amber: '\\x1b[33m',\n  red: '\\x1b[31m',\n  muted: '\\x1b[90m',\n  bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n  if (!enabled) return value\n  return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n  const padded = label.padEnd(9, ' ')\n  return `  ${colorize(padded, 'muted', enabled)} ${tools.join(colorize('  .  ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n  const color = options.color ?? true\n  const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n  const ascii = String.raw`\n __  __  ____ ____ \n|  \\/  |/ ___|  _ \\\n| |\\/| | |   | |_) |\n| |  | | |___|  __/\n|_|  |_|\\____|_|\n\n ____   ____ ____      _    ____  _____ ____\n/ ___| / ___|  _ \\    / \\  |  _ \\| ____|  _ \\\n\\___ \\| |   | |_) |  / _ \\ | |_) |  _| | |_) |\n ___) | |___|  _ <  / ___ \\|  __/| |___|  _ <\n|____/ \\____|_| \\_\\/_/   \\_\\_|   |_____|_| \\_\\\n`\n\n  const claudeCommand = [\n    `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n    '  mcp-scraper-cli agent install claude --apply',\n  ].join('\\n')\n\n  const codexConfig = [\n    '[mcp_servers.mcp-scraper]',\n    'command = \"npx\"',\n    'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n    `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n  ].join('\\n')\n\n  return [\n    colorize(`mcp-scraper v${options.version}`, 'bold', color),\n    colorize('> mcp-scraper-install', 'muted', color),\n    colorize(ascii, 'amber', color),\n    `${colorize('MCP Scraper Agent', 'cyan', color)}  .  v${options.version}  .  mcpscraper.dev`,\n    '1/1 install surfaces ready',\n    colorize('Newest: Resend remote MCP + bulk connected-data exports. OAuth stays tenant-isolated; one call paginates Gmail, Calendar, Zoom, or Resend into inline data or private JSONL.', 'lime', color),\n    '',\n    `${colorize('Tools', 'cyan', color)}  ${colorize('(156 MCP tools)', 'muted', color)}`,\n    toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n    toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n    toolRow('build', ['rank_tracker_workflow', 'cron plan', 'database prompt'], color),\n    toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n    toolRow('browser', ['browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n    toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n    toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n    toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n    `${colorize('Workflows', 'cyan', color)}  ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n    toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n    toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n    '',\n    colorize('Usage tips:', 'amber', color),\n    'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n    'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n    'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n    'Hosted browser sessions use direct/no-proxy egress by default.',\n    'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n    'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n    'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n    'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n    'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n    'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n    '',\n    `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n    '',\n    colorize('Setup doctor', 'amber', color),\n    'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n    '',\n    colorize('Hosted profile setup', 'amber', color),\n    'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n    'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n    '',\n    colorize('Claude Code one-command setup', 'amber', color),\n    claudeCommand,\n    'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n    '',\n    colorize('Codex config', 'amber', color),\n    codexConfig,\n    '',\n    colorize('Claude Desktop Extension', 'amber', color),\n    'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n    '',\n    colorize('Safety note:', 'muted', color),\n    'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n    'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n    '',\n  ].join('\\n')\n}\n"],"mappings":";AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,gLAAgL,QAAQ,KAAK;AAAA,IACtM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,yBAAyB,aAAa,iBAAiB,GAAG,KAAK;AAAA,IACjF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACxN,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IACzN,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;","names":[]}