mcp-scraper 0.75.1 → 0.77.0

package/CHANGELOG.md

@@ -4,6 +4,27 @@ All notable changes to MCP Scraper are documented here. The format is based on [
 
 ## [Unreleased]
 
+## [0.77.0] - 2026-08-28
+
+### Fixed
+
+- The visual event picker can now minimize away from covered page headers, keeps the selected target visibly confirmed, and waits for a dashboard acknowledgement before showing “Added”; the confirmation explicitly distinguishes an editable draft from a saved event.
+- Required-consent Pixels now render the newest complete published gateway on approved websites, inherit the host font, apply the saved brand treatment, preserve equal-effort choices and Global Privacy Control, and durably record necessary-only decisions instead of exposing non-rendering helper code.
+
+## [0.76.0] - 2026-08-28
+
+### Added
+
+- Added a stable `mcp-scraper-core` executable that gives Thorbit exactly 34 direct public-web scraping and analysis tools while keeping integrations, connected-account actions, Scheduler, Memory, browser-agent, analytics, publishing, Commons, lead-list, Local Sourcebook, and generic workflows unregistered and uncallable.
+
+### Changed
+
+- Shared the canonical schemas and handlers between the full and restricted stdio entrypoints, with fail-closed startup when a required restricted tool is missing or malformed; the normal `mcp-scraper` executable remains full-surface.
+
+### Fixed
+
+- Made Commons ledger reads deterministic when multiple writes share the same timestamp, preventing release verification from observing an arbitrary operation order.
+
 ## [0.75.1] - 2026-08-28
 
 ### Fixed
@@ -1398,7 +1419,8 @@ All notable changes to MCP Scraper are documented here. The format is based on [
 - Write actions remain unavailable until the account owner explicitly enables them.
 - Provider-specific connection data is normalized into one agent-facing contract.
 
-[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.75.1...HEAD
+[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.76.0...HEAD
+[0.76.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.75.1...v0.76.0
 [0.75.1]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.75.0...v0.75.1
 [0.75.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.74.0...v0.75.0
 [0.74.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.8...v0.74.0

package/README.md

@@ -23,6 +23,7 @@ Use the MCPB Desktop Extension for the branded Claude Desktop install, or use th
 MCP Scraper ships one stdio entrypoint plus human-facing helper CLIs:
 
 - `mcp-scraper` — the single MCP server with every tool: live web intelligence (SERP, PAA, site extraction, YouTube, Facebook ads and organic video transcripts, Maps, directory, rank tracker blueprint, credits) plus agent-controlled direct/no-proxy hosted browser sessions (screenshots, clicks, typing, scrolling, watch URLs, replay links, MP4 replay download, and saved profile/login setup for authenticated AI visibility workflows). It is context-aware: in a human terminal it prints the branded ASCII install card; in an MCP client it runs as a protocol-clean stdio server. This is the entrypoint used by the MCPB Desktop Extension.
+- `mcp-scraper-core` — a restricted stdio entrypoint for Thorbit that registers exactly 34 direct public-web scraping and analysis tools. It excludes MCP Scraper integrations and connected-account actions, Scheduler, Memory, browser-agent, analytics, publishing, Commons, lead-list, Local Sourcebook, and generic workflow surfaces. The normal `mcp-scraper` entrypoint remains full-surface.
 - `mcp-scraper-install` — explicit alias for the human-facing terminal installer card with the branded ASCII intro and copyable install commands.
 - `mcp-scraper-cli` — a human-facing CLI for setup checks, AI-agent config generation, workflow prompts, SEO workflow runs, and HTML reports. This command is safe to print because it is not an MCP stdio server.
 
@@ -173,7 +174,7 @@ Build the branded one-click bundle:
 npm run build:mcpb
 ```
 
-The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.75.1`, SHA-256 `c24d80648f4479678dea1cef1119ef55ff43676f0aa085b53c510cbd344bf729`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
+The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.77.0`, SHA-256 `d59c2c55eeff7889a78d0fcddfb93d8062003e2ebd18f8dc8a03a73d262fc25a`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
 
 The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
 

package/dist/bin/api-server.js

@@ -17,8 +17,8 @@ loadDotEnv();
 async function main() {
   const [{ serve }, { app }, { startWorker }, { migrate }] = await Promise.all([
     import("@hono/node-server"),
-    import("../server-ZJUMRVCT.js"),
-    import("../worker-7ASMH3MT.js"),
+    import("../server-EKRNKKLU.js"),
+    import("../worker-DR6R7J46.js"),
     import("../db-566MOHHD.js")
   ]);
   const PORT = parseInt(process.env.PORT ?? "3001");

package/dist/bin/mcp-scraper-cli.js

@@ -13,7 +13,7 @@ import "../chunk-VXLU74YZ.js";
 import "../chunk-GGZEC22A.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-IUEELMHS.js";
+} from "../chunk-7HF3T7P2.js";
 
 // src/cli/human-cli.ts
 import { Command } from "commander";

package/dist/bin/mcp-scraper-core.js

@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+import {
+  runMcpScraperStdio
+} from "../chunk-IP7UDXMU.js";
+import "../chunk-DTJ56S5B.js";
+import "../chunk-PGJQDMC2.js";
+import "../chunk-DNM65UCK.js";
+import "../chunk-T3MZISOF.js";
+import "../chunk-7RQULQF3.js";
+import "../chunk-6W4ADSWE.js";
+import "../chunk-OM7HVEJ3.js";
+import "../chunk-CXY5WV45.js";
+import "../chunk-GT6MP4SU.js";
+import "../chunk-VXLU74YZ.js";
+import "../chunk-GGZEC22A.js";
+import "../chunk-7HF3T7P2.js";
+import "../chunk-WEFPBAAG.js";
+
+// src/mcp/thorbit-restricted-tool-surface.ts
+var THORBIT_RESTRICTED_TOOL_NAMES = [
+  "harvest_paa",
+  "search_serp",
+  "extract_url",
+  "diff_page",
+  "map_site_urls",
+  "map_wayback_snapshots",
+  "extract_site",
+  "analyze_site_similarity",
+  "audit_site",
+  "check_site_export",
+  "site_export_read",
+  "site_export_image",
+  "archive_read",
+  "youtube_harvest",
+  "youtube_transcribe",
+  "facebook_page_intel",
+  "facebook_ad_search",
+  "reddit_thread",
+  "reddit_trending",
+  "video_frame_analysis",
+  "video_frame_analysis_status",
+  "facebook_ad_transcribe",
+  "google_ads_search",
+  "google_ads_page_intel",
+  "google_ads_transcribe",
+  "facebook_video_transcribe",
+  "instagram_profile_content",
+  "instagram_media_download",
+  "maps_place_intel",
+  "maps_search",
+  "trustpilot_reviews",
+  "g2_reviews",
+  "capture_serp_snapshot",
+  "capture_serp_page_snapshots"
+];
+
+// bin/mcp-scraper-core.ts
+runMcpScraperStdio({
+  toolsets: /* @__PURE__ */ new Set(["paa", "serp"]),
+  allowedToolNames: THORBIT_RESTRICTED_TOOL_NAMES
+});

package/dist/bin/mcp-scraper-install.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import {
   renderInstallTerminal
-} from "../chunk-45AZ5CR2.js";
+} from "../chunk-GT6MP4SU.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-IUEELMHS.js";
+} from "../chunk-7HF3T7P2.js";
 
 // bin/mcp-scraper-install.ts
 var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;

package/dist/bin/mcp-stdio-server.js

@@ -1,173 +1,20 @@
 #!/usr/bin/env node
 import {
-  HttpMcpToolExecutor,
-  MEMORY_TOOL_REGISTRY,
-  SERVER_INSTRUCTIONS,
-  ScheduledResultsMcpExecutor,
-  hashOwnerId,
-  installInputFieldDescriptions,
-  permitsLocalNetworkAccess,
-  registerBrowserAgentMcpTools,
-  registerMemoryMcpTools,
-  registerPaaExtractorMcpTools,
-  registerScheduledResultsMcpTools,
-  registerSerpIntelligenceCaptureTools,
-  resolveDeploymentProfile
-} from "../chunk-VFG5P77C.js";
+  runMcpScraperStdio
+} from "../chunk-IP7UDXMU.js";
+import "../chunk-DTJ56S5B.js";
 import "../chunk-PGJQDMC2.js";
+import "../chunk-DNM65UCK.js";
 import "../chunk-T3MZISOF.js";
 import "../chunk-7RQULQF3.js";
+import "../chunk-6W4ADSWE.js";
 import "../chunk-OM7HVEJ3.js";
 import "../chunk-CXY5WV45.js";
-import {
-  renderInstallTerminal
-} from "../chunk-45AZ5CR2.js";
+import "../chunk-GT6MP4SU.js";
 import "../chunk-VXLU74YZ.js";
 import "../chunk-GGZEC22A.js";
-import {
-  PACKAGE_VERSION
-} from "../chunk-IUEELMHS.js";
-import "../chunk-DNM65UCK.js";
-import "../chunk-6W4ADSWE.js";
+import "../chunk-7HF3T7P2.js";
 import "../chunk-WEFPBAAG.js";
 
 // bin/mcp-stdio-server.ts
-import { readFileSync } from "fs";
-import { homedir } from "os";
-import { join } from "path";
-import { serveStdio } from "@modelcontextprotocol/server/stdio";
-import { McpServer } from "@modelcontextprotocol/server";
-
-// src/mcp/hosted-memory-mcp-tool-executor.ts
-var publicToolNameByUpstreamName = new Map(
-  MEMORY_TOOL_REGISTRY.map((schema) => [schema.upstreamName, schema.id])
-);
-function parseJsonRpcEnvelope(text) {
-  const payloads = text.split("\n").filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trim()).filter(Boolean);
-  if (!payloads.length) return JSON.parse(text);
-  for (const payload of payloads) {
-    const parsed = JSON.parse(payload);
-    if (parsed.result || parsed.error) return parsed;
-  }
-  throw new Error("hosted MCP returned no JSON-RPC result");
-}
-function errorResult(message) {
-  return {
-    content: [{ type: "text", text: JSON.stringify({ ok: false, error: message }) }],
-    isError: true
-  };
-}
-var HostedMemoryMcpToolExecutor = class {
-  baseUrl;
-  apiKey;
-  constructor(baseUrl2, apiKey2) {
-    this.baseUrl = baseUrl2.replace(/\/$/, "");
-    this.apiKey = apiKey2;
-  }
-  async callMemoryTool(upstreamName, args) {
-    const publicToolName = publicToolNameByUpstreamName.get(upstreamName);
-    if (!publicToolName) return errorResult(`unknown memory tool: ${upstreamName}`);
-    try {
-      const res = await fetch(`${this.baseUrl}/mcp`, {
-        method: "POST",
-        headers: {
-          "content-type": "application/json",
-          accept: "application/json, text/event-stream",
-          "x-api-key": this.apiKey
-        },
-        body: JSON.stringify({
-          jsonrpc: "2.0",
-          id: `memory:${publicToolName}`,
-          method: "tools/call",
-          params: { name: publicToolName, arguments: args }
-        })
-      });
-      const text = await res.text();
-      if (!res.ok) return errorResult(`hosted memory ${publicToolName} failed (HTTP ${res.status})`);
-      const envelope = parseJsonRpcEnvelope(text);
-      if (envelope.error) {
-        return errorResult(envelope.error.message ?? `hosted memory ${publicToolName} failed`);
-      }
-      return envelope.result ?? errorResult(`hosted memory ${publicToolName} returned no result`);
-    } catch (err) {
-      return errorResult(err instanceof Error ? err.message : `hosted memory ${publicToolName} call failed`);
-    }
-  }
-};
-
-// bin/mcp-stdio-server.ts
-var forceStdio = process.argv.includes("--stdio") || process.env.MCP_SCRAPER_FORCE_STDIO === "1";
-var interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY);
-var wantsHelp = process.argv.includes("--help") || process.argv.includes("-h");
-if (!forceStdio && (interactiveTerminal || wantsHelp)) {
-  const noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
-  process.stdout.write(renderInstallTerminal({
-    version: PACKAGE_VERSION,
-    color: !noColor,
-    apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim())
-  }));
-  process.exit(0);
-}
-function readApiKeyFile() {
-  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
-  const paths = [explicitPath, join(homedir(), ".mcp-scraper-key")].filter(Boolean);
-  for (const path of paths) {
-    try {
-      const value = readFileSync(path, "utf8").trim();
-      if (value) return value;
-    } catch {
-    }
-  }
-  return void 0;
-}
-var apiKey = (process.env.MCP_SCRAPER_API_KEY ?? process.env.MCP_SCRAPER_KEY ?? process.env.MCP_API_KEY ?? readApiKeyFile())?.trim();
-if (!apiKey) {
-  process.stderr.write("MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\n");
-  process.exit(1);
-}
-var requiredApiKey = apiKey;
-var baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || "https://mcpscraper.dev";
-var deploymentProfile = resolveDeploymentProfile();
-var localNetworkAccess = permitsLocalNetworkAccess({
-  deploymentProfile,
-  transportProfile: "stdio",
-  baseUrl,
-  explicitlyEnabled: process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK === "1"
-});
-var consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
-function buildStdioServer() {
-  const server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, {
-    instructions: SERVER_INSTRUCTIONS,
-    cacheHints: {
-      "server/discover": { ttlMs: 3e5, cacheScope: "private" },
-      "tools/list": { ttlMs: 3e5, cacheScope: "private" },
-      "resources/list": { ttlMs: 3e5, cacheScope: "private" },
-      "resources/templates/list": { ttlMs: 3e5, cacheScope: "private" },
-      "resources/read": { ttlMs: 6e4, cacheScope: "private" }
-    }
-  });
-  installInputFieldDescriptions(server);
-  const httpExecutor = new HttpMcpToolExecutor(baseUrl, requiredApiKey, { localNetworkAccess });
-  registerPaaExtractorMcpTools(server, httpExecutor, {
-    ownerId: hashOwnerId(requiredApiKey),
-    deploymentProfile,
-    transportProfile: "stdio",
-    baseUrl,
-    localNetworkAccess,
-    taskHandleSecret: requiredApiKey
-  });
-  registerSerpIntelligenceCaptureTools(server, httpExecutor, {
-    exposeDevelopmentDiagnostics: deploymentProfile === "development" || deploymentProfile === "test"
-  });
-  registerBrowserAgentMcpTools(server, { baseUrl, apiKey: requiredApiKey, consoleBaseUrl });
-  registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, requiredApiKey));
-  registerMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, requiredApiKey));
-  return server;
-}
-serveStdio(() => buildStdioServer(), {
-  legacy: "serve",
-  onerror(error) {
-    process.stderr.write(`${error.message}
-`);
-  }
-});
+runMcpScraperStdio();

package/dist/{chunk-IUEELMHS.js → chunk-7HF3T7P2.js}

@@ -1,5 +1,5 @@
 // src/version.ts
-var PACKAGE_VERSION = "0.75.1";
+var PACKAGE_VERSION = "0.77.0";
 
 export {
   PACKAGE_VERSION