mcp-scraper 0.86.0 → 0.86.1

package/CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to MCP Scraper are documented here. The format is based on [
 
 ## [Unreleased]
 
+## [0.86.1] - 2026-09-02
+
+### Fixed
+
+- Retried Reddit thread hydration through as many as two fresh primary residential sessions and two bounded managed-browser backup sessions, stopping after the first accepted result while retaining per-attempt provider and cost telemetry.
+- Restored visually hidden accessible labels for native CRM and Research search, select, submit, deal-stage, and X-Ray evidence controls.
+- Prevented the native CRM and Research toolbar from stretching into a full-height column at tablet and phone breakpoints.
+- Routed the native `/research/*` and `/crm/*` REST facades through the production API function instead of allowing the canonical host to fall through to a 404.
+
 ## [0.86.0] - 2026-09-01
 
 ### Added

package/README.md

@@ -175,7 +175,7 @@ Build the branded one-click bundle:
 npm run build:mcpb
 ```
 
-The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.86.0`, SHA-256 `f8de9e415f4215c7d244ed18222640aabea0743c063c95ce5ad7f3d44303700e`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
+The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.86.1`, SHA-256 `c735311afd38095d4187fb5cbb79e5abed47b7c09d78e56ee6c2ecacd1b9270f`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
 
 The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
 

package/dist/bin/api-server.js

@@ -17,7 +17,7 @@ loadDotEnv();
 async function main() {
   const [{ serve }, { app, personalAssistantProductionStartup }, { startWorker }, { migrate }] = await Promise.all([
     import("@hono/node-server"),
-    import("../server-OZBTZIDZ.js"),
+    import("../server-UQ33P5V5.js"),
     import("../worker-OQSFSLNG.js"),
     import("../db-Z34LPZNR.js")
   ]);

package/dist/bin/mcp-scraper-cli.js

@@ -19,7 +19,7 @@ import {
 import "../chunk-GGZEC22A.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-36EAKRR6.js";
+} from "../chunk-C4C4WUTV.js";
 
 // src/cli/human-cli.ts
 import { Command } from "commander";

package/dist/bin/mcp-scraper-core.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import {
   runMcpScraperStdio
-} from "../chunk-QKWNEMGH.js";
-import "../chunk-FBB6DEUV.js";
+} from "../chunk-TYLVUAK3.js";
+import "../chunk-3H7QTJLV.js";
 import "../chunk-PGJQDMC2.js";
 import "../chunk-DNM65UCK.js";
 import "../chunk-T3MZISOF.js";
@@ -10,10 +10,10 @@ import "../chunk-6HAV7LCE.js";
 import "../chunk-NKRO2SUO.js";
 import "../chunk-OM7HVEJ3.js";
 import "../chunk-OPQIGAFB.js";
-import "../chunk-7523VOQT.js";
+import "../chunk-3JRZZVWL.js";
 import "../chunk-P7FWOMU7.js";
 import "../chunk-GGZEC22A.js";
-import "../chunk-36EAKRR6.js";
+import "../chunk-C4C4WUTV.js";
 import "../chunk-YXNDOQXN.js";
 
 // src/mcp/thorbit-restricted-tool-surface.ts

package/dist/bin/mcp-scraper-install.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import {
   renderInstallTerminal
-} from "../chunk-7523VOQT.js";
+} from "../chunk-3JRZZVWL.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-36EAKRR6.js";
+} from "../chunk-C4C4WUTV.js";
 
 // bin/mcp-scraper-install.ts
 var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;

package/dist/bin/mcp-stdio-server.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import {
   runMcpScraperStdio
-} from "../chunk-QKWNEMGH.js";
-import "../chunk-FBB6DEUV.js";
+} from "../chunk-TYLVUAK3.js";
+import "../chunk-3H7QTJLV.js";
 import "../chunk-PGJQDMC2.js";
 import "../chunk-DNM65UCK.js";
 import "../chunk-T3MZISOF.js";
@@ -10,10 +10,10 @@ import "../chunk-6HAV7LCE.js";
 import "../chunk-NKRO2SUO.js";
 import "../chunk-OM7HVEJ3.js";
 import "../chunk-OPQIGAFB.js";
-import "../chunk-7523VOQT.js";
+import "../chunk-3JRZZVWL.js";
 import "../chunk-P7FWOMU7.js";
 import "../chunk-GGZEC22A.js";
-import "../chunk-36EAKRR6.js";
+import "../chunk-C4C4WUTV.js";
 import "../chunk-YXNDOQXN.js";
 
 // bin/mcp-stdio-server.ts

package/dist/{chunk-FBB6DEUV.js → chunk-3H7QTJLV.js}

@@ -50,7 +50,7 @@ import {
 } from "./chunk-P7FWOMU7.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-36EAKRR6.js";
+} from "./chunk-C4C4WUTV.js";
 import {
   PUBLIC_ERROR_CODES,
   buildPublicErrorEnvelope,
@@ -5732,13 +5732,13 @@ seam is noted so you can chain them.
 
 ## Reddit
 - Read ONE known reddit.com thread/post URL -> **reddit_thread**. It opens current Reddit through a managed
-  residential browser and returns a bounded rendered comment tree. After primary failure, its managed-browser
-  backup can recover the post and up to 25 comments.
+  residential browser and returns a bounded rendered comment tree. It makes up to two fresh primary attempts,
+  then up to two managed-browser backup attempts that can recover the post and up to 25 comments.
 - DISCOVER what a niche is talking about (topic, no known thread) -> **reddit_trending** (takes a topic, optional
   subreddit and a week, 30-day, or all-time window; returns top threads ranked by engagement plus the questions people asked \u2014
   feed winning \`rankedThreads[].url\` values into \`reddit_thread\` for the full comment tree). It uses a site-restricted
-  web search, then opens selected threads on current Reddit through managed residential browsers; each failed hydration can use the same
-  bounded backup for up to 25 comments. Before interpreting an empty result, inspect
+  web search, then opens selected threads on current Reddit through managed residential browsers; each hydration uses the same two-primary,
+  two-backup ladder, with the backup capped at 25 comments. Before interpreting an empty result, inspect
   \`resultQuality\`, \`discoverySource\`, \`degradationReasons\`, \`retryRecommended\`, and \`billingRefunded\`;
   a degraded empty result is not evidence that the topic has no Reddit discussion.
 
@@ -17452,14 +17452,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   }, async (input) => formatFacebookAdSearch(await executor.facebookAdSearch(input), input));
   server.registerTool("reddit_thread", {
     title: "Reddit Thread + Comments",
-    description: "Read one known reddit.com post and its rendered comment tree on current Reddit. The managed residential browser is primary; a bounded managed-browser backup can recover the post and up to 25 comments when the primary fails. Use reddit_trending when no thread URL is known. Costs one Reddit base lookup plus the per-comment rate for comments returned; pass maxComments to bound both output and cost.",
+    description: "Read one known reddit.com post and its rendered comment tree on current Reddit. It makes up to two fresh managed residential browser attempts, then up to two managed-browser backup attempts that can recover the post and 25 comments, stopping on the first accepted result. Use reddit_trending when no thread URL is known. Costs one Reddit base lookup plus the per-comment rate for comments returned; pass maxComments to bound both output and cost.",
     inputSchema: RedditThreadInputSchema,
     outputSchema: recordOutputSchema("reddit_thread", RedditThreadOutputSchema),
     annotations: liveWebToolAnnotations("Reddit Thread + Comments")
   }, async (input) => formatRedditThread(await executor.redditThread(input), input));
   server.registerTool("reddit_trending", {
     title: "Reddit Trending",
-    description: "Discover Reddit conversations through a site-restricted web search for the last week, month, or all time. With includeComments:true, each candidate is opened on current Reddit through a managed residential browser, with a bounded backup of up to 25 comments after primary failure, and ranked from rendered engagement; discovery, each hydrated thread, and returned comments are billed separately. Set includeComments:false for the faster discovery-only result, and inspect resultQuality and discoverySource before treating an empty result as meaningful.",
+    description: "Discover Reddit conversations through a site-restricted web search for the last week, month, or all time. With includeComments:true, each candidate is hydrated with up to two fresh managed residential browser attempts followed by up to two managed-browser backup attempts capped at 25 comments, stopping on the first accepted result, and ranked from rendered engagement; discovery, each hydrated thread, and returned comments are billed separately. Set includeComments:false for the faster discovery-only result, and inspect resultQuality and discoverySource before treating an empty result as meaningful.",
     inputSchema: RedditTrendingInputSchema,
     outputSchema: recordOutputSchema("reddit_trending", RedditTrendingOutputSchema),
     annotations: liveWebToolAnnotations("Reddit Trending")

package/dist/{chunk-7523VOQT.js → chunk-3JRZZVWL.js}

@@ -1,6 +1,6 @@
 // release-message.json
 var release_message_default = {
-  message: "Keep X-Ray measurement, seven-model attribution, structured surveys, privacy controls, and truthful evidence states while withdrawing provider setup that is not yet production-proven."
+  message: "Reddit thread hydration now retries two fresh residential sessions and two managed-browser backups, stopping at the first accepted result while preserving attempt and provider-cost telemetry."
 };
 
 // src/install-terminal.ts

package/dist/{chunk-36EAKRR6.js → chunk-C4C4WUTV.js}

@@ -1,5 +1,5 @@
 // src/version.ts
-var PACKAGE_VERSION = "0.86.0";
+var PACKAGE_VERSION = "0.86.1";
 
 export {
   PACKAGE_VERSION

package/dist/{chunk-QKWNEMGH.js → chunk-TYLVUAK3.js}

@@ -12,13 +12,13 @@ import {
   registerScheduledResultsMcpTools,
   registerSerpIntelligenceCaptureTools,
   resolveDeploymentProfile
-} from "./chunk-FBB6DEUV.js";
+} from "./chunk-3H7QTJLV.js";
 import {
   renderInstallTerminal
-} from "./chunk-7523VOQT.js";
+} from "./chunk-3JRZZVWL.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-36EAKRR6.js";
+} from "./chunk-C4C4WUTV.js";
 
 // src/mcp/stdio-runtime.ts
 import { readFileSync } from "fs";

package/dist/{server-OZBTZIDZ.js → server-UQ33P5V5.js}

@@ -388,7 +388,7 @@ import {
   transcribeMediaUrl,
   transitionAnalyticsExperiment,
   validateAnalyticsRestResponseForRequest
-} from "./chunk-FBB6DEUV.js";
+} from "./chunk-3H7QTJLV.js";
 import {
   auditImageUrls,
   auditImages,
@@ -509,7 +509,7 @@ import {
 } from "./chunk-GGZEC22A.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-36EAKRR6.js";
+} from "./chunk-C4C4WUTV.js";
 import {
   CaptchaError,
   MapsSearchIncompletePageError,
@@ -21499,25 +21499,36 @@ async function recordPendingBrightDataRedditUsage(providerSessionId, telemetry)
 async function refreshFinalizedRedditProbeCosts() {
   const db = getDb();
   const probes = await db.execute({
-    sql: `SELECT v.probe_run_id, SUM(v.est_cost_usd) AS cost
+    sql: `SELECT v.probe_run_id,
+                 SUM(CASE WHEN v.method = ? THEN v.est_cost_usd ELSE 0 END) AS cost,
+                 SUM(CASE WHEN v.method = ? THEN 1 ELSE 0 END) AS pending_count
           FROM vendor_usage_log v JOIN cost_probe_runs p ON p.id = v.probe_run_id
-          WHERE v.method = ? AND v.probe_run_id IS NOT NULL
+          WHERE v.vendor = 'brightdata_browser_api' AND v.method IN (?, ?) AND v.probe_run_id IS NOT NULL
           GROUP BY v.probe_run_id
-          HAVING ABS(SUM(v.est_cost_usd) - MAX(p.vendor_cost_usd)) > 0.000000000001
-             OR MAX(CASE WHEN json_valid(p.notes) THEN json_extract(p.notes, '$.providerCostPending') ELSE 0 END) = 1
+          HAVING ABS(SUM(CASE WHEN v.method = ? THEN v.est_cost_usd ELSE 0 END) - MAX(p.vendor_cost_usd)) > 0.000000000001
+             OR MAX(CASE WHEN json_valid(p.notes) THEN COALESCE(json_extract(p.notes, '$.providerCostPending'), 0) ELSE 0 END)
+                != CASE WHEN SUM(CASE WHEN v.method = ? THEN 1 ELSE 0 END) > 0 THEN 1 ELSE 0 END
           LIMIT 100`,
-    args: [REDDIT_COMPLETE_METHOD]
+    args: [
+      REDDIT_COMPLETE_METHOD,
+      REDDIT_PENDING_METHOD,
+      REDDIT_COMPLETE_METHOD,
+      REDDIT_PENDING_METHOD,
+      REDDIT_COMPLETE_METHOD,
+      REDDIT_PENDING_METHOD
+    ]
   });
   for (const probe of probes.rows) {
     const cost = Number(probe.cost ?? 0);
+    const providerCostPending = Number(probe.pending_count ?? 0) > 0;
     await db.execute({
       sql: `UPDATE cost_probe_runs SET
               margin_usd_headless_starter = margin_usd_headless_starter - (? - vendor_cost_usd),
               margin_usd_headful_starter = margin_usd_headful_starter - (? - vendor_cost_usd),
               vendor_cost_usd = ?,
-              notes = CASE WHEN json_valid(notes) THEN json_set(notes, '$.providerCostPending', json('false')) ELSE notes END
+              notes = CASE WHEN json_valid(notes) THEN json_set(notes, '$.providerCostPending', json(?)) ELSE notes END
             WHERE id = ?`,
-      args: [cost, cost, cost, probe.probe_run_id]
+      args: [cost, cost, cost, providerCostPending ? "true" : "false", probe.probe_run_id]
     });
   }
 }
@@ -21725,39 +21736,77 @@ async function runBrightDataRedditThreadBackup(url2, requestedComments) {
 
 // src/reddit/thread-retrieval.ts
 async function retrieveRedditThread(kernelApiKey, input) {
-  let primaryFailure;
-  if (!kernelApiKey) primaryFailure = new Error("Kernel Reddit primary is not configured");
-  else {
-    try {
-      const primary = await runKernelRedditThread(kernelApiKey, input);
-      if (!isAcceptedRedditThread(primary.thread, input.maxComments)) {
-        throw new Error(`Kernel Reddit primary returned ${primary.thread.comments.length} comments below the ${redditThreadAcceptanceTarget(primary.thread, input.maxComments)} comment acceptance target`);
+  const kernelFailures = [];
+  let kernelAttempts = 0;
+  if (!kernelApiKey) {
+    kernelFailures.push("Kernel Reddit primary is not configured");
+  } else {
+    for (let attempt = 1; attempt <= 2; attempt += 1) {
+      kernelAttempts = attempt;
+      try {
+        const primary = await runKernelRedditThread(kernelApiKey, input);
+        if (!isAcceptedRedditThread(primary.thread, input.maxComments)) {
+          throw new Error(`Kernel Reddit primary returned ${primary.thread.comments.length} comments below the ${redditThreadAcceptanceTarget(primary.thread, input.maxComments)} comment acceptance target`);
+        }
+        return {
+          ...primary,
+          diagnostics: {
+            ...primary.diagnostics,
+            fallbackUsed: false,
+            retrievalProvider: "kernel",
+            kernelAttempts,
+            brightDataAttempts: 0,
+            totalAttempts: kernelAttempts,
+            kernelFailures
+          }
+        };
+      } catch (error) {
+        const message = sanitizeProviderLocalMessage(error instanceof Error ? error.message : String(error)) ?? "Kernel Reddit primary failed";
+        kernelFailures.push(message);
+        console.warn(JSON.stringify({
+          event: "reddit_kernel_primary_rejected",
+          attempt,
+          max_attempts: 2,
+          will_retry: attempt < 2,
+          error: message,
+          fallback: attempt === 2 ? "bright_data_browser_api" : null
+        }));
       }
+    }
+  }
+  const brightDataFailures = [];
+  for (let attempt = 1; attempt <= 2; attempt += 1) {
+    try {
+      const backup = await runBrightDataRedditThreadBackup(input.url, Math.min(input.maxComments, 25));
       return {
-        ...primary,
-        diagnostics: { ...primary.diagnostics, fallbackUsed: false, retrievalProvider: "kernel" }
+        ...backup,
+        diagnostics: {
+          ...backup.diagnostics,
+          retrievalProvider: "bright_data_browser_api",
+          primaryProvider: "kernel",
+          primaryFailure: kernelFailures.at(-1) ?? "Kernel Reddit primary failed",
+          kernelAttempts,
+          brightDataAttempts: attempt,
+          totalAttempts: kernelAttempts + attempt,
+          kernelFailures,
+          brightDataFailures
+        }
       };
     } catch (error) {
-      primaryFailure = error instanceof Error ? error : new Error(String(error));
+      const message = sanitizeProviderLocalMessage(error instanceof Error ? error.message : String(error)) ?? "Bright Data Reddit backup failed";
+      brightDataFailures.push(message);
+      console.warn(JSON.stringify({
+        event: "reddit_bright_data_backup_rejected",
+        attempt,
+        max_attempts: 2,
+        will_retry: attempt < 2,
+        error: message
+      }));
     }
   }
-  const primaryError = sanitizeProviderLocalMessage(primaryFailure.message) ?? "Kernel Reddit primary failed";
-  console.warn(JSON.stringify({ event: "reddit_kernel_primary_rejected", error: primaryError, fallback: "bright_data_browser_api" }));
-  try {
-    const backup = await runBrightDataRedditThreadBackup(input.url, Math.min(input.maxComments, 25));
-    return {
-      ...backup,
-      diagnostics: {
-        ...backup.diagnostics,
-        retrievalProvider: "bright_data_browser_api",
-        primaryProvider: "kernel",
-        primaryFailure: primaryError
-      }
-    };
-  } catch (backupError) {
-    const backupMessage = sanitizeProviderLocalMessage(backupError instanceof Error ? backupError.message : String(backupError)) ?? "Bright Data Reddit backup failed";
-    throw new Error(`Reddit retrieval failed: Kernel primary: ${primaryError}; Bright Data backup: ${backupMessage}`);
-  }
+  throw new Error(
+    `Reddit retrieval failed: Kernel attempts: ${kernelFailures.join(" | ")}; Bright Data attempts: ${brightDataFailures.join(" | ")}`
+  );
 }
 
 // src/api/reddit-routes.ts
@@ -21827,7 +21876,7 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
       canonicalUrl: receipt4.canonicalUrl,
       oldRedditUrl: null,
       retrievalSource: "www_reddit",
-      attempts: receipt4.diagnostics.fallbackUsed ? 2 : 1,
+      attempts: receipt4.diagnostics.totalAttempts ?? (receipt4.diagnostics.fallbackUsed ? 2 : 1),
       diagnostics: receipt4.diagnostics
     };
     const commentCount = Array.isArray(result3.comments) ? result3.comments.length : 0;
@@ -57747,11 +57796,17 @@ function parsedObject(c) {
   const parsed = ObjectSchema.safeParse(c.req.param("object"));
   return parsed.success ? parsed.data : null;
 }
+function stripRelationshipDeploymentQuery(query) {
+  const publicQuery = { ...query };
+  delete publicQuery.researchSubpath;
+  delete publicQuery.crmSubpath;
+  return publicQuery;
+}
 var researchApp = new Hono37();
 researchApp.use("*", createApiKeyAuth());
 for (const object2 of ["people", "organizations"]) {
   researchApp.get(`/${object2}`, async (c) => {
-    const query = QuerySchema.safeParse(c.req.query());
+    const query = QuerySchema.safeParse(stripRelationshipDeploymentQuery(c.req.query()));
     if (!query.success) return c.json(safeFailure2("The Research filters are invalid.", "invalid_request"), 400);
     return call(c, researchTools[object2].search, query.data);
   });
@@ -57827,7 +57882,7 @@ crmApp.get("/summary", async (c) => {
 });
 for (const object2 of ["people", "organizations"]) {
   crmApp.get(`/${object2}`, async (c) => {
-    const query = QuerySchema.safeParse(c.req.query());
+    const query = QuerySchema.safeParse(stripRelationshipDeploymentQuery(c.req.query()));
     if (!query.success) return c.json(safeFailure2("The CRM filters are invalid.", "invalid_request"), 400);
     return call(c, crmTools[object2].search, query.data);
   });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.86.0",
+  "version": "0.86.1",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",