mcp-scraper 0.3.9 → 0.3.11

package/dist/bin/mcp-scraper-cli.js

@@ -10,7 +10,7 @@ import {
 import "../chunk-LFATOGDF.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-7VNTN4Q7.js";
+} from "../chunk-RRE7WVHQ.js";
 
 // src/cli/human-cli.ts
 import { Command } from "commander";

package/dist/bin/mcp-scraper-combined-stdio-server.cjs

@@ -270,7 +270,7 @@ function browserServiceProfileSaveChanges() {
 }
 
 // src/version.ts
-var PACKAGE_VERSION = "0.3.9";
+var PACKAGE_VERSION = "0.3.11";
 
 // src/mcp/browser-agent-tool-schemas.ts
 var import_zod = require("zod");
@@ -296,10 +296,17 @@ var BrowserOpenInputSchema = {
 var BrowserProfileOnboardInputSchema = {
   email: import_zod.z.string().optional().describe("Optional account email used to derive a stable MCP Scraper profile name. This does not import cookies or browser state."),
   profile: import_zod.z.string().optional().describe("Optional saved MCP Scraper hosted browser profile name to create or load. If omitted, email is used to derive one."),
-  url: import_zod.z.string().url().optional().describe("Setup URL to open after creating the direct no-proxy browser. Defaults to https://google.com/ so the user can click Sign in."),
+  domain: import_zod.z.string().optional().describe("Authenticated site domain to connect to the profile. Defaults to chatgpt.com for AI visibility fan-out tracking."),
+  login_url: import_zod.z.string().url().optional().describe("Optional login URL for the target domain. Defaults to https://chatgpt.com/ when domain is omitted."),
+  url: import_zod.z.string().url().optional().describe("Deprecated alias for login_url. Use login_url for the target site login page."),
   label: import_zod.z.string().optional().describe("Optional human label for this setup session."),
   timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the direct no-proxy setup session may live before auto-termination. Defaults to 600.")
 };
+var BrowserProfileStatusInputSchema = {
+  connection_id: import_zod.z.string().optional().describe("Auth connection id returned by browser_profile_onboard. Prefer this when available."),
+  profile: import_zod.z.string().optional().describe("Saved hosted profile name to check when connection_id is unavailable."),
+  domain: import_zod.z.string().optional().describe("Authenticated site domain to check with profile. Defaults to chatgpt.com when omitted.")
+};
 var BrowserSessionInputSchema = {
   session_id: import_zod.z.string().describe("The session id returned by browser_open or browser_list_sessions. Use only a returned session_id; do not construct one yourself.")
 };
@@ -400,7 +407,7 @@ var BrowserCaptureFanoutInputSchema = {
   wait_ms: import_zod.z.number().int().min(0).max(18e4).optional().describe("How long to wait for the answer stream to finish before parsing. Defaults to 90000 when a prompt is sent, 8000 for passive capture. Capture ends as soon as the answer stream finishes."),
   first_party_domain: import_zod.z.string().optional().describe("The brand/site being researched, for example example.com. Sources on this domain are tagged First-party/vendor. Set this when the prompt is about a specific business."),
   reset: import_zod.z.boolean().default(false).describe("Clear any previously buffered stream for this page before capturing. Use when re-running to avoid mixing turns."),
-  export: import_zod.z.boolean().default(false).describe("When true, write JSON, CSV, TSV exports (queries, citations, sources, domains) and a self-contained HTML report to MCP_SCRAPER_OUTPUT_DIR/fanout and return their paths.")
+  export: import_zod.z.boolean().default(false).describe("When true, write JSON, CSV, TSV exports (queries, citations, all sources, browsed-only sources, snippets, domains) and a self-contained HTML report to MCP_SCRAPER_OUTPUT_DIR/fanout, returning paths relative to MCP_SCRAPER_OUTPUT_DIR.")
 };
 var FanoutSourceOutput = import_zod.z.object({
   url: import_zod.z.string(),
@@ -442,15 +449,18 @@ var BrowserCaptureFanoutOutputSchema = {
   }).describe("Objective aggregates: top sourced sites by frequency, citation order, and URL-category counts."),
   first_party_domain: import_zod.z.string().nullable(),
   exports: import_zod.z.object({
+    relativeTo: import_zod.z.string(),
     dir: import_zod.z.string(),
     json: import_zod.z.string(),
     queriesCsv: import_zod.z.string(),
     queriesTsv: import_zod.z.string(),
     citationsCsv: import_zod.z.string(),
     sourcesCsv: import_zod.z.string(),
+    browsedOnlyCsv: import_zod.z.string(),
+    snippetsCsv: import_zod.z.string(),
     domainsCsv: import_zod.z.string(),
     report: import_zod.z.string()
-  }).nullable().describe("Local file paths when export=true, otherwise null."),
+  }).nullable().describe("Relative export paths when export=true, otherwise null. Paths are relative to MCP_SCRAPER_OUTPUT_DIR, or ~/Downloads/mcp-scraper when that env var is not set."),
   debug: import_zod.z.object({
     interceptorReady: import_zod.z.boolean(),
     rawBytes: import_zod.z.number().int().min(0),
@@ -471,15 +481,36 @@ var BrowserOpenOutputSchema = {
 var BrowserProfileOnboardOutputSchema = {
   ok: import_zod.z.boolean(),
   tool: import_zod.z.literal("browser_profile_onboard"),
-  session_id: import_zod.z.string(),
+  session_id: NullableString,
+  auth_connection_id: import_zod.z.string(),
   watch_url: import_zod.z.string(),
   live_view_url: NullableString,
   profile: import_zod.z.string(),
+  domain: import_zod.z.string(),
   setup_url: import_zod.z.string(),
   account_email: NullableString,
+  status: import_zod.z.string(),
+  flow_status: NullableString,
+  flow_step: NullableString,
+  flow_expires_at: NullableString,
+  post_login_url: NullableString,
   next_steps: import_zod.z.array(import_zod.z.string()),
   raw: BrowserRawObject.optional()
 };
+var BrowserProfileStatusOutputSchema = {
+  ok: import_zod.z.boolean(),
+  tool: import_zod.z.literal("browser_profile_status"),
+  session_id: NullableString,
+  auth_connection_id: import_zod.z.string(),
+  profile: import_zod.z.string(),
+  domain: import_zod.z.string(),
+  status: import_zod.z.string(),
+  flow_status: NullableString,
+  flow_step: NullableString,
+  flow_expires_at: NullableString,
+  post_login_url: NullableString,
+  raw: BrowserRawObject.optional()
+};
 var BrowserScreenshotOutputSchema = {
   ...BrowserBaseOutput,
   tool: import_zod.z.literal("browser_screenshot"),
@@ -886,10 +917,16 @@ function replayFilePath(sessionId, replayId, filename) {
   const name = requested ? safeFilePart(requested).replace(/\.mp4$/i, "") : `${stamp}-${safeFilePart(sessionId)}-${safeFilePart(replayId)}`;
   return (0, import_node_path2.join)(outputBaseDir(), "browser-replays", `${name}.mp4`);
 }
-function savedProfileNameFromEmail(email) {
-  const value = email?.trim().toLowerCase();
+function slugPart(value) {
+  const trimmed = value?.trim().toLowerCase();
+  if (!trimmed) return null;
+  return trimmed.replace(/^https?:\/\//, "").replace(/^www\./, "").replace(/\/.*$/, "").replace(/[^a-z0-9._-]+/g, "-").replace(/[._-]+/g, "-").replace(/^-+|-+$/g, "");
+}
+function savedProfileNameFromEmail(email, domain) {
+  const value = slugPart(email);
   if (!value) return null;
-  return value.replace(/[^a-z0-9._-]+/g, "-").replace(/[._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || null;
+  const domainPart = slugPart(domain);
+  return `${value}${domainPart ? `-${domainPart}` : ""}`.slice(0, 80) || null;
 }
 function annotatedReplayFilePath(sessionId, replayId, filename) {
   const requested = filename?.trim();
@@ -975,45 +1012,83 @@ function registerBrowserAgentMcpTools(server2, opts) {
     "browser_profile_onboard",
     {
       title: "Connect Hosted Browser Profile",
-      description: "Create or load a saved MCP Scraper hosted browser profile and open a direct no-proxy setup browser with profile saving enabled. This is the customer auth setup and reconnect flow: open Google, give the user the watch_url, have them click Sign in and complete login, then call browser_close to save cookies and storage. If a saved profile later shows login controls, run this same setup flow again.",
+      description: "Create or load a saved MCP Scraper hosted browser profile auth connection and return a user takeover login URL. This is the customer auth setup and reconnect flow for AI visibility tracking: give the user the watch_url, let them complete login for the target domain, then call browser_profile_status until status is AUTHENTICATED. If a saved profile later shows login controls, run this same setup flow again.",
       inputSchema: BrowserProfileOnboardInputSchema,
       outputSchema: BrowserProfileOnboardOutputSchema,
       annotations: annotations("Connect Hosted Browser Profile")
     },
     async (input) => {
-      const profile = input.profile?.trim() || savedProfileNameFromEmail(input.email) || browserServiceProfileName();
+      const domain = slugPart(input.domain) || "chatgpt.com";
+      const setupUrl = input.login_url ?? input.url ?? (domain === "chatgpt.com" ? "https://chatgpt.com/" : `https://${domain}/`);
+      const profile = input.profile?.trim() || savedProfileNameFromEmail(input.email, domain) || browserServiceProfileName();
       if (!profile) {
         return errorResult("browser_profile_onboard", {
           error: "profile or email is required when BROWSER_AGENT_PROFILE_NAME is not available"
         });
       }
-      const setupUrl = input.url ?? "https://google.com/";
-      const open = await req("POST", "/agent/sessions", {
+      const open = await req("POST", "/agent/profiles/onboard", {
         label: input.label ?? `Profile setup: ${profile}`,
         profile,
-        save_profile_changes: true,
-        disable_default_proxy: true,
-        timeout_seconds: input.timeout_seconds,
-        url: setupUrl
+        domain,
+        login_url: setupUrl
       });
       if (!open.ok) return errorResult("browser_profile_onboard", open.data);
-      const session = open.data;
       return structuredResult({
         ok: true,
         tool: "browser_profile_onboard",
-        session_id: session.session_id,
-        watch_url: `${consoleBase}/console/${session.session_id}`,
-        live_view_url: session.live_view_url ?? null,
+        session_id: null,
+        auth_connection_id: open.data.connection_id,
+        watch_url: open.data.hosted_url,
+        live_view_url: open.data.live_view_url ?? null,
         profile,
+        domain,
         setup_url: setupUrl,
         account_email: input.email?.trim() || null,
+        status: open.data.status ?? null,
+        flow_status: open.data.flow_status ?? null,
+        flow_step: open.data.flow_step ?? null,
+        flow_expires_at: open.data.flow_expires_at ?? null,
+        post_login_url: open.data.post_login_url ?? null,
         next_steps: [
-          "Open the watch_url and let the user click Sign in on Google.",
-          "After the user confirms login is complete, call browser_close with this session_id to persist cookies and storage to the saved profile.",
-          "If the saved profile later shows login controls or disconnects, repeat browser_profile_onboard and close the setup session again.",
-          "Use BROWSER_AGENT_PROFILE_NAME or browser_open.profile with this profile name for future authenticated sessions."
+          "Open the watch_url and let the user complete login for the target domain.",
+          "Call browser_profile_status with auth_connection_id until status is AUTHENTICATED.",
+          "After authentication, call browser_open with this profile name and the target site URL.",
+          "If the saved profile later shows login controls or disconnects, repeat browser_profile_onboard."
         ],
-        raw: session
+        raw: open.data
+      });
+    }
+  );
+  server2.registerTool(
+    "browser_profile_status",
+    {
+      title: "Check Browser Profile Auth Status",
+      description: "Check whether a hosted browser profile auth connection is authenticated after browser_profile_onboard. Use auth_connection_id when available. Once status is AUTHENTICATED, use browser_open with the returned profile.",
+      inputSchema: BrowserProfileStatusInputSchema,
+      outputSchema: BrowserProfileStatusOutputSchema,
+      annotations: annotations("Check Browser Profile Auth Status", true)
+    },
+    async (input) => {
+      const domain = slugPart(input.domain) || "chatgpt.com";
+      const res = await req("POST", "/agent/profiles/status", {
+        ...input.connection_id ? { connection_id: input.connection_id } : {},
+        ...input.profile ? { profile: input.profile } : {},
+        domain
+      });
+      if (!res.ok) return errorResult("browser_profile_status", res.data);
+      return structuredResult({
+        ok: true,
+        tool: "browser_profile_status",
+        session_id: null,
+        auth_connection_id: res.data.connection_id,
+        profile: res.data.profile,
+        domain: res.data.domain,
+        status: res.data.status,
+        flow_status: res.data.flow_status ?? null,
+        flow_step: res.data.flow_step ?? null,
+        flow_expires_at: res.data.flow_expires_at ?? null,
+        post_login_url: res.data.post_login_url ?? null,
+        raw: res.data
       });
     }
   );
@@ -1021,7 +1096,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
     "browser_open",
     {
       title: "Open Browser Session",
-      description: "Open a direct no-proxy hosted browser session you can drive. Pass a saved profile name to load an authenticated session for a site. For customer login setup or reconnects, use browser_profile_onboard first so the user signs in through a watch_url, then call browser_close to save the profile. Returns a session_id used by all other browser_* tools.",
+      description: "Open a direct no-proxy hosted browser session you can drive. Pass a saved profile name to load an authenticated session for a site. For customer login setup or reconnects, use browser_profile_onboard first, then call browser_profile_status until status is AUTHENTICATED. Returns a session_id used by all other browser_* tools.",
       inputSchema: BrowserOpenInputSchema,
       outputSchema: BrowserOpenOutputSchema,
       annotations: annotations("Open Browser Session")
@@ -1443,7 +1518,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
     "browser_capture_fanout",
     {
       title: "Capture AI Search Fan-Out",
-      description: `Capture the query fan-out behind a ChatGPT or Claude web-search answer for Answer Engine Optimization (AEO): the exact sub-queries the model issued, every researched URL split into cited vs browsed-only (with citation frequency and snippet), each source tagged by category (First-party/vendor, News/media, Reddit, Social/video, Encyclopedia, Review site, Docs, Blog), plus top sourced sites and citation order. This returns the raw structured data; YOU analyze it \u2014 classify each sub-query by funnel stage (Problem-aware, Solution-aware, Decision-aware, Retention) and type (BoFu, Branded, Comparison, How-to, Search operator), name the brands the model researched, and give AEO insights. Sample asks: "capture the fan-out for this ChatGPT answer", "what did Claude search and cite for best CRM", "show the sub-queries and sources behind this AI answer". WRITE NOTE: passing prompt submits a real message in the user's logged-in account (a new conversation turn) \u2014 only send when the user wants that; omit prompt to capture a prompt the user just ran. Setup, if not already connected or if the profile disconnected: call browser_profile_onboard, give the user the watch_url, let them click Sign in on Google, then call browser_close after they confirm login. After that, browser_open the saved direct no-proxy profile and go to chatgpt.com or claude.ai. Fan-out is captured only as it streams, so the session must be open when the prompt runs. NOT for Google AI Overview citations \u2014 use harvest_paa for those; this tool is ChatGPT and Claude only.`,
+      description: `Capture the query fan-out behind a ChatGPT or Claude web-search answer for Answer Engine Optimization (AEO): the exact sub-queries the model issued, every researched URL split into cited vs browsed-only (with citation frequency and snippet), each source tagged by category (First-party/vendor, News/media, Reddit, Social/video, Encyclopedia, Review site, Docs, Blog), plus top sourced sites and citation order. Set export=true for durable JSON, CSV, TSV, and HTML artifacts; export paths are returned relative to MCP_SCRAPER_OUTPUT_DIR. This returns the raw structured data; YOU analyze it \u2014 classify each sub-query by funnel stage (Problem-aware, Solution-aware, Decision-aware, Retention) and type (BoFu, Branded, Comparison, How-to, Search operator), name the brands the model researched, and give AEO insights. Sample asks: "capture the fan-out for this ChatGPT answer", "what did Claude search and cite for best CRM", "show the sub-queries and sources behind this AI answer". WRITE NOTE: passing prompt submits a real message in the user's logged-in account (a new conversation turn) \u2014 only send when the user wants that; omit prompt to capture a prompt the user just ran. Setup, if not already connected or if the profile disconnected: call browser_profile_onboard, give the user the watch_url, let them complete login, then call browser_profile_status until status is AUTHENTICATED. After that, browser_open the saved direct no-proxy profile and go to chatgpt.com or claude.ai. Fan-out is captured only as it streams, so the session must be open when the prompt runs. NOT for Google AI Overview citations \u2014 use harvest_paa for those; this tool is ChatGPT and Claude only.`,
       inputSchema: BrowserCaptureFanoutInputSchema,
       outputSchema: BrowserCaptureFanoutOutputSchema,
       annotations: annotations("Capture AI Search Fan-Out")
@@ -1472,7 +1547,8 @@ function registerBrowserAgentMcpTools(server2, opts) {
         prompt: input.prompt,
         wait_ms: input.wait_ms,
         first_party_domain: input.first_party_domain,
-        export: false
+        reset: input.reset,
+        export: input.export
       });
       if (!res.ok) return errorResult("browser_capture_fanout", res.data, input.session_id);
       const hosted = res.data?.result ?? res.data;
@@ -4607,14 +4683,14 @@ function renderInstallTerminal(options) {
     colorize(ascii, "amber", color),
     `${colorize("MCP Scraper Agent", "cyan", color)}  .  v${options.version}  .  mcpscraper.dev`,
     "1/1 install surfaces ready",
-    colorize("Newest: saved hosted browser profiles for AI visibility. Open Google, share the watch_url, user signs in, close to save. Browser sessions are direct/no-proxy by default.", "lime", color),
+    colorize("Newest: saved hosted browser profiles for AI visibility. Share the watch_url, user signs in, then verify with browser_profile_status. Browser sessions are direct/no-proxy by default.", "lime", color),
     "",
-    `${colorize("Tools", "cyan", color)}  ${colorize("(43 MCP tools)", "muted", color)}`,
+    `${colorize("Tools", "cyan", color)}  ${colorize("(44 MCP tools)", "muted", color)}`,
     toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
     toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "directory_workflow"], color),
     toolRow("build", ["rank_tracker_blueprint", "cron plan", "database prompt"], color),
     toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download"], color),
-    toolRow("browser", ["browser_open", "browser_profile_onboard", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
+    toolRow("browser", ["browser_open", "browser_profile_onboard", "browser_profile_status", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
     toolRow("account", ["credits_info", "reports", "MCP resources"], color),
     `${colorize("Workflows", "cyan", color)}  ${colorize("(MCP + CLI + API)", "muted", color)}`,
     toolRow("route", ["workflow_list", "workflow_suggest", "workflow_run", "workflow_step", "workflow_status", "workflow_artifact_read"], color),
@@ -4624,7 +4700,7 @@ function renderInstallTerminal(options) {
     "Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.",
     "Run mcp-scraper-combined in a human terminal to print this card; MCP clients get the same command as a silent stdio server.",
     "Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install",
-    "Customer auth setup: run browser_profile_onboard at https://google.com/, send the watch_url, let the user click Sign in, then browser_close to save.",
+    "Customer auth setup: run browser_profile_onboard, send the watch_url, let the user sign in, then call browser_profile_status until AUTHENTICATED.",
     "Reconnect setup: if a saved profile shows login controls later, run browser_profile_onboard again with the same profile name.",
     "Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.",
     "For MCP clients, use mcp-scraper-combined so one install can mix SERP, Maps, browser, reports, and saved MCP resources.",
@@ -4636,8 +4712,8 @@ function renderInstallTerminal(options) {
     "npx -y -p mcp-scraper@latest mcp-scraper-cli doctor",
     "",
     colorize("Hosted profile setup", "amber", color),
-    'In your MCP client, call browser_profile_onboard with profile="seo-example-com" and url="https://google.com/".',
-    "Give the returned watch_url to the user. After they sign in, call browser_close to persist the saved hosted profile.",
+    'In your MCP client, call browser_profile_onboard with email="seo@example.com" and domain="chatgpt.com".',
+    "Give the returned watch_url to the user. After they sign in, call browser_profile_status, then browser_open with the returned profile.",
     "",
     colorize("Claude Code one-command setup", "amber", color),
     claudeCommand,