mcp-scraper 0.3.3 → 0.3.5

package/dist/bin/api-server.js

@@ -17,7 +17,7 @@ loadDotEnv();
 async function main() {
   const [{ serve }, { app }, { startWorker }, { migrate }] = await Promise.all([
     import("@hono/node-server"),
-    import("../server-D3XHIEQN.js"),
+    import("../server-BC4Z2L4P.js"),
     import("../worker-56IXWOQU.js"),
     import("../db-BE4JVB3V.js")
   ]);

package/dist/bin/browser-agent-stdio-server.cjs

@@ -1436,7 +1436,7 @@ var LocalBrowserAgentService = class {
 };
 
 // src/version.ts
-var PACKAGE_VERSION = "0.3.3";
+var PACKAGE_VERSION = "0.3.5";
 
 // src/mcp/browser-agent-tool-schemas.ts
 var import_zod = require("zod");
@@ -1455,21 +1455,21 @@ var BrowserReplayBaseOutput = {
 var BrowserOpenInputSchema = {
   label: import_zod.z.string().optional().describe("Optional human label for this session, shown in the watch console."),
   url: import_zod.z.string().url().optional().describe("Optional URL to navigate to immediately after opening."),
-  profile: import_zod.z.string().optional().describe("Optional saved profile name to load a logged-in session for a site."),
+  profile: import_zod.z.string().optional().describe("Optional saved hosted profile name to load a logged-in session for a site."),
   save_profile_changes: import_zod.z.boolean().optional().describe("Persist cookies and local storage back to the named profile when the session is closed. Use this for profile setup or intentional auth refreshes; avoid parallel sessions writing to the same profile."),
-  timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the session may live before auto-termination. Defaults to 600. The browser idles into a zero-cost standby between actions, so a longer timeout is cheap.")
+  timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the direct no-proxy browser session may live before auto-termination. Defaults to 600. The browser idles into a zero-cost standby between actions, so a longer timeout is cheap.")
 };
 var BrowserProfileListInputSchema = {
   email: import_zod.z.string().optional().describe("Optional Chrome account email or displayed profile name to filter for, for example seo@example.com."),
   user_data_dir: import_zod.z.string().optional().describe("Optional Chrome user data directory. Defaults to the current OS Chrome profile root.")
 };
 var BrowserProfileOnboardInputSchema = {
-  email: import_zod.z.string().optional().describe("Optional local Chrome account email to find and convert into a suggested Kernel profile name."),
-  profile: import_zod.z.string().optional().describe("Optional Kernel browser profile name to create/load. If omitted, email is used to derive one."),
-  user_data_dir: import_zod.z.string().optional().describe("Optional Chrome user data directory for email lookup. Defaults to the current OS Chrome profile root."),
-  url: import_zod.z.string().url().optional().describe("Setup URL to open after creating the browser. Defaults to https://accounts.google.com/."),
+  email: import_zod.z.string().optional().describe("Optional account email used to derive a stable MCP Scraper profile name. This does not import cookies or browser state."),
+  profile: import_zod.z.string().optional().describe("Optional saved MCP Scraper hosted browser profile name to create or load. If omitted, email is used to derive one."),
+  user_data_dir: import_zod.z.string().optional().describe("Optional Chrome metadata directory for deriving a profile name only. This is not a local browser setup path."),
+  url: import_zod.z.string().url().optional().describe("Setup URL to open after creating the direct no-proxy browser. Defaults to https://google.com/ so the user can click Sign in."),
   label: import_zod.z.string().optional().describe("Optional human label for this setup session."),
-  timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the setup session may live before auto-termination. Defaults to 600.")
+  timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the direct no-proxy setup session may live before auto-termination. Defaults to 600.")
 };
 var BrowserProfileImportInputSchema = {
   email: import_zod.z.string().optional().describe("Optional Chrome account email or displayed profile name to select from Chrome Local State."),
@@ -1580,7 +1580,7 @@ var BrowserListInputSchema = {
   include_closed: import_zod.z.boolean().default(false).describe("Include closed sessions in the list.")
 };
 var BrowserCaptureFanoutInputSchema = {
-  session_id: import_zod.z.string().describe("The session id returned by browser_open or browser_list_sessions. The session must be a local-mode session on chatgpt.com or claude.ai that is logged in. Use only a returned session_id; do not construct one yourself."),
+  session_id: import_zod.z.string().describe("The session id returned by browser_open or browser_list_sessions. The session must be on chatgpt.com or claude.ai and logged in through a saved hosted profile. Use only a returned session_id; do not construct one yourself."),
   prompt: import_zod.z.string().optional().describe("Optional prompt to type into the chat composer and submit before capturing. Omit to passively capture the fan-out of a prompt the user just ran in the visible browser. The prompt must trigger web search for there to be a fan-out."),
   wait_ms: import_zod.z.number().int().min(0).max(18e4).optional().describe("How long to wait for the answer stream to finish before parsing. Defaults to 90000 when a prompt is sent, 8000 for passive capture. Capture ends as soon as the answer stream finishes."),
   first_party_domain: import_zod.z.string().optional().describe("The brand/site being researched, for example example.com. Sources on this domain are tagged First-party/vendor. Set this when the prompt is about a specific business."),
@@ -2196,11 +2196,11 @@ function registerBrowserAgentMcpTools(server2, opts) {
   server2.registerTool(
     "browser_profile_list",
     {
-      title: "List Local Chrome Profiles",
-      description: "List local Chrome profile metadata visible to this MCP server: account email, Chrome profile directory, local path, and a suggested managed browser profile name. This reads Chrome Local State only; it does not read cookies, passwords, browsing history, or copy local browser state. Use browser_profile_import when the user wants MCP Scraper to clone a local Chrome profile for local browser mode.",
+      title: "Developer: List Local Chrome Profiles",
+      description: "Developer/local-machine helper only. List local Chrome profile metadata visible to this MCP server: account email, Chrome profile directory, local path, and a suggested managed browser profile name. This reads Chrome Local State only; it does not read cookies, passwords, browsing history, or copy local browser state. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
       inputSchema: BrowserProfileListInputSchema,
       outputSchema: BrowserProfileListOutputSchema,
-      annotations: annotations("List Local Chrome Profiles", true)
+      annotations: annotations("Developer: List Local Chrome Profiles", true)
     },
     async (input) => {
       try {
@@ -2211,7 +2211,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
           session_id: null,
           profiles,
           count: profiles.length,
-          hint: "Use browser_profile_import to clone one profile for local mode. Use browser_profile_onboard only for hosted Kernel profile setup."
+          hint: "For customer auth setup, use browser_profile_onboard so the user signs in through the hosted watch_url. Local import is only for developer/local-machine mode."
         });
       } catch (err) {
         return errorResult("browser_profile_list", { error: err instanceof Error ? err.message : String(err) });
@@ -2221,11 +2221,11 @@ function registerBrowserAgentMcpTools(server2, opts) {
   server2.registerTool(
     "browser_profile_import",
     {
-      title: "Import Local Chrome Profile",
-      description: "Clone one local Chrome profile into MCP Scraper managed storage for local browser mode. This copies browser state files such as cookies, history, local storage, session storage, and password database metadata into ~/.mcp-scraper/browser-profiles by default; it skips cache and lock files and never prints cookie or password values. Use this when the user wants browser_open to run locally with their existing logged-in Chrome state.",
+      title: "Developer: Import Local Chrome Profile",
+      description: "Developer/local-machine helper only. Clone one local Chrome profile into MCP Scraper managed storage for local browser mode. This copies browser state files such as cookies, history, local storage, session storage, and password database metadata into ~/.mcp-scraper/browser-profiles by default; it skips cache and lock files and never prints cookie or password values. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
       inputSchema: BrowserProfileImportInputSchema,
       outputSchema: BrowserProfileImportOutputSchema,
-      annotations: { title: "Import Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
+      annotations: { title: "Developer: Import Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
     },
     async (input) => {
       try {
@@ -2250,7 +2250,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
             "Restart the MCP client so it starts the local browser mode process.",
             "Call browser_open with the same MCP server; it will open a local Chrome window using the cloned profile."
           ],
-          hint: "The clone is local to this machine. Re-run browser_profile_sync after logging into new sites in normal Chrome."
+          hint: "Developer-only local clone. For customer auth setup, use browser_profile_onboard at Google, provide the hosted watch_url, then browser_close to save the hosted profile."
         });
       } catch (err) {
         return errorResult("browser_profile_import", { error: err instanceof Error ? err.message : String(err) });
@@ -2260,11 +2260,11 @@ function registerBrowserAgentMcpTools(server2, opts) {
   server2.registerTool(
     "browser_profile_sync",
     {
-      title: "Sync Local Chrome Profile",
-      description: "Refresh an existing MCP Scraper managed local browser profile from the source Chrome profile recorded in its manifest. This overwrites the managed clone, copies browser state files again, skips cache and lock files, and never prints cookie or password values.",
+      title: "Developer: Sync Local Chrome Profile",
+      description: "Developer/local-machine helper only. Refresh an existing MCP Scraper managed local browser profile from the source Chrome profile recorded in its manifest. This overwrites the managed clone, copies browser state files again, skips cache and lock files, and never prints cookie or password values. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
       inputSchema: BrowserProfileSyncInputSchema,
       outputSchema: BrowserProfileSyncOutputSchema,
-      annotations: { title: "Sync Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
+      annotations: { title: "Developer: Sync Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
     },
     async (input) => {
       try {
@@ -2278,7 +2278,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
           session_id: null,
           profile: result.profile,
           source_profile: result.sourceProfile,
-          hint: `Refreshed ${result.profile.name}. Restart local browser sessions that were using this profile.`
+          hint: `Developer-only local clone refreshed for ${result.profile.name}. Customer reconnects should use browser_profile_onboard with the same hosted profile name.`
         });
       } catch (err) {
         return errorResult("browser_profile_sync", { error: err instanceof Error ? err.message : String(err) });
@@ -2288,11 +2288,11 @@ function registerBrowserAgentMcpTools(server2, opts) {
   server2.registerTool(
     "browser_profile_onboard",
     {
-      title: "Onboard Hosted Browser Profile",
-      description: "Create or load a saved hosted Kernel browser profile and open a live setup browser with profile saving enabled. Use this only for hosted browser mode. If email is provided, the tool reads local Chrome profile metadata only to suggest the Kernel profile name; it does not import local cookies. For local browser mode with real Chrome state, use browser_profile_import instead.",
+      title: "Connect Hosted Browser Profile",
+      description: "Create or load a saved MCP Scraper hosted browser profile and open a direct no-proxy setup browser with profile saving enabled. This is the customer auth setup and reconnect flow: open Google, give the user the watch_url, have them click Sign in and complete login, then call browser_close to save cookies and local storage. If a saved profile later shows login controls, run this same setup flow again. Do not use local Chrome import for this customer setup.",
       inputSchema: BrowserProfileOnboardInputSchema,
       outputSchema: BrowserProfileOnboardOutputSchema,
-      annotations: annotations("Onboard Hosted Browser Profile")
+      annotations: annotations("Connect Hosted Browser Profile")
     },
     async (input) => {
       let chromeProfile = null;
@@ -2309,11 +2309,12 @@ function registerBrowserAgentMcpTools(server2, opts) {
           error: "profile is required when no email match or BROWSER_AGENT_PROFILE_NAME is available"
         });
       }
-      const setupUrl = input.url ?? "https://accounts.google.com/";
+      const setupUrl = input.url ?? "https://google.com/";
       const open = await req("POST", "/agent/sessions", {
         label: input.label ?? `Profile setup: ${profile}`,
         profile,
         save_profile_changes: true,
+        disable_default_proxy: true,
         timeout_seconds: input.timeout_seconds
       });
       if (!open.ok) return errorResult("browser_profile_onboard", open.data);
@@ -2330,8 +2331,9 @@ function registerBrowserAgentMcpTools(server2, opts) {
         setup_url: setupUrl,
         chrome_profile: chromeProfile,
         next_steps: [
-          "Open the watch_url and complete the login manually.",
-          "After login is complete, call browser_close with this session_id to persist cookies and local storage to the Kernel profile.",
+          "Open the watch_url and let the user click Sign in on Google.",
+          "After the user confirms login is complete, call browser_close with this session_id to persist cookies and local storage to the saved profile.",
+          "If the saved profile later shows login controls or disconnects, repeat browser_profile_onboard and close the setup session again.",
           "Use BROWSER_AGENT_PROFILE_NAME or browser_open.profile with this profile name for future authenticated sessions."
         ],
         raw: session
@@ -2342,7 +2344,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
     "browser_open",
     {
       title: "Open Browser Session",
-      description: "Open a browser you can drive. By default this creates a hosted cloud browser with a watch_url; when MCP_SCRAPER_BROWSER_MODE=local it opens a local Google Chrome window against an imported MCP Scraper Chrome profile. Returns a session_id used by all other browser_* tools. In hosted mode, anti-bot stealth and automatic CAPTCHA/Cloudflare solving are on by default. Call browser_close when done.",
+      description: "Open a direct no-proxy hosted browser session you can drive. Pass a saved profile name to load an authenticated session for a site. For customer login setup or reconnects, use browser_profile_onboard first so the user signs in through a watch_url, then call browser_close to save the profile. Returns a session_id used by all other browser_* tools.",
       inputSchema: BrowserOpenInputSchema,
       outputSchema: BrowserOpenOutputSchema,
       annotations: annotations("Open Browser Session")
@@ -2371,6 +2373,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
         label: input.label,
         ...profile ? { profile } : {},
         ...profile && typeof saveProfileChanges === "boolean" ? { save_profile_changes: saveProfileChanges } : {},
+        disable_default_proxy: true,
         timeout_seconds: input.timeout_seconds
       });
       if (!open.ok) return errorResult("browser_open", open.data);
@@ -2908,7 +2911,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
     "browser_capture_fanout",
     {
       title: "Capture AI Search Fan-Out",
-      description: `Capture the query fan-out behind a ChatGPT or Claude web-search answer for Answer Engine Optimization (AEO): the exact sub-queries the model issued, every researched URL split into cited vs browsed-only (with citation frequency and snippet), each source tagged by category (First-party/vendor, News/media, Reddit, Social/video, Encyclopedia, Review site, Docs, Blog), plus top sourced sites and citation order. This returns the raw structured data; YOU analyze it \u2014 classify each sub-query by funnel stage (Problem-aware, Solution-aware, Decision-aware, Retention) and type (BoFu, Branded, Comparison, How-to, Search operator), name the brands the model researched, and give AEO insights. Sample asks: "capture the fan-out for this ChatGPT answer", "what did Claude search and cite for best CRM", "show the sub-queries and sources behind this AI answer". WRITE NOTE: passing prompt submits a real message in the user's logged-in account (a new conversation turn) \u2014 only send when the user wants that; omit prompt to capture a prompt the user just ran. Setup, if not already done: import a logged-in session with browser_profile_import (local Chrome) or browser_profile_onboard (hosted Kernel), then browser_open and browser_goto to chatgpt.com or claude.ai. Fan-out is captured only as it streams, so the session must be open when the prompt runs. NOT for Google AI Overview citations \u2014 use harvest_paa for those; this tool is ChatGPT and Claude only.`,
+      description: `Capture the query fan-out behind a ChatGPT or Claude web-search answer for Answer Engine Optimization (AEO): the exact sub-queries the model issued, every researched URL split into cited vs browsed-only (with citation frequency and snippet), each source tagged by category (First-party/vendor, News/media, Reddit, Social/video, Encyclopedia, Review site, Docs, Blog), plus top sourced sites and citation order. This returns the raw structured data; YOU analyze it \u2014 classify each sub-query by funnel stage (Problem-aware, Solution-aware, Decision-aware, Retention) and type (BoFu, Branded, Comparison, How-to, Search operator), name the brands the model researched, and give AEO insights. Sample asks: "capture the fan-out for this ChatGPT answer", "what did Claude search and cite for best CRM", "show the sub-queries and sources behind this AI answer". WRITE NOTE: passing prompt submits a real message in the user's logged-in account (a new conversation turn) \u2014 only send when the user wants that; omit prompt to capture a prompt the user just ran. Setup, if not already connected or if the profile disconnected: call browser_profile_onboard, give the user the watch_url, let them click Sign in on Google, then call browser_close after they confirm login. After that, browser_open the saved direct no-proxy profile and go to chatgpt.com or claude.ai. Fan-out is captured only as it streams, so the session must be open when the prompt runs. NOT for Google AI Overview citations \u2014 use harvest_paa for those; this tool is ChatGPT and Claude only.`,
       inputSchema: BrowserCaptureFanoutInputSchema,
       outputSchema: BrowserCaptureFanoutOutputSchema,
       annotations: annotations("Capture AI Search Fan-Out")