mcp-scraper 0.3.8 → 0.3.10

package/README.md

@@ -119,7 +119,7 @@ For customer AI visibility setup, use the hosted profile connection flow. Open t
 
 Browser-agent hosted sessions are direct/no-proxy by default. They do not inherit the configured SERP/Maps proxy. Use proxies for SERP/Maps tools when those tools need location or anti-bot handling, not for the interactive login browser.
 
-`BROWSER_AGENT_PROFILE_NAME` sets the default saved hosted browser profile for future sessions. For first-time setup, use `browser_profile_onboard`, open the returned `watch_url`, complete login in the hosted browser, then call `browser_close` to persist cookies and storage into the saved profile.
+`BROWSER_AGENT_PROFILE_NAME` sets the default saved hosted browser profile for future sessions. For first-time setup, use `browser_profile_onboard`, open the returned `watch_url`, complete login, then call `browser_profile_status` until the connection is authenticated.
 
 Inside MCP clients, use `browser_profile_onboard` for customer setup and reconnects. The customer signs in through the hosted `watch_url` so they control the login session directly.
 
@@ -202,7 +202,8 @@ env = { MCP_SCRAPER_API_KEY = "sk_live_your_key" }
 ### Browser-agent tools
 
 - `browser_open` — open a direct/no-proxy hosted browser session. Pass a saved profile name to load an authenticated session. Returns a human `watch_url`.
-- `browser_profile_onboard` — create or load a saved hosted browser profile and open a direct/no-proxy setup browser at Google with profile saving enabled. The user clicks Sign in through the `watch_url`, then `browser_close` persists cookies and storage into the saved profile. Use this again whenever the profile disconnects.
+- `browser_profile_onboard` — create or load a saved hosted browser profile auth connection and return a user takeover `watch_url`. The user signs in through that URL, then `browser_profile_status` verifies the profile is authenticated. Use this again whenever the profile disconnects.
+- `browser_profile_status` — check whether a hosted browser profile auth connection is authenticated before launching it with `browser_open`.
 - `browser_screenshot` — capture a screenshot plus visible text and clickable element center coordinates and DOM bounds.
 - `browser_read` — read the current page text and elements with center coordinates and DOM bounds, without an image.
 - `browser_locate` — locate exact visible DOM elements or text ranges and return screenshot-pixel bounds.
@@ -230,7 +231,7 @@ The MCPB bundle and `mcp-scraper-combined` expose both sections through one MCP
 
 All MCP tools expose output schemas and return `structuredContent` with the IDs, URLs, CSV paths, transcripts, browser session handles, replay paths, artifacts, recipe fields, or blueprint fields needed by the next step. Browser Agent tools keep a JSON text block for older clients, but structured data is the primary contract. All tools carry MCP annotations; file-writing tools such as replay downloads and annotations state their filesystem side effects.
 
-The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The split `mcp-scraper` stdio server exposes 24 web-intelligence/workflow tools. The split `browser-agent` stdio server exposes 19 browser tools. The combined stdio server exposes 43 tools. The hosted MCP endpoint at `https://mcpscraper.dev/mcp` exposes the 24 web-intelligence/workflow tools plus `capture_serp_snapshot` and `capture_serp_page_snapshots` (26 total). Across all public surfaces there are 45 unique tool names.
+The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The split `mcp-scraper` stdio server exposes 24 web-intelligence/workflow tools. The split `browser-agent` stdio server exposes 20 browser tools. The combined stdio server exposes 44 tools. The hosted MCP endpoint at `https://mcpscraper.dev/mcp` exposes the 24 web-intelligence/workflow tools plus `capture_serp_snapshot` and `capture_serp_page_snapshots` (26 total). Across all public surfaces there are 46 unique tool names.
 
 ## Resources
 

package/dist/bin/api-server.cjs

@@ -21921,7 +21921,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.3.8";
+    PACKAGE_VERSION = "0.3.10";
   }
 });
 
@@ -24639,22 +24639,82 @@ async function ensureProfile(k, name) {
     if (!isProfileConflict(err)) throw err;
   }
 }
+function profileConnectionMatches(connection, profileName, domain) {
+  if (!connection || typeof connection !== "object") return false;
+  const data = connection;
+  return data.profile_name === profileName && data.domain === domain;
+}
+async function findProfileConnection(k, profileName, domain) {
+  for await (const connection of k.auth.connections.list({ profile_name: profileName, domain })) {
+    if (profileConnectionMatches(connection, profileName, domain)) return connection;
+  }
+  return null;
+}
+async function findOrCreateProfileConnection(k, opts) {
+  try {
+    return await k.auth.connections.create({
+      domain: opts.domain,
+      profile_name: opts.profileName,
+      ...opts.loginUrl ? { login_url: opts.loginUrl } : {}
+    });
+  } catch (err) {
+    if (!isProfileConflict(err)) throw err;
+    const existing = await findProfileConnection(k, opts.profileName, opts.domain);
+    if (existing) return existing;
+    throw err;
+  }
+}
+function profileOnboardResult(connection, hostedUrl = null, liveViewUrl = null) {
+  return {
+    connectionId: String(connection.id),
+    profileName: String(connection.profile_name),
+    domain: String(connection.domain),
+    status: String(connection.status),
+    flowStatus: typeof connection.flow_status === "string" ? connection.flow_status : null,
+    flowStep: typeof connection.flow_step === "string" ? connection.flow_step : null,
+    flowExpiresAt: typeof connection.flow_expires_at === "string" ? connection.flow_expires_at : null,
+    hostedUrl: hostedUrl ?? (typeof connection.hosted_url === "string" ? connection.hosted_url : null),
+    liveViewUrl: liveViewUrl ?? (typeof connection.live_view_url === "string" ? connection.live_view_url : null),
+    browserSessionId: typeof connection.browser_session_id === "string" ? connection.browser_session_id : null,
+    postLoginUrl: typeof connection.post_login_url === "string" ? connection.post_login_url : null
+  };
+}
 function explicitProxyId(value) {
   const trimmed = value?.trim();
   return trimmed || void 0;
 }
+async function startProfileOnboarding(opts) {
+  const k = client();
+  const connection = await findOrCreateProfileConnection(k, opts);
+  const login = await k.auth.connections.login(connection.id);
+  const state = await k.auth.connections.retrieve(connection.id);
+  return profileOnboardResult(state, login.hosted_url ?? null, login.live_view_url ?? null);
+}
+async function getProfileOnboardingStatus(input) {
+  const k = client();
+  let connection = null;
+  if (input.connectionId) {
+    connection = await k.auth.connections.retrieve(input.connectionId);
+  } else if (input.profileName && input.domain) {
+    connection = await findProfileConnection(k, input.profileName, input.domain);
+  }
+  if (!connection) throw new Error("profile auth connection not found");
+  return profileOnboardResult(connection);
+}
 async function createSession(opts = {}) {
   const k = client();
   const resolvedProxyId = explicitProxyId(opts.proxyId);
   const resolvedProfileName = opts.profileName ?? browserServiceProfileName();
   const resolvedSaveProfileChanges = opts.saveProfileChanges ?? browserServiceProfileSaveChanges();
   const disableDefaultProxy = opts.disableDefaultProxy ?? !resolvedProxyId;
+  const startUrl = opts.startUrl?.trim();
   if (resolvedProfileName && resolvedSaveProfileChanges === true) {
     await ensureProfile(k, resolvedProfileName);
   }
-  const browser = await k.browsers.create({
+  const createPayload = {
     stealth: opts.stealth ?? true,
     timeout_seconds: opts.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS,
+    ...startUrl ? { start_url: startUrl } : {},
     ...resolvedProxyId ? { proxy_id: resolvedProxyId } : {},
     ...resolvedProfileName ? {
       profile: {
@@ -24662,7 +24722,8 @@ async function createSession(opts = {}) {
         ...typeof resolvedSaveProfileChanges === "boolean" ? { save_changes: resolvedSaveProfileChanges } : {}
       }
     } : {}
-  });
+  };
+  const browser = await k.browsers.create(createPayload);
   const runtimeSessionId = browser.session_id;
   if (disableDefaultProxy) {
     try {
@@ -25042,6 +25103,56 @@ function buildBrowserAgentRoutes() {
     return next();
   });
   app2.use("*", auth);
+  app2.post("/profiles/onboard", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const profileName = typeof body.profile === "string" ? body.profile.trim() : "";
+    const domain = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : "chatgpt.com";
+    const loginUrl = typeof body.login_url === "string" ? body.login_url.trim() : void 0;
+    if (!profileName) return c.json({ error: "profile is required" }, 400);
+    if (!domain) return c.json({ error: "domain is required" }, 400);
+    try {
+      const result = await startProfileOnboarding({ profileName, domain, loginUrl });
+      return c.json({
+        connection_id: result.connectionId,
+        profile: result.profileName,
+        domain: result.domain,
+        status: result.status,
+        flow_status: result.flowStatus,
+        flow_step: result.flowStep,
+        flow_expires_at: result.flowExpiresAt,
+        hosted_url: result.hostedUrl,
+        live_view_url: result.liveViewUrl,
+        browser_session_id: result.browserSessionId,
+        post_login_url: result.postLoginUrl
+      });
+    } catch (err) {
+      return c.json(failure(err), 502);
+    }
+  });
+  app2.post("/profiles/status", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const connectionId = typeof body.connection_id === "string" ? body.connection_id.trim() : void 0;
+    const profileName = typeof body.profile === "string" ? body.profile.trim() : void 0;
+    const domain = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : void 0;
+    try {
+      const result = await getProfileOnboardingStatus({ connectionId, profileName, domain });
+      return c.json({
+        connection_id: result.connectionId,
+        profile: result.profileName,
+        domain: result.domain,
+        status: result.status,
+        flow_status: result.flowStatus,
+        flow_step: result.flowStep,
+        flow_expires_at: result.flowExpiresAt,
+        hosted_url: result.hostedUrl,
+        live_view_url: result.liveViewUrl,
+        browser_session_id: result.browserSessionId,
+        post_login_url: result.postLoginUrl
+      });
+    } catch (err) {
+      return c.json(failure(err), 404);
+    }
+  });
   app2.post("/sessions", async (c) => {
     const user = c.get("user");
     if (Number(user.balance_mc ?? 0) < BROWSER_OPEN_MIN_BALANCE_MC) {
@@ -25049,6 +25160,7 @@ function buildBrowserAgentRoutes() {
     }
     const body = await c.req.json().catch(() => ({}));
     const timeoutSeconds = typeof body.timeout_seconds === "number" ? body.timeout_seconds : void 0;
+    const startUrl = typeof body.url === "string" ? body.url : void 0;
     const gate = await acquireConcurrencyGate(user, "browser_agent_session", {
       ttlSeconds: browserSessionLockTtlSeconds(timeoutSeconds),
       metadata: { label: typeof body.label === "string" ? body.label : null }
@@ -25061,6 +25173,7 @@ function buildBrowserAgentRoutes() {
         proxyId: typeof body.proxy_id === "string" ? body.proxy_id : void 0,
         profileName: typeof body.profile === "string" ? body.profile : void 0,
         saveProfileChanges: typeof body.save_profile_changes === "boolean" ? body.save_profile_changes : void 0,
+        startUrl,
         disableDefaultProxy: typeof body.disable_default_proxy === "boolean" ? body.disable_default_proxy : void 0,
         viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0
       });