mcp-scraper 0.3.11 → 0.3.13

package/README.md

@@ -186,8 +186,8 @@ env = { MCP_SCRAPER_API_KEY = "sk_live_your_key" }
 - `facebook_page_intel`
 - `facebook_ad_transcribe` — transcribe a direct Facebook ad video URL returned by `facebook_page_intel`.
 - `facebook_video_transcribe` — transcribe an organic Facebook reel, video, watch, post, or share URL, including `fb.watch` links. The tool renders the page, extracts the best matching public Facebook CDN MP4 URL, then returns transcript text, timestamped chunks, selected quality, video metadata, and the extracted MP4 URL for follow-up download.
-- `instagram_profile_content` — discover Instagram profile grid content links for a handle or profile URL. Returns collected post/reel/tv URLs, profile counts, type counts, shortcodes, browser details, pagination attempts, stop reason, and limitations.
-- `instagram_media_download` — extract and download one Instagram post/reel/tv URL. Returns text/caption, image URL/downloads, selected video/audio MP4 tracks, optional muxed MP4 when `ffmpeg` is available, optional transcript, and browser details.
+- `instagram_profile_content` — discover Instagram profile grid content links for a handle or profile URL, optionally through a saved hosted browser `profile` for authenticated access. Returns collected post/reel/tv URLs, profile counts, type counts, shortcodes, browser details, pagination attempts, stop reason, and limitations.
+- `instagram_media_download` — extract and download one Instagram post/reel/tv URL, optionally through a saved hosted browser `profile` for authenticated access. Returns text/caption, image URL/downloads, selected video/audio MP4 tracks, optional muxed MP4 when `ffmpeg` is available, optional transcript, and browser details.
 - `maps_search` — search Google Maps for multiple business/profile candidates. Use for GMB/GBP prospect lists, competitors, categories, and anything needing more than the Google 3-pack. In default `proxyMode: "location"`, retryable failures rotate to a new residential proxy and new browser session for up to 5 attempts. `maxResults` defaults to 10 and is capped at 50.
 - `maps_place_intel` — hydrate one known/named Google Maps business with profile details and optional reviews. Use after `maps_search` when a selected candidate needs full details.
 - `directory_workflow` — build city-by-city directory/prospecting datasets from Census place selection plus Google Maps searches. Use it for requests like "all cities over 100k population in Tennessee, then get 20 roofers from Maps." In default `proxyMode: "location"`, each city search rotates retryable failures to a new residential proxy and new browser session for up to 5 attempts. The saved CSV includes `source_location`, `result_position`, `business_name`, `review_stars`, `review_count`, `category`, `address`, `phone`, `hours_status`, `website_url`, `directions_url`, `place_url`, `cid`, `cid_decimal`, Census population, and ZIP groups.

package/dist/bin/api-server.cjs

@@ -9164,6 +9164,16 @@ function proxyIdSuffix(proxyId) {
 function errorText(err) {
   return err instanceof Error ? err.message : String(err);
 }
+function isKernelProfileConflict(err) {
+  return /\b409\b|conflict|already exists/i.test(errorText(err));
+}
+async function ensureKernelProfile(kernelClient, name) {
+  try {
+    await kernelClient.profiles.create({ name });
+  } catch (err) {
+    if (!isKernelProfileConflict(err)) throw err;
+  }
+}
 function rankCheckContextOptions(config) {
   return {
     viewport: config.viewport,
@@ -9266,10 +9276,19 @@ var init_BrowserDriver = __esm({
         if (config.kernelApiKey) {
           this.kernelClient = new import_sdk4.default({ apiKey: config.kernelApiKey });
           const timeoutSeconds = positiveIntFromEnv("KERNEL_BROWSER_TIMEOUT_SECONDS", DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS);
+          if (config.kernelProfileName && config.kernelProfileSaveChanges === true) {
+            await ensureKernelProfile(this.kernelClient, config.kernelProfileName);
+          }
           const kernelBrowser = await this.kernelClient.browsers.create({
             stealth: true,
             timeout_seconds: timeoutSeconds,
-            ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {}
+            ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {},
+            ...config.kernelProfileName ? {
+              profile: {
+                name: config.kernelProfileName,
+                ...typeof config.kernelProfileSaveChanges === "boolean" ? { save_changes: config.kernelProfileSaveChanges } : {}
+              }
+            } : {}
           });
           this.kernelSessionId = kernelBrowser.session_id;
           let defaultProxyDisabled = null;
@@ -9313,6 +9332,7 @@ var init_BrowserDriver = __esm({
             timeout_seconds: timeoutSeconds,
             proxy_mode: proxyMode,
             proxy_id_present: Boolean(config.kernelProxyId),
+            profile_name_present: Boolean(config.kernelProfileName),
             proxy_resolution_source: config.kernelProxyResolution?.source
           }));
           if (this.debugEnabled) {
@@ -13121,8 +13141,8 @@ function ownerFromMetadata(metadata) {
   return match?.[1]?.trim() || null;
 }
 async function collectInstagramProfileContentFromPage(page, opts) {
-  const maxItems = Math.min(500, Math.max(1, opts.maxItems ?? 50));
-  const maxScrolls = Math.min(100, Math.max(0, opts.maxScrolls ?? 10));
+  const maxItems = Math.min(2e3, Math.max(1, opts.maxItems ?? 50));
+  const maxScrolls = Math.min(250, Math.max(0, opts.maxScrolls ?? 10));
   const scrollDelayMs = Math.min(5e3, Math.max(250, opts.scrollDelayMs ?? 1200));
   const stableScrollLimit = Math.min(10, Math.max(1, opts.stableScrollLimit ?? 4));
   const seen = /* @__PURE__ */ new Set();
@@ -13343,21 +13363,32 @@ var init_InstagramContentExtractor = __esm({
 function invalidRequest2(message) {
   return { error_code: "invalid_request", message };
 }
-async function kernelLaunchOptsDirect() {
+function resolveSaveProfileChanges(body) {
+  return body.saveProfileChanges ?? body.save_profile_changes ?? browserServiceProfileSaveChanges();
+}
+function resolveProfileName(body) {
+  const explicit = body.profile?.trim();
+  return explicit || browserServiceProfileName();
+}
+async function kernelLaunchOptsDirect(profileName, saveProfileChanges) {
   return {
     headless: true,
     kernelApiKey: browserServiceApiKey(),
+    ...profileName ? { kernelProfileName: profileName } : {},
+    ...typeof saveProfileChanges === "boolean" ? { kernelProfileSaveChanges: saveProfileChanges } : {},
     viewport: { width: 1280, height: 900 },
     locale: "en-US"
   };
 }
-async function resolveInstagramLaunch() {
+async function resolveInstagramLaunch(body) {
+  const profileName = resolveProfileName(body);
+  const saveProfileChanges = resolveSaveProfileChanges(body);
   return {
-    config: await kernelLaunchOptsDirect(),
+    config: await kernelLaunchOptsDirect(profileName, saveProfileChanges),
     browser: {
       mode: "hosted",
       requestedMode: "hosted",
-      profileName: null,
+      profileName: profileName ?? null,
       profileSource: "hosted",
       profileDirConfigured: false,
       executablePathConfigured: false
@@ -13449,8 +13480,11 @@ var init_instagram_routes = __esm({
     InstagramProfileContentBodySchema = import_zod17.z.object({
       handle: import_zod17.z.string().trim().optional(),
       url: import_zod17.z.string().trim().optional(),
-      maxItems: import_zod17.z.number().int().min(1).max(500).default(50),
-      maxScrolls: import_zod17.z.number().int().min(0).max(100).default(10),
+      profile: import_zod17.z.string().trim().min(1).optional(),
+      saveProfileChanges: import_zod17.z.boolean().optional(),
+      save_profile_changes: import_zod17.z.boolean().optional(),
+      maxItems: import_zod17.z.number().int().min(1).max(2e3).default(50),
+      maxScrolls: import_zod17.z.number().int().min(0).max(250).default(10),
       scrollDelayMs: import_zod17.z.number().int().min(250).max(5e3).default(1200),
       stableScrollLimit: import_zod17.z.number().int().min(1).max(10).default(4)
     }).refine((d) => !!d.handle || !!d.url, {
@@ -13459,6 +13493,9 @@ var init_instagram_routes = __esm({
     InstagramMediaTypeSchema = import_zod17.z.enum(["image", "video", "audio"]);
     InstagramMediaDownloadBodySchema = import_zod17.z.object({
       url: import_zod17.z.string().trim().min(1, "url is required"),
+      profile: import_zod17.z.string().trim().min(1).optional(),
+      saveProfileChanges: import_zod17.z.boolean().optional(),
+      save_profile_changes: import_zod17.z.boolean().optional(),
       mediaTypes: import_zod17.z.array(InstagramMediaTypeSchema).default(["image", "video", "audio"]),
       downloadMedia: import_zod17.z.boolean().default(true),
       downloadAllTracks: import_zod17.z.boolean().default(false),
@@ -13489,7 +13526,7 @@ var init_instagram_routes = __esm({
         const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_profile, LedgerOperation.INSTAGRAM_PROFILE, target.profileUrl);
         if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_profile), 402);
         debited = true;
-        const launch = await resolveInstagramLaunch();
+        const launch = await resolveInstagramLaunch(body);
         await driver.launch(launch.config);
         await driver.navigateTo(target.profileUrl);
         const page = driver.getPage();
@@ -13536,7 +13573,7 @@ var init_instagram_routes = __esm({
         const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA, sourceUrl.href);
         if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_media), 402);
         mediaDebited = true;
-        const launch = await resolveInstagramLaunch();
+        const launch = await resolveInstagramLaunch(body);
         await driver.launch(launch.config);
         const page = driver.getPage();
         const capturedMediaUrls = [];
@@ -16133,7 +16170,7 @@ function formatInstagramProfileContent(raw, input) {
   const itemRows = items.slice(0, 100).map(
     (item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
   ).join("\n");
-  const browserLabel = "hosted browser";
+  const browserLabel = browser.profileName ? `hosted browser profile ${browser.profileName}` : "hosted browser";
   const full = [
     `# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
     `**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
@@ -16208,7 +16245,7 @@ function formatInstagramMediaDownload(raw, input) {
   const transcriptText = transcript?.text ?? "";
   const chunks = transcript?.chunks ?? [];
   const browser = structuredInstagramBrowser(d.browser);
-  const browserLabel = "hosted browser";
+  const browserLabel = browser.profileName ? `hosted browser profile ${browser.profileName}` : "hosted browser";
   const downloadRows = downloads.map((download, i) => {
     const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
     return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
@@ -21921,7 +21958,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.3.11";
+    PACKAGE_VERSION = "0.3.13";
   }
 });
 
@@ -21992,13 +22029,17 @@ var init_mcp_tool_schemas = __esm({
     InstagramProfileContentInputSchema = {
       handle: import_zod27.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url."),
       url: import_zod27.z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
-      maxItems: import_zod27.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values only when the user asks for a fuller archive."),
-      maxScrolls: import_zod27.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase for long profiles when Instagram continues loading more grid links."),
+      profile: import_zod27.z.string().min(1).optional().describe("Optional saved hosted browser profile name to load authenticated Instagram access. If omitted, the server uses its configured default profile when present."),
+      saveProfileChanges: import_zod27.z.boolean().optional().describe("Whether to save browser changes back to the hosted profile. Leave unset unless intentionally updating the saved login session."),
+      maxItems: import_zod27.z.number().int().min(1).max(2e3).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 2000. Use higher values only when the user asks for a fuller archive."),
+      maxScrolls: import_zod27.z.number().int().min(0).max(250).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 250. Increase for long profiles when Instagram continues loading more grid links."),
       scrollDelayMs: import_zod27.z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
       stableScrollLimit: import_zod27.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
     };
     InstagramMediaDownloadInputSchema = {
       url: import_zod27.z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
+      profile: import_zod27.z.string().min(1).optional().describe("Optional saved hosted browser profile name to load authenticated Instagram access. If omitted, the server uses its configured default profile when present."),
+      saveProfileChanges: import_zod27.z.boolean().optional().describe("Whether to save browser changes back to the hosted profile. Leave unset unless intentionally updating the saved login session."),
       mediaTypes: import_zod27.z.array(import_zod27.z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download when downloadMedia is true. Reels commonly expose separate video-only and audio-only MP4 tracks."),
       downloadMedia: import_zod27.z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
       downloadAllTracks: import_zod27.z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
@@ -22410,8 +22451,8 @@ var init_mcp_tool_schemas = __esm({
       executablePathConfigured: import_zod27.z.boolean()
     });
     InstagramPaginationOutput = import_zod27.z.object({
-      maxItems: import_zod27.z.number().int().min(1).max(500),
-      maxScrolls: import_zod27.z.number().int().min(0).max(100),
+      maxItems: import_zod27.z.number().int().min(1).max(2e3),
+      maxScrolls: import_zod27.z.number().int().min(0).max(250),
       attemptedScrolls: import_zod27.z.number().int().min(0),
       stableScrolls: import_zod27.z.number().int().min(0),
       stableScrollLimit: import_zod27.z.number().int().min(1).max(10),
@@ -23288,11 +23329,44 @@ function youtubeVideoIdFromUrl(url) {
   }
   return null;
 }
+async function readResponseData(res) {
+  const text = await res.text();
+  if (!text.trim()) return null;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return text;
+  }
+}
+function httpErrorPayload(path6, res, data) {
+  const objectData = data && typeof data === "object" && !Array.isArray(data) ? data : null;
+  const bodyMessage = objectData ? objectData.message ?? objectData.error ?? objectData.error_code : data;
+  return {
+    ...objectData ?? { body: data },
+    error: objectData?.error ?? objectData?.error_code ?? "mcp_http_error",
+    error_type: "http",
+    retryable: res.status === 429 || res.status >= 500,
+    status: res.status,
+    statusText: res.statusText,
+    path: path6,
+    message: typeof bodyMessage === "string" && bodyMessage.trim() ? bodyMessage : `MCP Scraper HTTP ${res.status}${res.statusText ? ` ${res.statusText}` : ""} for ${path6}`
+  };
+}
+function withDefaultHostedProfile(input) {
+  const profile = input.profile?.trim() || browserServiceProfileName();
+  const saveProfileChanges = input.saveProfileChanges ?? browserServiceProfileSaveChanges();
+  return {
+    ...input,
+    ...profile ? { profile } : {},
+    ...typeof saveProfileChanges === "boolean" ? { saveProfileChanges } : {}
+  };
+}
 var HttpMcpToolExecutor;
 var init_http_mcp_tool_executor = __esm({
   "src/mcp/http-mcp-tool-executor.ts"() {
     "use strict";
     init_harvest_timeout();
+    init_browser_service_env();
     HttpMcpToolExecutor = class {
       baseUrl;
       apiKey;
@@ -23320,9 +23394,9 @@ var init_http_mcp_tool_executor = __esm({
             body: JSON.stringify(body),
             signal: AbortSignal.timeout(timeoutMs)
           });
-          const data = await res.json();
+          const data = await readResponseData(res);
           if (!res.ok) {
-            return { content: [{ type: "text", text: JSON.stringify(data) }], isError: true };
+            return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path6, res, data)) }], isError: true };
           }
           return { content: [{ type: "text", text: JSON.stringify(data) }] };
         } catch (err) {
@@ -23355,9 +23429,9 @@ var init_http_mcp_tool_executor = __esm({
             },
             signal: AbortSignal.timeout(timeoutMs)
           });
-          const data = await res.json();
+          const data = await readResponseData(res);
           if (!res.ok) {
-            return { content: [{ type: "text", text: JSON.stringify(data) }], isError: true };
+            return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path6, res, data)) }], isError: true };
           }
           return { content: [{ type: "text", text: JSON.stringify(data) }] };
         } catch (err) {
@@ -23447,10 +23521,10 @@ var init_http_mcp_tool_executor = __esm({
         return this.call("/facebook/video-transcribe", input, this.httpTimeoutOverrideMs ?? 24e4);
       }
       instagramProfileContent(input) {
-        return this.call("/instagram/profile-content", input, this.httpTimeoutOverrideMs ?? 24e4);
+        return this.call("/instagram/profile-content", withDefaultHostedProfile(input), this.httpTimeoutOverrideMs ?? 24e4);
       }
       instagramMediaDownload(input) {
-        return this.call("/instagram/media-download", input, this.httpTimeoutOverrideMs ?? 3e5);
+        return this.call("/instagram/media-download", withDefaultHostedProfile(input), this.httpTimeoutOverrideMs ?? 3e5);
       }
       mapsPlaceIntel(input) {
         return this.call("/maps/place", input);