mcp-scraper 0.3.0 → 0.3.2

package/dist/{chunk-SIXJ6EVS.js → chunk-KO223PSJ.js}

@@ -7,7 +7,7 @@ import {
 } from "./chunk-LFATOGDF.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-PGWJ2EJ3.js";
+} from "./chunk-VB65G7PS.js";
 
 // src/harvest-timeout.ts
 var VERCEL_FUNCTION_MAX_MS = 3e5;
@@ -1478,6 +1478,216 @@ ${chunkRows}` : "",
     }
   };
 }
+function structuredInstagramBrowser(raw) {
+  const browser = raw && typeof raw === "object" ? raw : {};
+  const mode = browser.mode === "local" ? "local" : "hosted";
+  const requestedMode = browser.requestedMode === "local" || browser.requestedMode === "hosted" || browser.requestedMode === "auto" ? browser.requestedMode : "auto";
+  const profileSource = browser.profileSource === "managed_profile" || browser.profileSource === "direct_profile_dir" ? browser.profileSource : "hosted";
+  return {
+    mode,
+    requestedMode,
+    profileName: typeof browser.profileName === "string" ? browser.profileName : null,
+    profileSource,
+    profileDirConfigured: browser.profileDirConfigured === true,
+    executablePathConfigured: browser.executablePathConfigured === true
+  };
+}
+function structuredInstagramPagination(raw, input) {
+  const pagination = raw && typeof raw === "object" ? raw : {};
+  const stages = Array.isArray(pagination.stages) ? pagination.stages : [];
+  const stoppedReason = typeof pagination.stoppedReason === "string" && ["max_items", "reported_post_count", "stable_scrolls", "max_scrolls", "no_scrolls"].includes(pagination.stoppedReason) ? pagination.stoppedReason : "no_scrolls";
+  return {
+    maxItems: typeof pagination.maxItems === "number" ? pagination.maxItems : Number(input.maxItems ?? 50),
+    maxScrolls: typeof pagination.maxScrolls === "number" ? pagination.maxScrolls : Number(input.maxScrolls ?? 10),
+    attemptedScrolls: typeof pagination.attemptedScrolls === "number" ? pagination.attemptedScrolls : 0,
+    stableScrolls: typeof pagination.stableScrolls === "number" ? pagination.stableScrolls : 0,
+    stableScrollLimit: typeof pagination.stableScrollLimit === "number" ? pagination.stableScrollLimit : Number(input.stableScrollLimit ?? 4),
+    scrollDelayMs: typeof pagination.scrollDelayMs === "number" ? pagination.scrollDelayMs : Number(input.scrollDelayMs ?? 1200),
+    reachedMaxItems: pagination.reachedMaxItems === true,
+    reachedReportedPostCount: pagination.reachedReportedPostCount === true,
+    finalScrollHeight: typeof pagination.finalScrollHeight === "number" ? pagination.finalScrollHeight : null,
+    stoppedReason,
+    stages: stages.map((stage) => {
+      const row = stage && typeof stage === "object" ? stage : {};
+      return {
+        stage: String(row.stage ?? ""),
+        itemCount: typeof row.itemCount === "number" ? row.itemCount : 0,
+        addedCount: typeof row.addedCount === "number" ? row.addedCount : 0,
+        scrollY: typeof row.scrollY === "number" ? row.scrollY : null,
+        scrollHeight: typeof row.scrollHeight === "number" ? row.scrollHeight : null
+      };
+    })
+  };
+}
+function formatInstagramProfileContent(raw, input) {
+  const parsed = parseData(raw);
+  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
+  const d = parsed.data;
+  const items = Array.isArray(d.items) ? d.items : [];
+  const typeCounts = d.typeCounts;
+  const limitations = Array.isArray(d.limitations) ? d.limitations.map(String) : [];
+  const browser = structuredInstagramBrowser(d.browser);
+  const pagination = structuredInstagramPagination(d.pagination, input);
+  const itemRows = items.slice(0, 100).map(
+    (item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
+  ).join("\n");
+  const browserLabel = browser.mode === "local" ? `local profile ${browser.profileName ? `\`${browser.profileName}\`` : "(unnamed)"}` : "hosted browser";
+  const full = [
+    `# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
+    `**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
+    `**Browser:** ${browserLabel}`,
+    `**Pagination:** ${pagination.attemptedScrolls} scrolls \xB7 stopped: ${pagination.stoppedReason}`,
+    d.reportedPostCountText ? `**Profile count:** ${d.reportedPostCountText}` : "",
+    d.followerCountText ? `**Followers:** ${d.followerCountText}` : "",
+    `
+## Content Links
+| # | Type | Shortcode | URL | First Seen |
+|---|------|-----------|-----|------------|
+${itemRows || "| \u2014 | \u2014 | \u2014 | \u2014 | \u2014 |"}`,
+    limitations.length ? `
+## Limits
+${limitations.map((l) => `- ${l}`).join("\n")}` : "",
+    `
+---
+\u{1F4A1} Use \`instagram_media_download\` with one of these URLs to download text, image, reel tracks, and optional transcript.`
+  ].filter(Boolean).join("\n");
+  return {
+    ...oneBlock(full),
+    structuredContent: {
+      handle: String(d.handle ?? input.handle ?? ""),
+      profileUrl: String(d.profileUrl ?? input.url ?? ""),
+      pageUrl: String(d.pageUrl ?? d.profileUrl ?? input.url ?? ""),
+      browser,
+      profileName: typeof d.profileName === "string" ? d.profileName : null,
+      reportedPostCount: typeof d.reportedPostCount === "number" ? d.reportedPostCount : null,
+      reportedPostCountText: typeof d.reportedPostCountText === "string" ? d.reportedPostCountText : null,
+      followerCountText: typeof d.followerCountText === "string" ? d.followerCountText : null,
+      followingCountText: typeof d.followingCountText === "string" ? d.followingCountText : null,
+      collectedContentCount: items.length,
+      typeCounts: {
+        post: Number(typeCounts?.post ?? 0),
+        reel: Number(typeCounts?.reel ?? 0),
+        tv: Number(typeCounts?.tv ?? 0)
+      },
+      pagination,
+      limited: d.limited === true,
+      limitations,
+      items: items.map((item) => ({
+        url: String(item.url ?? ""),
+        type: item.type,
+        shortcode: String(item.shortcode ?? ""),
+        anchorText: item.anchorText ?? null,
+        firstSeenStage: String(item.firstSeenStage ?? "")
+      }))
+    }
+  };
+}
+function structuredInstagramTrack(track) {
+  if (!track) return null;
+  return {
+    url: String(track.url ?? ""),
+    streamType: track.streamType ?? "unknown",
+    bitrate: typeof track.bitrate === "number" ? track.bitrate : null,
+    durationSec: typeof track.durationSec === "number" ? track.durationSec : null,
+    vencodeTag: track.vencodeTag ?? null,
+    width: typeof track.width === "number" ? track.width : null,
+    height: typeof track.height === "number" ? track.height : null
+  };
+}
+function formatInstagramMediaDownload(raw, input) {
+  const parsed = parseData(raw);
+  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
+  const d = parsed.data;
+  const tracks = Array.isArray(d.tracks) ? d.tracks : [];
+  const downloads = Array.isArray(d.downloads) ? d.downloads : [];
+  const warnings = Array.isArray(d.warnings) ? d.warnings.map(String) : [];
+  const limitations = Array.isArray(d.limitations) ? d.limitations.map(String) : [];
+  const transcript = d.transcript;
+  const transcriptText = transcript?.text ?? "";
+  const chunks = transcript?.chunks ?? [];
+  const browser = structuredInstagramBrowser(d.browser);
+  const browserLabel = browser.mode === "local" ? `local profile ${browser.profileName ? `\`${browser.profileName}\`` : "(unnamed)"}` : "hosted browser";
+  const downloadRows = downloads.map((download, i) => {
+    const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
+    return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
+  }).join("\n");
+  const trackRows = tracks.slice(0, 20).map(
+    (track, i) => `| ${i + 1} | ${track.streamType} | ${track.bitrate ?? "\u2014"} | ${track.durationSec ?? "\u2014"} | ${cell(track.vencodeTag ?? "")} |`
+  ).join("\n");
+  const full = [
+    `# Instagram Media Download`,
+    `**URL:** ${d.pageUrl ?? input.url}`,
+    `**Browser:** ${browserLabel}`,
+    d.ownerName ? `**Owner:** ${d.ownerName}` : "",
+    d.shortcode ? `**Shortcode:** \`${d.shortcode}\`` : "",
+    d.caption ? `
+## Caption
+${truncate(String(d.caption), 1200)}` : "",
+    d.imageUrl ? `
+## Image
+${d.imageUrl}` : "",
+    tracks.length ? `
+## Media Tracks
+| # | Type | Bitrate | Duration | Tag |
+|---|------|---------|----------|-----|
+${trackRows}` : "\n## Media Tracks\n*None captured.*",
+    downloads.length ? `
+## Downloads
+| # | Kind | File | Status |
+|---|------|------|--------|
+${downloadRows}` : "",
+    d.outputDir ? `
+**Output directory:** \`${d.outputDir}\`` : "",
+    transcript ? `
+## Transcript
+**${wordCount(transcriptText)} words** \xB7 ${chunks.length} chunks
+
+${transcriptText}` : "",
+    warnings.length ? `
+## Warnings
+${warnings.map((w) => `- ${w}`).join("\n")}` : "",
+    limitations.length ? `
+## Limits
+${limitations.map((l) => `- ${l}`).join("\n")}` : "",
+    `
+---
+\u{1F4A1} Reels may expose separate video-only and audio-only MP4 tracks. Use the muxed file when present; otherwise use the selected video/audio track URLs or saved files.`
+  ].filter(Boolean).join("\n");
+  return {
+    ...oneBlock(full),
+    structuredContent: {
+      sourceUrl: String(d.sourceUrl ?? input.url),
+      pageUrl: String(d.pageUrl ?? input.url),
+      browser,
+      type: d.type === "post" || d.type === "reel" || d.type === "tv" ? d.type : null,
+      shortcode: typeof d.shortcode === "string" ? d.shortcode : null,
+      ownerName: typeof d.ownerName === "string" ? d.ownerName : null,
+      caption: typeof d.caption === "string" ? d.caption : null,
+      imageUrl: typeof d.imageUrl === "string" ? d.imageUrl : null,
+      trackCount: tracks.length,
+      selectedVideoTrack: structuredInstagramTrack(d.selectedVideoTrack),
+      selectedAudioTrack: structuredInstagramTrack(d.selectedAudioTrack),
+      downloads: downloads.map((download) => ({
+        kind: download.kind,
+        url: download.url ?? null,
+        savedPath: download.savedPath ?? null,
+        sizeBytes: typeof download.sizeBytes === "number" ? download.sizeBytes : null,
+        mimeType: download.mimeType ?? null,
+        error: download.error ?? null
+      })),
+      outputDir: typeof d.outputDir === "string" ? d.outputDir : null,
+      warnings,
+      limitations,
+      transcript: transcript ? {
+        wordCount: wordCount(transcriptText),
+        chunkCount: chunks.length,
+        durationMs: typeof transcript.durationMs === "number" ? transcript.durationMs : null,
+        transcriptText,
+        chunks: structuredTranscriptChunks(chunks)
+      } : null
+    }
+  };
+}
 function formatCaptureSerpSnapshot(raw, input) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
@@ -1628,6 +1838,26 @@ var FacebookVideoTranscribeInputSchema = {
   url: z.string().url().describe("Organic Facebook reel, video, watch, post, or share URL from facebook.com, m.facebook.com, or fb.watch. The tool renders the page, extracts the best matching public Facebook CDN MP4 URL, then transcribes it. Use this when the user pastes a normal Facebook video page URL and asks for the transcript or downloadable MP4."),
   quality: z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
 };
+var InstagramProfileContentInputSchema = {
+  handle: z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url. Use only public profiles unless a configured browser profile is available server-side."),
+  url: z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
+  maxItems: z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values with an authenticated browser profile when the user asks for a fuller archive."),
+  maxScrolls: z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase when using authenticated Chromium and the profile has a long history."),
+  scrollDelayMs: z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
+  stableScrollLimit: z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4."),
+  browserMode: z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
+  browserProfile: z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
+};
+var InstagramMediaDownloadInputSchema = {
+  url: z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
+  mediaTypes: z.array(z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download when downloadMedia is true. Reels commonly expose separate video-only and audio-only MP4 tracks."),
+  downloadMedia: z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
+  downloadAllTracks: z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
+  includeTranscript: z.boolean().default(false).describe("Transcribe the selected audio track when available. This adds transcription cost and may take longer."),
+  mux: z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable."),
+  browserMode: z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
+  browserProfile: z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
+};
 var MapsPlaceIntelInputSchema = {
   businessName: z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
   location: z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
@@ -2024,6 +2254,101 @@ var TranscriptChunkOutput = z.object({
   endSec: z.number(),
   text: z.string()
 });
+var InstagramBrowserOutput = z.object({
+  mode: z.enum(["hosted", "local"]),
+  requestedMode: z.enum(["auto", "hosted", "local"]),
+  profileName: NullableString,
+  profileSource: z.enum(["hosted", "managed_profile", "direct_profile_dir"]),
+  profileDirConfigured: z.boolean(),
+  executablePathConfigured: z.boolean()
+});
+var InstagramPaginationOutput = z.object({
+  maxItems: z.number().int().min(1).max(500),
+  maxScrolls: z.number().int().min(0).max(100),
+  attemptedScrolls: z.number().int().min(0),
+  stableScrolls: z.number().int().min(0),
+  stableScrollLimit: z.number().int().min(1).max(10),
+  scrollDelayMs: z.number().int().min(250).max(5e3),
+  reachedMaxItems: z.boolean(),
+  reachedReportedPostCount: z.boolean(),
+  finalScrollHeight: z.number().int().nullable(),
+  stoppedReason: z.enum(["max_items", "reported_post_count", "stable_scrolls", "max_scrolls", "no_scrolls"]),
+  stages: z.array(z.object({
+    stage: z.string(),
+    itemCount: z.number().int().min(0),
+    addedCount: z.number().int().min(0),
+    scrollY: z.number().nullable(),
+    scrollHeight: z.number().nullable()
+  }))
+});
+var InstagramProfileContentOutputSchema = {
+  handle: z.string(),
+  profileUrl: z.string().url(),
+  pageUrl: z.string().url(),
+  browser: InstagramBrowserOutput,
+  profileName: NullableString,
+  reportedPostCount: z.number().int().nullable(),
+  reportedPostCountText: NullableString,
+  followerCountText: NullableString,
+  followingCountText: NullableString,
+  collectedContentCount: z.number().int().min(0),
+  typeCounts: z.object({
+    post: z.number().int().min(0),
+    reel: z.number().int().min(0),
+    tv: z.number().int().min(0)
+  }),
+  pagination: InstagramPaginationOutput,
+  limited: z.boolean(),
+  limitations: z.array(z.string()),
+  items: z.array(z.object({
+    url: z.string().url(),
+    type: z.enum(["post", "reel", "tv"]),
+    shortcode: z.string(),
+    anchorText: NullableString,
+    firstSeenStage: z.string()
+  }))
+};
+var InstagramMediaTrackOutput = z.object({
+  url: z.string().url(),
+  streamType: z.enum(["video", "audio", "unknown"]),
+  bitrate: z.number().int().nullable(),
+  durationSec: z.number().nullable(),
+  vencodeTag: NullableString,
+  width: z.number().int().nullable(),
+  height: z.number().int().nullable()
+});
+var InstagramDownloadOutput = z.object({
+  kind: z.enum(["text", "image", "video", "audio", "muxed_video"]),
+  url: z.string().url().nullable(),
+  savedPath: NullableString,
+  sizeBytes: z.number().int().nullable(),
+  mimeType: NullableString,
+  error: NullableString
+});
+var InstagramMediaDownloadOutputSchema = {
+  sourceUrl: z.string().url(),
+  pageUrl: z.string().url(),
+  browser: InstagramBrowserOutput,
+  type: z.enum(["post", "reel", "tv"]).nullable(),
+  shortcode: NullableString,
+  ownerName: NullableString,
+  caption: NullableString,
+  imageUrl: z.string().url().nullable(),
+  trackCount: z.number().int().min(0),
+  selectedVideoTrack: InstagramMediaTrackOutput.nullable(),
+  selectedAudioTrack: InstagramMediaTrackOutput.nullable(),
+  downloads: z.array(InstagramDownloadOutput),
+  outputDir: NullableString,
+  warnings: z.array(z.string()),
+  limitations: z.array(z.string()),
+  transcript: z.object({
+    wordCount: z.number().int().min(0),
+    chunkCount: z.number().int().min(0),
+    durationMs: z.number().nullable(),
+    transcriptText: z.string(),
+    chunks: z.array(TranscriptChunkOutput)
+  }).nullable()
+};
 var YoutubeTranscribeOutputSchema = {
   videoId: NullableString,
   url: NullableString,
@@ -2673,6 +2998,20 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
     outputSchema: FacebookVideoTranscribeOutputSchema,
     annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
   }, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
+  server.registerTool("instagram_profile_content", {
+    title: "Instagram Profile Content Discovery",
+    description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser mode, pagination attempts, stop reason, and limitations. For authenticated collection, first use browser_profile_import/sync, then pass browserProfile and raise maxItems/maxScrolls for longer profile pagination."),
+    inputSchema: InstagramProfileContentInputSchema,
+    outputSchema: InstagramProfileContentOutputSchema,
+    annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
+  }, async (input) => formatInstagramProfileContent(await executor.instagramProfileContent(input), input));
+  server.registerTool("instagram_media_download", {
+    title: "Instagram Post/Reel Media Download",
+    description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available. Pass browserProfile from browser_profile_import/sync when the media is login-gated."),
+    inputSchema: InstagramMediaDownloadInputSchema,
+    outputSchema: InstagramMediaDownloadOutputSchema,
+    annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
+  }, async (input) => formatInstagramMediaDownload(await executor.instagramMediaDownload(input), input));
   server.registerTool("maps_place_intel", {
     title: "Google Maps Business Profile Details",
     description: withReportNote('Extract Google Maps business intelligence for one known/named business: rating, review count, category, address, phone, website, hours, booking URL, review histogram, review topics, about attributes, entity IDs, and optional review cards. Do not use this for category searches, local market prospect lists, or requests for multiple GMB/GBP profiles; use maps_search first for those. Split business name from location (e.g. "Elite Roofing Denver CO" => businessName "Elite Roofing", location "Denver, CO"). Pass includeReviews true when the user asks for reviews/customer pain.'),
@@ -2933,6 +3272,12 @@ var HttpMcpToolExecutor = class {
   facebookVideoTranscribe(input) {
     return this.call("/facebook/video-transcribe", input, this.httpTimeoutOverrideMs ?? 24e4);
   }
+  instagramProfileContent(input) {
+    return this.call("/instagram/profile-content", input, this.httpTimeoutOverrideMs ?? 24e4);
+  }
+  instagramMediaDownload(input) {
+    return this.call("/instagram/media-download", input, this.httpTimeoutOverrideMs ?? 3e5);
+  }
   mapsPlaceIntel(input) {
     return this.call("/maps/place", input);
   }
@@ -2995,4 +3340,4 @@ export {
   registerPaaExtractorMcpTools,
   HttpMcpToolExecutor
 };
-//# sourceMappingURL=chunk-SIXJ6EVS.js.map
+//# sourceMappingURL=chunk-KO223PSJ.js.map