mcp-scraper 0.3.10 → 0.3.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +4 -3
  2. package/dist/bin/api-server.cjs +115 -40
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +2 -2
  5. package/dist/bin/browser-agent-stdio-server.cjs +206 -31
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +3 -2
  8. package/dist/bin/browser-agent-stdio-server.js.map +1 -1
  9. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  10. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-cli.js +1 -1
  12. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +262 -60
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -3
  15. package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
  16. package/dist/bin/mcp-scraper-install.cjs +1 -1
  17. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  18. package/dist/bin/mcp-scraper-install.js +1 -1
  19. package/dist/bin/mcp-stdio-server.cjs +38 -11
  20. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  21. package/dist/bin/mcp-stdio-server.js +2 -2
  22. package/dist/bin/paa-harvest.cjs +21 -1
  23. package/dist/bin/paa-harvest.cjs.map +1 -1
  24. package/dist/bin/paa-harvest.js +1 -1
  25. package/dist/chunk-3YGKXXUG.js +133 -0
  26. package/dist/chunk-3YGKXXUG.js.map +1 -0
  27. package/dist/{chunk-UWSG3C5J.js → chunk-4OPKIDON.js} +22 -2
  28. package/dist/chunk-4OPKIDON.js.map +1 -0
  29. package/dist/{chunk-CED7X4WB.js → chunk-7R7VBQRV.js} +39 -12
  30. package/dist/chunk-7R7VBQRV.js.map +1 -0
  31. package/dist/{chunk-QPFF3V2R.js → chunk-BW3DGFNQ.js} +36 -9
  32. package/dist/chunk-BW3DGFNQ.js.map +1 -0
  33. package/dist/chunk-FB5ZJGFN.js +7 -0
  34. package/dist/chunk-FB5ZJGFN.js.map +1 -0
  35. package/dist/index.cjs +21 -1
  36. package/dist/index.cjs.map +1 -1
  37. package/dist/index.js +1 -1
  38. package/dist/{server-QTV2EUKA.js → server-CUPDJLWM.js} +44 -142
  39. package/dist/server-CUPDJLWM.js.map +1 -0
  40. package/dist/{worker-56IXWOQU.js → worker-FG7ZWEGA.js} +2 -2
  41. package/docs/mcp-tool-craft-lint.generated.md +1 -1
  42. package/docs/mcp-tool-manifest.generated.json +3 -3
  43. package/package.json +1 -1
  44. package/dist/chunk-CED7X4WB.js.map +0 -1
  45. package/dist/chunk-NEGW2ZEJ.js +0 -7
  46. package/dist/chunk-NEGW2ZEJ.js.map +0 -1
  47. package/dist/chunk-QPFF3V2R.js.map +0 -1
  48. package/dist/chunk-UWSG3C5J.js.map +0 -1
  49. package/dist/server-QTV2EUKA.js.map +0 -1
  50. /package/dist/{worker-56IXWOQU.js.map → worker-FG7ZWEGA.js.map} +0 -0
@@ -7,7 +7,7 @@ import {
7
7
  } from "./chunk-LFATOGDF.js";
8
8
  import {
9
9
  PACKAGE_VERSION
10
- } from "./chunk-NEGW2ZEJ.js";
10
+ } from "./chunk-FB5ZJGFN.js";
11
11
 
12
12
  // src/harvest-timeout.ts
13
13
  var VERCEL_FUNCTION_MAX_MS = 3e5;
@@ -1530,7 +1530,7 @@ function formatInstagramProfileContent(raw, input) {
1530
1530
  const itemRows = items.slice(0, 100).map(
1531
1531
  (item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
1532
1532
  ).join("\n");
1533
- const browserLabel = "hosted browser";
1533
+ const browserLabel = browser.profileName ? `hosted browser profile ${browser.profileName}` : "hosted browser";
1534
1534
  const full = [
1535
1535
  `# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
1536
1536
  `**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
@@ -1605,7 +1605,7 @@ function formatInstagramMediaDownload(raw, input) {
1605
1605
  const transcriptText = transcript?.text ?? "";
1606
1606
  const chunks = transcript?.chunks ?? [];
1607
1607
  const browser = structuredInstagramBrowser(d.browser);
1608
- const browserLabel = "hosted browser";
1608
+ const browserLabel = browser.profileName ? `hosted browser profile ${browser.profileName}` : "hosted browser";
1609
1609
  const downloadRows = downloads.map((download, i) => {
1610
1610
  const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
1611
1611
  return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
@@ -1840,13 +1840,17 @@ var FacebookVideoTranscribeInputSchema = {
1840
1840
  var InstagramProfileContentInputSchema = {
1841
1841
  handle: z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url."),
1842
1842
  url: z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
1843
- maxItems: z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values only when the user asks for a fuller archive."),
1844
- maxScrolls: z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase for long profiles when Instagram continues loading more grid links."),
1843
+ profile: z.string().min(1).optional().describe("Optional saved hosted browser profile name to load authenticated Instagram access. If omitted, the server uses its configured default profile when present."),
1844
+ saveProfileChanges: z.boolean().optional().describe("Whether to save browser changes back to the hosted profile. Leave unset unless intentionally updating the saved login session."),
1845
+ maxItems: z.number().int().min(1).max(2e3).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 2000. Use higher values only when the user asks for a fuller archive."),
1846
+ maxScrolls: z.number().int().min(0).max(250).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 250. Increase for long profiles when Instagram continues loading more grid links."),
1845
1847
  scrollDelayMs: z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
1846
1848
  stableScrollLimit: z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
1847
1849
  };
1848
1850
  var InstagramMediaDownloadInputSchema = {
1849
1851
  url: z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
1852
+ profile: z.string().min(1).optional().describe("Optional saved hosted browser profile name to load authenticated Instagram access. If omitted, the server uses its configured default profile when present."),
1853
+ saveProfileChanges: z.boolean().optional().describe("Whether to save browser changes back to the hosted profile. Leave unset unless intentionally updating the saved login session."),
1850
1854
  mediaTypes: z.array(z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download when downloadMedia is true. Reels commonly expose separate video-only and audio-only MP4 tracks."),
1851
1855
  downloadMedia: z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
1852
1856
  downloadAllTracks: z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
@@ -2258,8 +2262,8 @@ var InstagramBrowserOutput = z.object({
2258
2262
  executablePathConfigured: z.boolean()
2259
2263
  });
2260
2264
  var InstagramPaginationOutput = z.object({
2261
- maxItems: z.number().int().min(1).max(500),
2262
- maxScrolls: z.number().int().min(0).max(100),
2265
+ maxItems: z.number().int().min(1).max(2e3),
2266
+ maxScrolls: z.number().int().min(0).max(250),
2263
2267
  attemptedScrolls: z.number().int().min(0),
2264
2268
  stableScrolls: z.number().int().min(0),
2265
2269
  stableScrollLimit: z.number().int().min(1).max(10),
@@ -3114,6 +3118,29 @@ function youtubeVideoIdFromUrl(url) {
3114
3118
  }
3115
3119
  return null;
3116
3120
  }
3121
+ async function readResponseData(res) {
3122
+ const text = await res.text();
3123
+ if (!text.trim()) return null;
3124
+ try {
3125
+ return JSON.parse(text);
3126
+ } catch {
3127
+ return text;
3128
+ }
3129
+ }
3130
+ function httpErrorPayload(path, res, data) {
3131
+ const objectData = data && typeof data === "object" && !Array.isArray(data) ? data : null;
3132
+ const bodyMessage = objectData ? objectData.message ?? objectData.error ?? objectData.error_code : data;
3133
+ return {
3134
+ ...objectData ?? { body: data },
3135
+ error: objectData?.error ?? objectData?.error_code ?? "mcp_http_error",
3136
+ error_type: "http",
3137
+ retryable: res.status === 429 || res.status >= 500,
3138
+ status: res.status,
3139
+ statusText: res.statusText,
3140
+ path,
3141
+ message: typeof bodyMessage === "string" && bodyMessage.trim() ? bodyMessage : `MCP Scraper HTTP ${res.status}${res.statusText ? ` ${res.statusText}` : ""} for ${path}`
3142
+ };
3143
+ }
3117
3144
  var HttpMcpToolExecutor = class {
3118
3145
  baseUrl;
3119
3146
  apiKey;
@@ -3141,9 +3168,9 @@ var HttpMcpToolExecutor = class {
3141
3168
  body: JSON.stringify(body),
3142
3169
  signal: AbortSignal.timeout(timeoutMs)
3143
3170
  });
3144
- const data = await res.json();
3171
+ const data = await readResponseData(res);
3145
3172
  if (!res.ok) {
3146
- return { content: [{ type: "text", text: JSON.stringify(data) }], isError: true };
3173
+ return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path, res, data)) }], isError: true };
3147
3174
  }
3148
3175
  return { content: [{ type: "text", text: JSON.stringify(data) }] };
3149
3176
  } catch (err) {
@@ -3176,9 +3203,9 @@ var HttpMcpToolExecutor = class {
3176
3203
  },
3177
3204
  signal: AbortSignal.timeout(timeoutMs)
3178
3205
  });
3179
- const data = await res.json();
3206
+ const data = await readResponseData(res);
3180
3207
  if (!res.ok) {
3181
- return { content: [{ type: "text", text: JSON.stringify(data) }], isError: true };
3208
+ return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path, res, data)) }], isError: true };
3182
3209
  }
3183
3210
  return { content: [{ type: "text", text: JSON.stringify(data) }] };
3184
3211
  } catch (err) {
@@ -3335,4 +3362,4 @@ export {
3335
3362
  registerPaaExtractorMcpTools,
3336
3363
  HttpMcpToolExecutor
3337
3364
  };
3338
- //# sourceMappingURL=chunk-CED7X4WB.js.map
3365
+ //# sourceMappingURL=chunk-7R7VBQRV.js.map