mcp-scraper 0.2.13 → 0.2.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +12 -2
  2. package/dist/bin/api-server.cjs +977 -200
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/browser-agent-stdio-server.cjs +1 -1
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +2 -2
  8. package/dist/bin/mcp-scraper-cli.cjs +46 -3
  9. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-cli.js +42 -2
  11. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  12. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +135 -5
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
  15. package/dist/bin/mcp-scraper-install.cjs +4 -3
  16. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  17. package/dist/bin/mcp-scraper-install.js +4 -3
  18. package/dist/bin/mcp-scraper-install.js.map +1 -1
  19. package/dist/bin/mcp-stdio-server.cjs +135 -5
  20. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  21. package/dist/bin/mcp-stdio-server.js +2 -2
  22. package/dist/chunk-2CQXHSWC.js +7 -0
  23. package/dist/chunk-2CQXHSWC.js.map +1 -0
  24. package/dist/{chunk-ROS67BNV.js → chunk-BSYPATSM.js} +21 -2
  25. package/dist/chunk-BSYPATSM.js.map +1 -0
  26. package/dist/{chunk-AYTOCZBS.js → chunk-F44RBOJ5.js} +6 -3
  27. package/dist/chunk-F44RBOJ5.js.map +1 -0
  28. package/dist/{chunk-DKJ2XCY7.js → chunk-Q4DFONIK.js} +136 -6
  29. package/dist/chunk-Q4DFONIK.js.map +1 -0
  30. package/dist/{chunk-BQTWXY6G.js → chunk-TIPUIEJN.js} +2 -2
  31. package/dist/{chunk-7GCCOT3M.js → chunk-TL7YTFLH.js} +15 -1
  32. package/dist/chunk-TL7YTFLH.js.map +1 -0
  33. package/dist/{db-BVHYI57K.js → db-P5X6UQ3E.js} +2 -2
  34. package/dist/{server-LUNOI26E.js → server-VD2TD3AD.js} +773 -184
  35. package/dist/server-VD2TD3AD.js.map +1 -0
  36. package/dist/{worker-TDJQ6TH3.js → worker-OZSWIS3F.js} +3 -3
  37. package/package.json +1 -1
  38. package/dist/chunk-7GCCOT3M.js.map +0 -1
  39. package/dist/chunk-AYTOCZBS.js.map +0 -1
  40. package/dist/chunk-DKJ2XCY7.js.map +0 -1
  41. package/dist/chunk-RDROUQ4E.js +0 -7
  42. package/dist/chunk-RDROUQ4E.js.map +0 -1
  43. package/dist/chunk-ROS67BNV.js.map +0 -1
  44. package/dist/server-LUNOI26E.js.map +0 -1
  45. /package/dist/{chunk-BQTWXY6G.js.map → chunk-TIPUIEJN.js.map} +0 -0
  46. /package/dist/{db-BVHYI57K.js.map → db-P5X6UQ3E.js.map} +0 -0
  47. /package/dist/{worker-TDJQ6TH3.js.map → worker-OZSWIS3F.js.map} +0 -0
@@ -3,7 +3,7 @@ import {
3
3
  } from "./chunk-M2S27J6Z.js";
4
4
  import {
5
5
  PACKAGE_VERSION
6
- } from "./chunk-RDROUQ4E.js";
6
+ } from "./chunk-2CQXHSWC.js";
7
7
 
8
8
  // src/harvest-timeout.ts
9
9
  var VERCEL_FUNCTION_MAX_MS = 3e5;
@@ -660,6 +660,8 @@ function formatCreditsInfo(raw, input) {
660
660
  const costs = d.costs ?? [];
661
661
  const matched = d.matched_cost;
662
662
  const ledger = d.ledger ?? [];
663
+ const concurrencyRaw = d.concurrency;
664
+ const upgradeRaw = concurrencyRaw?.upgrade;
663
665
  const costRows = costs.map((c) => {
664
666
  const notes = c.notes ? ` ${c.notes}` : "";
665
667
  return `| ${c.label} | ${c.credits} | ${c.unit}${notes} |`;
@@ -675,10 +677,20 @@ function formatCreditsInfo(raw, input) {
675
677
  ${matched.notes}` : ""}` : input.item ? `
676
678
  ## Matched Cost
677
679
  No exact cost match found for "${input.item}". See the full cost table below.` : "";
680
+ const concurrencySection = concurrencyRaw ? [
681
+ `
682
+ ## Concurrency`,
683
+ `**Current limit:** ${concurrencyRaw.current_limit ?? "unknown"} concurrent operation${concurrencyRaw.current_limit === 1 ? "" : "s"}`,
684
+ `**Extra slots:** ${concurrencyRaw.current_extra_slots ?? "unknown"}`,
685
+ `**Extra concurrency slot:** ${upgradeRaw?.price_label ?? "$5/month"}`,
686
+ `**Upgrade in terminal:** \`${upgradeRaw?.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"}\``,
687
+ `**Billing URL:** ${upgradeRaw?.billing_url ?? "https://mcpscraper.dev/billing"}`
688
+ ].join("\n") : "";
678
689
  const full = [
679
690
  `# Credits`,
680
691
  `**Balance:** ${balance ?? "unknown"} credits`,
681
692
  matchedSection,
693
+ concurrencySection,
682
694
  costs.length ? `
683
695
  ## Cost Table
684
696
  | Item | Credits | Unit |
@@ -707,7 +719,22 @@ ${ledgerRows}` : ""
707
719
  operation: String(row.operation ?? ""),
708
720
  credits: row.amount_mc / 1e3,
709
721
  description: row.description ?? null
710
- }))
722
+ })),
723
+ concurrency: concurrencyRaw && upgradeRaw ? {
724
+ currentExtraSlots: Number(concurrencyRaw.current_extra_slots ?? 0),
725
+ currentLimit: Number(concurrencyRaw.current_limit ?? 1),
726
+ hasSubscription: concurrencyRaw.has_subscription === true,
727
+ upgrade: {
728
+ product: String(upgradeRaw.product ?? "Extra concurrency slot"),
729
+ priceLabel: String(upgradeRaw.price_label ?? "$5/month"),
730
+ unitAmountUsd: Number(upgradeRaw.unit_amount_usd ?? 5),
731
+ currency: String(upgradeRaw.currency ?? "usd"),
732
+ interval: String(upgradeRaw.interval ?? "month"),
733
+ billingUrl: String(upgradeRaw.billing_url ?? "https://mcpscraper.dev/billing"),
734
+ terminalCommand: String(upgradeRaw.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"),
735
+ terminalCommandWithApiKeyEnv: String(upgradeRaw.terminal_command_with_api_key_env ?? "MCP_SCRAPER_API_KEY=sk_live_your_key npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout")
736
+ }
737
+ } : null
711
738
  }
712
739
  };
713
740
  }
@@ -993,6 +1020,62 @@ ${chunkRows}` : "",
993
1020
  ].filter(Boolean).join("\n");
994
1021
  return oneBlock(full);
995
1022
  }
1023
+ function formatFacebookVideoTranscribe(raw, input) {
1024
+ const parsed = parseData(raw);
1025
+ if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
1026
+ const d = parsed.data;
1027
+ const text = d.text ?? "";
1028
+ const chunks = d.chunks ?? [];
1029
+ const wordCount = text.trim() ? text.trim().split(/\s+/).length : 0;
1030
+ const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
1031
+ const videoDuration = typeof d.videoDurationSec === "number" ? `${Math.round(d.videoDurationSec)}s` : "\u2014";
1032
+ const label = d.videoId ? `Facebook Organic Video \`${d.videoId}\`` : "Facebook Organic Video";
1033
+ const chunkRows = chunks.slice(0, 50).map((c) => {
1034
+ const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
1035
+ const mm = String(Math.floor(sec / 60)).padStart(2, "0");
1036
+ const ss = String(sec % 60).padStart(2, "0");
1037
+ return `| ${mm}:${ss} | ${cell(truncate(c.text, 120))} |`;
1038
+ }).join("\n");
1039
+ const full = [
1040
+ `# ${label} Transcript`,
1041
+ d.ownerName ? `**Owner:** ${d.ownerName}` : "",
1042
+ `**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${wordCount} words**`,
1043
+ d.pageUrl ? `**Page URL:** ${d.pageUrl}` : `**Page URL:** ${input.url}`,
1044
+ d.videoUrl ? `**Extracted MP4:** \`${d.videoUrl}\`` : "",
1045
+ `
1046
+ ## Full Transcript
1047
+ ${text}`,
1048
+ chunks.length ? `
1049
+ ## Timestamped Chunks
1050
+ | Time | Text |
1051
+ |------|------|
1052
+ ${chunkRows}` : "",
1053
+ `
1054
+ ---
1055
+ \u{1F4A1} To download the extracted MP4, call the Facebook media endpoint with the extracted MP4 URL.`
1056
+ ].filter(Boolean).join("\n");
1057
+ return {
1058
+ ...oneBlock(full),
1059
+ structuredContent: {
1060
+ sourceUrl: d.sourceUrl ?? input.url,
1061
+ pageUrl: d.pageUrl ?? input.url,
1062
+ videoId: d.videoId ?? null,
1063
+ ownerName: d.ownerName ?? null,
1064
+ selectedQuality: d.selectedQuality ?? input.quality ?? "best",
1065
+ bitrate: typeof d.bitrate === "number" ? d.bitrate : null,
1066
+ videoDurationSec: typeof d.videoDurationSec === "number" ? d.videoDurationSec : null,
1067
+ videoUrl: d.videoUrl ?? "",
1068
+ wordCount,
1069
+ chunkCount: chunks.length,
1070
+ transcriptText: text,
1071
+ chunks: chunks.map((c) => ({
1072
+ startSec: Number.isFinite(c.timestamp[0]) ? c.timestamp[0] : 0,
1073
+ endSec: Number.isFinite(c.timestamp[1]) ? c.timestamp[1] : 0,
1074
+ text: c.text
1075
+ }))
1076
+ }
1077
+ };
1078
+ }
996
1079
 
997
1080
  // src/mcp/mcp-tool-schemas.ts
998
1081
  import { z } from "zod";
@@ -1048,6 +1131,10 @@ var FacebookAdSearchInputSchema = {
1048
1131
  var FacebookAdTranscribeInputSchema = {
1049
1132
  videoUrl: z.string().url().describe("Facebook CDN video URL from a facebook_page_intel result")
1050
1133
  };
1134
+ var FacebookVideoTranscribeInputSchema = {
1135
+ url: z.string().url().describe("Organic Facebook reel, video, watch, or share URL. The tool renders the page, extracts the best public Facebook CDN MP4 URL, then transcribes it."),
1136
+ quality: z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
1137
+ };
1051
1138
  var MapsPlaceIntelInputSchema = {
1052
1139
  businessName: z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
1053
1140
  location: z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
@@ -1341,7 +1428,22 @@ var CreditsInfoOutputSchema = {
1341
1428
  operation: z.string(),
1342
1429
  credits: z.number(),
1343
1430
  description: NullableString
1344
- }))
1431
+ })),
1432
+ concurrency: z.object({
1433
+ currentExtraSlots: z.number().int().min(0),
1434
+ currentLimit: z.number().int().min(1),
1435
+ hasSubscription: z.boolean(),
1436
+ upgrade: z.object({
1437
+ product: z.string(),
1438
+ priceLabel: z.string(),
1439
+ unitAmountUsd: z.number(),
1440
+ currency: z.string(),
1441
+ interval: z.string(),
1442
+ billingUrl: z.string().url(),
1443
+ terminalCommand: z.string(),
1444
+ terminalCommandWithApiKeyEnv: z.string()
1445
+ })
1446
+ }).nullable()
1345
1447
  };
1346
1448
  var MapSiteUrlsOutputSchema = {
1347
1449
  startUrl: z.string(),
@@ -1398,8 +1500,26 @@ var FacebookPageIntelOutputSchema = {
1398
1500
  variations: z.number().int().nullable()
1399
1501
  }))
1400
1502
  };
1503
+ var FacebookVideoTranscribeOutputSchema = {
1504
+ sourceUrl: z.string().url(),
1505
+ pageUrl: z.string().url(),
1506
+ videoId: NullableString,
1507
+ ownerName: NullableString,
1508
+ selectedQuality: z.string(),
1509
+ bitrate: z.number().int().nullable(),
1510
+ videoDurationSec: z.number().nullable(),
1511
+ videoUrl: z.string().url(),
1512
+ wordCount: z.number().int().min(0),
1513
+ chunkCount: z.number().int().min(0),
1514
+ transcriptText: z.string(),
1515
+ chunks: z.array(z.object({
1516
+ startSec: z.number(),
1517
+ endSec: z.number(),
1518
+ text: z.string()
1519
+ }))
1520
+ };
1401
1521
  var CreditsInfoInputSchema = {
1402
- item: z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", or "YouTube transcription"'),
1522
+ item: z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
1403
1523
  includeLedger: z.boolean().default(false).describe("Whether to include recent credit ledger entries")
1404
1524
  };
1405
1525
  var SearchSerpInputSchema = {
@@ -1897,6 +2017,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
1897
2017
  inputSchema: FacebookAdTranscribeInputSchema,
1898
2018
  annotations: liveWebToolAnnotations("Facebook Ad Transcription")
1899
2019
  }, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
2020
+ server.registerTool("facebook_video_transcribe", {
2021
+ title: "Facebook Organic Video Transcription",
2022
+ description: withReportNote("Transcribe audio from an organic Facebook reel, video, watch, or share URL. Renders the Facebook page, extracts the best public Facebook CDN MP4 URL from the page state, then returns the full transcript, timestamped chunks, and extracted MP4 URL."),
2023
+ inputSchema: FacebookVideoTranscribeInputSchema,
2024
+ outputSchema: FacebookVideoTranscribeOutputSchema,
2025
+ annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
2026
+ }, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
1900
2027
  server.registerTool("maps_place_intel", {
1901
2028
  title: "Google Maps Business Profile Details",
1902
2029
  description: withReportNote('Extract Google Maps business intelligence for one known/named business: rating, review count, category, address, phone, website, hours, booking URL, review histogram, review topics, about attributes, entity IDs, and optional review cards. Do not use this for category searches, local market prospect lists, or requests for multiple GMB/GBP profiles; use maps_search first for those. Split business name from location (e.g. "Elite Roofing Denver CO" => businessName "Elite Roofing", location "Denver, CO"). Pass includeReviews true when the user asks for reviews/customer pain.'),
@@ -1927,7 +2054,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
1927
2054
  }, async (input) => buildRankTrackerBlueprint(input));
1928
2055
  server.registerTool("credits_info", {
1929
2056
  title: "MCP Scraper Credits & Costs",
1930
- description: "Answer questions about MCP Scraper credits: current credit balance, what a specific tool/action costs, the full cost table, and optionally recent credit ledger entries. Does not expose payment methods or credit card information.",
2057
+ description: "Answer questions about MCP Scraper credits, usage limits, and concurrency upgrades: current credit balance, what a specific tool/action costs, the full cost table, current concurrency limit, the extra-slot price, billing URL, and the terminal checkout command. Does not expose payment methods or credit card information.",
1931
2058
  inputSchema: CreditsInfoInputSchema,
1932
2059
  outputSchema: CreditsInfoOutputSchema,
1933
2060
  annotations: {
@@ -2026,6 +2153,9 @@ var HttpMcpToolExecutor = class {
2026
2153
  facebookAdTranscribe(input) {
2027
2154
  return this.call("/facebook/transcribe", input);
2028
2155
  }
2156
+ facebookVideoTranscribe(input) {
2157
+ return this.call("/facebook/video-transcribe", input, this.httpTimeoutOverrideMs ?? 24e4);
2158
+ }
2029
2159
  mapsPlaceIntel(input) {
2030
2160
  return this.call("/maps/place", input);
2031
2161
  }
@@ -2060,4 +2190,4 @@ export {
2060
2190
  registerPaaExtractorMcpTools,
2061
2191
  HttpMcpToolExecutor
2062
2192
  };
2063
- //# sourceMappingURL=chunk-DKJ2XCY7.js.map
2193
+ //# sourceMappingURL=chunk-Q4DFONIK.js.map