mcp-scraper 0.2.13 → 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -2
- package/dist/bin/api-server.cjs +977 -200
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +46 -3
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +42 -2
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +135 -5
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +4 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +4 -3
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +135 -5
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/chunk-2CQXHSWC.js +7 -0
- package/dist/chunk-2CQXHSWC.js.map +1 -0
- package/dist/{chunk-ROS67BNV.js → chunk-BSYPATSM.js} +21 -2
- package/dist/chunk-BSYPATSM.js.map +1 -0
- package/dist/{chunk-AYTOCZBS.js → chunk-F44RBOJ5.js} +6 -3
- package/dist/chunk-F44RBOJ5.js.map +1 -0
- package/dist/{chunk-DKJ2XCY7.js → chunk-Q4DFONIK.js} +136 -6
- package/dist/chunk-Q4DFONIK.js.map +1 -0
- package/dist/{chunk-BQTWXY6G.js → chunk-TIPUIEJN.js} +2 -2
- package/dist/{chunk-7GCCOT3M.js → chunk-TL7YTFLH.js} +15 -1
- package/dist/chunk-TL7YTFLH.js.map +1 -0
- package/dist/{db-BVHYI57K.js → db-P5X6UQ3E.js} +2 -2
- package/dist/{server-LUNOI26E.js → server-VD2TD3AD.js} +773 -184
- package/dist/server-VD2TD3AD.js.map +1 -0
- package/dist/{worker-TDJQ6TH3.js → worker-OZSWIS3F.js} +3 -3
- package/package.json +1 -1
- package/dist/chunk-7GCCOT3M.js.map +0 -1
- package/dist/chunk-AYTOCZBS.js.map +0 -1
- package/dist/chunk-DKJ2XCY7.js.map +0 -1
- package/dist/chunk-RDROUQ4E.js +0 -7
- package/dist/chunk-RDROUQ4E.js.map +0 -1
- package/dist/chunk-ROS67BNV.js.map +0 -1
- package/dist/server-LUNOI26E.js.map +0 -1
- /package/dist/{chunk-BQTWXY6G.js.map → chunk-TIPUIEJN.js.map} +0 -0
- /package/dist/{db-BVHYI57K.js.map → db-P5X6UQ3E.js.map} +0 -0
- /package/dist/{worker-TDJQ6TH3.js.map → worker-OZSWIS3F.js.map} +0 -0
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
} from "./chunk-M2S27J6Z.js";
|
|
4
4
|
import {
|
|
5
5
|
PACKAGE_VERSION
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-2CQXHSWC.js";
|
|
7
7
|
|
|
8
8
|
// src/harvest-timeout.ts
|
|
9
9
|
var VERCEL_FUNCTION_MAX_MS = 3e5;
|
|
@@ -660,6 +660,8 @@ function formatCreditsInfo(raw, input) {
|
|
|
660
660
|
const costs = d.costs ?? [];
|
|
661
661
|
const matched = d.matched_cost;
|
|
662
662
|
const ledger = d.ledger ?? [];
|
|
663
|
+
const concurrencyRaw = d.concurrency;
|
|
664
|
+
const upgradeRaw = concurrencyRaw?.upgrade;
|
|
663
665
|
const costRows = costs.map((c) => {
|
|
664
666
|
const notes = c.notes ? ` ${c.notes}` : "";
|
|
665
667
|
return `| ${c.label} | ${c.credits} | ${c.unit}${notes} |`;
|
|
@@ -675,10 +677,20 @@ function formatCreditsInfo(raw, input) {
|
|
|
675
677
|
${matched.notes}` : ""}` : input.item ? `
|
|
676
678
|
## Matched Cost
|
|
677
679
|
No exact cost match found for "${input.item}". See the full cost table below.` : "";
|
|
680
|
+
const concurrencySection = concurrencyRaw ? [
|
|
681
|
+
`
|
|
682
|
+
## Concurrency`,
|
|
683
|
+
`**Current limit:** ${concurrencyRaw.current_limit ?? "unknown"} concurrent operation${concurrencyRaw.current_limit === 1 ? "" : "s"}`,
|
|
684
|
+
`**Extra slots:** ${concurrencyRaw.current_extra_slots ?? "unknown"}`,
|
|
685
|
+
`**Extra concurrency slot:** ${upgradeRaw?.price_label ?? "$5/month"}`,
|
|
686
|
+
`**Upgrade in terminal:** \`${upgradeRaw?.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"}\``,
|
|
687
|
+
`**Billing URL:** ${upgradeRaw?.billing_url ?? "https://mcpscraper.dev/billing"}`
|
|
688
|
+
].join("\n") : "";
|
|
678
689
|
const full = [
|
|
679
690
|
`# Credits`,
|
|
680
691
|
`**Balance:** ${balance ?? "unknown"} credits`,
|
|
681
692
|
matchedSection,
|
|
693
|
+
concurrencySection,
|
|
682
694
|
costs.length ? `
|
|
683
695
|
## Cost Table
|
|
684
696
|
| Item | Credits | Unit |
|
|
@@ -707,7 +719,22 @@ ${ledgerRows}` : ""
|
|
|
707
719
|
operation: String(row.operation ?? ""),
|
|
708
720
|
credits: row.amount_mc / 1e3,
|
|
709
721
|
description: row.description ?? null
|
|
710
|
-
}))
|
|
722
|
+
})),
|
|
723
|
+
concurrency: concurrencyRaw && upgradeRaw ? {
|
|
724
|
+
currentExtraSlots: Number(concurrencyRaw.current_extra_slots ?? 0),
|
|
725
|
+
currentLimit: Number(concurrencyRaw.current_limit ?? 1),
|
|
726
|
+
hasSubscription: concurrencyRaw.has_subscription === true,
|
|
727
|
+
upgrade: {
|
|
728
|
+
product: String(upgradeRaw.product ?? "Extra concurrency slot"),
|
|
729
|
+
priceLabel: String(upgradeRaw.price_label ?? "$5/month"),
|
|
730
|
+
unitAmountUsd: Number(upgradeRaw.unit_amount_usd ?? 5),
|
|
731
|
+
currency: String(upgradeRaw.currency ?? "usd"),
|
|
732
|
+
interval: String(upgradeRaw.interval ?? "month"),
|
|
733
|
+
billingUrl: String(upgradeRaw.billing_url ?? "https://mcpscraper.dev/billing"),
|
|
734
|
+
terminalCommand: String(upgradeRaw.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"),
|
|
735
|
+
terminalCommandWithApiKeyEnv: String(upgradeRaw.terminal_command_with_api_key_env ?? "MCP_SCRAPER_API_KEY=sk_live_your_key npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout")
|
|
736
|
+
}
|
|
737
|
+
} : null
|
|
711
738
|
}
|
|
712
739
|
};
|
|
713
740
|
}
|
|
@@ -993,6 +1020,62 @@ ${chunkRows}` : "",
|
|
|
993
1020
|
].filter(Boolean).join("\n");
|
|
994
1021
|
return oneBlock(full);
|
|
995
1022
|
}
|
|
1023
|
+
function formatFacebookVideoTranscribe(raw, input) {
|
|
1024
|
+
const parsed = parseData(raw);
|
|
1025
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
1026
|
+
const d = parsed.data;
|
|
1027
|
+
const text = d.text ?? "";
|
|
1028
|
+
const chunks = d.chunks ?? [];
|
|
1029
|
+
const wordCount = text.trim() ? text.trim().split(/\s+/).length : 0;
|
|
1030
|
+
const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
|
|
1031
|
+
const videoDuration = typeof d.videoDurationSec === "number" ? `${Math.round(d.videoDurationSec)}s` : "\u2014";
|
|
1032
|
+
const label = d.videoId ? `Facebook Organic Video \`${d.videoId}\`` : "Facebook Organic Video";
|
|
1033
|
+
const chunkRows = chunks.slice(0, 50).map((c) => {
|
|
1034
|
+
const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
|
|
1035
|
+
const mm = String(Math.floor(sec / 60)).padStart(2, "0");
|
|
1036
|
+
const ss = String(sec % 60).padStart(2, "0");
|
|
1037
|
+
return `| ${mm}:${ss} | ${cell(truncate(c.text, 120))} |`;
|
|
1038
|
+
}).join("\n");
|
|
1039
|
+
const full = [
|
|
1040
|
+
`# ${label} Transcript`,
|
|
1041
|
+
d.ownerName ? `**Owner:** ${d.ownerName}` : "",
|
|
1042
|
+
`**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${wordCount} words**`,
|
|
1043
|
+
d.pageUrl ? `**Page URL:** ${d.pageUrl}` : `**Page URL:** ${input.url}`,
|
|
1044
|
+
d.videoUrl ? `**Extracted MP4:** \`${d.videoUrl}\`` : "",
|
|
1045
|
+
`
|
|
1046
|
+
## Full Transcript
|
|
1047
|
+
${text}`,
|
|
1048
|
+
chunks.length ? `
|
|
1049
|
+
## Timestamped Chunks
|
|
1050
|
+
| Time | Text |
|
|
1051
|
+
|------|------|
|
|
1052
|
+
${chunkRows}` : "",
|
|
1053
|
+
`
|
|
1054
|
+
---
|
|
1055
|
+
\u{1F4A1} To download the extracted MP4, call the Facebook media endpoint with the extracted MP4 URL.`
|
|
1056
|
+
].filter(Boolean).join("\n");
|
|
1057
|
+
return {
|
|
1058
|
+
...oneBlock(full),
|
|
1059
|
+
structuredContent: {
|
|
1060
|
+
sourceUrl: d.sourceUrl ?? input.url,
|
|
1061
|
+
pageUrl: d.pageUrl ?? input.url,
|
|
1062
|
+
videoId: d.videoId ?? null,
|
|
1063
|
+
ownerName: d.ownerName ?? null,
|
|
1064
|
+
selectedQuality: d.selectedQuality ?? input.quality ?? "best",
|
|
1065
|
+
bitrate: typeof d.bitrate === "number" ? d.bitrate : null,
|
|
1066
|
+
videoDurationSec: typeof d.videoDurationSec === "number" ? d.videoDurationSec : null,
|
|
1067
|
+
videoUrl: d.videoUrl ?? "",
|
|
1068
|
+
wordCount,
|
|
1069
|
+
chunkCount: chunks.length,
|
|
1070
|
+
transcriptText: text,
|
|
1071
|
+
chunks: chunks.map((c) => ({
|
|
1072
|
+
startSec: Number.isFinite(c.timestamp[0]) ? c.timestamp[0] : 0,
|
|
1073
|
+
endSec: Number.isFinite(c.timestamp[1]) ? c.timestamp[1] : 0,
|
|
1074
|
+
text: c.text
|
|
1075
|
+
}))
|
|
1076
|
+
}
|
|
1077
|
+
};
|
|
1078
|
+
}
|
|
996
1079
|
|
|
997
1080
|
// src/mcp/mcp-tool-schemas.ts
|
|
998
1081
|
import { z } from "zod";
|
|
@@ -1048,6 +1131,10 @@ var FacebookAdSearchInputSchema = {
|
|
|
1048
1131
|
var FacebookAdTranscribeInputSchema = {
|
|
1049
1132
|
videoUrl: z.string().url().describe("Facebook CDN video URL from a facebook_page_intel result")
|
|
1050
1133
|
};
|
|
1134
|
+
var FacebookVideoTranscribeInputSchema = {
|
|
1135
|
+
url: z.string().url().describe("Organic Facebook reel, video, watch, or share URL. The tool renders the page, extracts the best public Facebook CDN MP4 URL, then transcribes it."),
|
|
1136
|
+
quality: z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
|
|
1137
|
+
};
|
|
1051
1138
|
var MapsPlaceIntelInputSchema = {
|
|
1052
1139
|
businessName: z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
1053
1140
|
location: z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
|
|
@@ -1341,7 +1428,22 @@ var CreditsInfoOutputSchema = {
|
|
|
1341
1428
|
operation: z.string(),
|
|
1342
1429
|
credits: z.number(),
|
|
1343
1430
|
description: NullableString
|
|
1344
|
-
}))
|
|
1431
|
+
})),
|
|
1432
|
+
concurrency: z.object({
|
|
1433
|
+
currentExtraSlots: z.number().int().min(0),
|
|
1434
|
+
currentLimit: z.number().int().min(1),
|
|
1435
|
+
hasSubscription: z.boolean(),
|
|
1436
|
+
upgrade: z.object({
|
|
1437
|
+
product: z.string(),
|
|
1438
|
+
priceLabel: z.string(),
|
|
1439
|
+
unitAmountUsd: z.number(),
|
|
1440
|
+
currency: z.string(),
|
|
1441
|
+
interval: z.string(),
|
|
1442
|
+
billingUrl: z.string().url(),
|
|
1443
|
+
terminalCommand: z.string(),
|
|
1444
|
+
terminalCommandWithApiKeyEnv: z.string()
|
|
1445
|
+
})
|
|
1446
|
+
}).nullable()
|
|
1345
1447
|
};
|
|
1346
1448
|
var MapSiteUrlsOutputSchema = {
|
|
1347
1449
|
startUrl: z.string(),
|
|
@@ -1398,8 +1500,26 @@ var FacebookPageIntelOutputSchema = {
|
|
|
1398
1500
|
variations: z.number().int().nullable()
|
|
1399
1501
|
}))
|
|
1400
1502
|
};
|
|
1503
|
+
var FacebookVideoTranscribeOutputSchema = {
|
|
1504
|
+
sourceUrl: z.string().url(),
|
|
1505
|
+
pageUrl: z.string().url(),
|
|
1506
|
+
videoId: NullableString,
|
|
1507
|
+
ownerName: NullableString,
|
|
1508
|
+
selectedQuality: z.string(),
|
|
1509
|
+
bitrate: z.number().int().nullable(),
|
|
1510
|
+
videoDurationSec: z.number().nullable(),
|
|
1511
|
+
videoUrl: z.string().url(),
|
|
1512
|
+
wordCount: z.number().int().min(0),
|
|
1513
|
+
chunkCount: z.number().int().min(0),
|
|
1514
|
+
transcriptText: z.string(),
|
|
1515
|
+
chunks: z.array(z.object({
|
|
1516
|
+
startSec: z.number(),
|
|
1517
|
+
endSec: z.number(),
|
|
1518
|
+
text: z.string()
|
|
1519
|
+
}))
|
|
1520
|
+
};
|
|
1401
1521
|
var CreditsInfoInputSchema = {
|
|
1402
|
-
item: z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url",
|
|
1522
|
+
item: z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
|
|
1403
1523
|
includeLedger: z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
1404
1524
|
};
|
|
1405
1525
|
var SearchSerpInputSchema = {
|
|
@@ -1897,6 +2017,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
1897
2017
|
inputSchema: FacebookAdTranscribeInputSchema,
|
|
1898
2018
|
annotations: liveWebToolAnnotations("Facebook Ad Transcription")
|
|
1899
2019
|
}, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
|
|
2020
|
+
server.registerTool("facebook_video_transcribe", {
|
|
2021
|
+
title: "Facebook Organic Video Transcription",
|
|
2022
|
+
description: withReportNote("Transcribe audio from an organic Facebook reel, video, watch, or share URL. Renders the Facebook page, extracts the best public Facebook CDN MP4 URL from the page state, then returns the full transcript, timestamped chunks, and extracted MP4 URL."),
|
|
2023
|
+
inputSchema: FacebookVideoTranscribeInputSchema,
|
|
2024
|
+
outputSchema: FacebookVideoTranscribeOutputSchema,
|
|
2025
|
+
annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
|
|
2026
|
+
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
1900
2027
|
server.registerTool("maps_place_intel", {
|
|
1901
2028
|
title: "Google Maps Business Profile Details",
|
|
1902
2029
|
description: withReportNote('Extract Google Maps business intelligence for one known/named business: rating, review count, category, address, phone, website, hours, booking URL, review histogram, review topics, about attributes, entity IDs, and optional review cards. Do not use this for category searches, local market prospect lists, or requests for multiple GMB/GBP profiles; use maps_search first for those. Split business name from location (e.g. "Elite Roofing Denver CO" => businessName "Elite Roofing", location "Denver, CO"). Pass includeReviews true when the user asks for reviews/customer pain.'),
|
|
@@ -1927,7 +2054,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
1927
2054
|
}, async (input) => buildRankTrackerBlueprint(input));
|
|
1928
2055
|
server.registerTool("credits_info", {
|
|
1929
2056
|
title: "MCP Scraper Credits & Costs",
|
|
1930
|
-
description: "Answer questions about MCP Scraper credits: current credit balance, what a specific tool/action costs, the full cost table, and
|
|
2057
|
+
description: "Answer questions about MCP Scraper credits, usage limits, and concurrency upgrades: current credit balance, what a specific tool/action costs, the full cost table, current concurrency limit, the extra-slot price, billing URL, and the terminal checkout command. Does not expose payment methods or credit card information.",
|
|
1931
2058
|
inputSchema: CreditsInfoInputSchema,
|
|
1932
2059
|
outputSchema: CreditsInfoOutputSchema,
|
|
1933
2060
|
annotations: {
|
|
@@ -2026,6 +2153,9 @@ var HttpMcpToolExecutor = class {
|
|
|
2026
2153
|
facebookAdTranscribe(input) {
|
|
2027
2154
|
return this.call("/facebook/transcribe", input);
|
|
2028
2155
|
}
|
|
2156
|
+
facebookVideoTranscribe(input) {
|
|
2157
|
+
return this.call("/facebook/video-transcribe", input, this.httpTimeoutOverrideMs ?? 24e4);
|
|
2158
|
+
}
|
|
2029
2159
|
mapsPlaceIntel(input) {
|
|
2030
2160
|
return this.call("/maps/place", input);
|
|
2031
2161
|
}
|
|
@@ -2060,4 +2190,4 @@ export {
|
|
|
2060
2190
|
registerPaaExtractorMcpTools,
|
|
2061
2191
|
HttpMcpToolExecutor
|
|
2062
2192
|
};
|
|
2063
|
-
//# sourceMappingURL=chunk-
|
|
2193
|
+
//# sourceMappingURL=chunk-Q4DFONIK.js.map
|