mcp-scraper 0.2.14 → 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/bin/api-server.cjs +314 -23
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +89 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +3 -3
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +89 -1
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/chunk-2CQXHSWC.js +7 -0
- package/dist/chunk-2CQXHSWC.js.map +1 -0
- package/dist/{chunk-IWQTNY2E.js → chunk-Q4DFONIK.js} +90 -2
- package/dist/chunk-Q4DFONIK.js.map +1 -0
- package/dist/{chunk-ZV2XXYR7.js → chunk-TIPUIEJN.js} +2 -2
- package/dist/{server-7QP6HQJJ.js → server-VD2TD3AD.js} +220 -23
- package/dist/server-VD2TD3AD.js.map +1 -0
- package/package.json +1 -1
- package/dist/chunk-IWQTNY2E.js.map +0 -1
- package/dist/chunk-ROCXJOVY.js +0 -7
- package/dist/chunk-ROCXJOVY.js.map +0 -1
- package/dist/server-7QP6HQJJ.js.map +0 -1
- /package/dist/{chunk-ZV2XXYR7.js.map → chunk-TIPUIEJN.js.map} +0 -0
package/README.md
CHANGED
|
@@ -155,6 +155,7 @@ env = { MCP_SCRAPER_API_KEY = "sk_live_your_key" }
|
|
|
155
155
|
- `facebook_ad_search`
|
|
156
156
|
- `facebook_page_intel`
|
|
157
157
|
- `facebook_ad_transcribe`
|
|
158
|
+
- `facebook_video_transcribe` — transcribe an organic Facebook reel, video, watch, or share URL. The tool renders the page, extracts the best public Facebook CDN MP4 URL, then returns transcript text, timestamped chunks, and the extracted MP4 URL for follow-up download.
|
|
158
159
|
- `maps_search` — search Google Maps for multiple business/profile candidates. Use for GMB/GBP prospect lists, competitors, categories, and anything needing more than the Google 3-pack. In default `proxyMode: "location"`, retryable failures rotate to a new residential proxy and new browser session for up to 5 attempts. `maxResults` defaults to 10 and is capped at 50.
|
|
159
160
|
- `maps_place_intel` — hydrate one known/named Google Maps business with profile details and optional reviews. Use after `maps_search` when a selected candidate needs full details.
|
|
160
161
|
- `directory_workflow` — build city-by-city directory/prospecting datasets from Census place selection plus Google Maps searches. Use it for requests like "all cities over 100k population in Tennessee, then get 20 roofers from Maps." In default `proxyMode: "location"`, each city search rotates retryable failures to a new residential proxy and new browser session for up to 5 attempts. The saved CSV includes `source_location`, `result_position`, `business_name`, `review_stars`, `review_count`, `category`, `address`, `phone`, `hours_status`, `website_url`, `directions_url`, `place_url`, `cid`, `cid_decimal`, Census population, and ZIP groups.
|
|
@@ -189,9 +190,9 @@ For Google Maps tools (`maps_search` and `directory_workflow`), keep `proxyMode`
|
|
|
189
190
|
|
|
190
191
|
The MCPB bundle and `mcp-scraper-combined` expose both sections through one local MCP server. The split `mcp-scraper` entrypoint exposes only the web-intelligence tools, and the split `browser-agent` entrypoint exposes only the browser-agent tools.
|
|
191
192
|
|
|
192
|
-
Chaining and planning tools (`maps_search`, `directory_workflow`, `rank_tracker_blueprint`, `map_site_urls`, `youtube_harvest`, `facebook_ad_search`, `facebook_page_intel`) advertise an `outputSchema` and return `structuredContent` with the IDs, URLs, CSV paths, or blueprint fields needed by the next step. All tools carry MCP annotations (`readOnlyHint: true`; live-web tools use `openWorldHint: true`, while local planning and credit tools are closed-world/idempotent).
|
|
193
|
+
Chaining and planning tools (`maps_search`, `directory_workflow`, `rank_tracker_blueprint`, `map_site_urls`, `youtube_harvest`, `facebook_ad_search`, `facebook_page_intel`, `facebook_video_transcribe`) advertise an `outputSchema` and return `structuredContent` with the IDs, URLs, CSV paths, transcripts, or blueprint fields needed by the next step. All tools carry MCP annotations (`readOnlyHint: true`; live-web tools use `openWorldHint: true`, while local planning and credit tools are closed-world/idempotent).
|
|
193
194
|
|
|
194
|
-
The hosted MCP endpoint at `https://mcpscraper.dev/mcp` exposes the
|
|
195
|
+
The hosted MCP endpoint at `https://mcpscraper.dev/mcp` exposes the 16 web-intelligence tools plus `capture_serp_snapshot` and `capture_serp_page_snapshots` (18 total). Browser-agent tools are local stdio tools backed by the REST API under `https://mcpscraper.dev/agent/*`.
|
|
195
196
|
|
|
196
197
|
## Resources
|
|
197
198
|
|
package/dist/bin/api-server.cjs
CHANGED
|
@@ -11618,6 +11618,132 @@ var init_FacebookAdGraphql = __esm({
|
|
|
11618
11618
|
}
|
|
11619
11619
|
});
|
|
11620
11620
|
|
|
11621
|
+
// src/extractor/FacebookOrganicVideoExtractor.ts
|
|
11622
|
+
function htmlDecode(value) {
|
|
11623
|
+
return value.replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'").replace(/</g, "<").replace(/>/g, ">");
|
|
11624
|
+
}
|
|
11625
|
+
function decodeEscapedValue(raw) {
|
|
11626
|
+
let out = raw;
|
|
11627
|
+
try {
|
|
11628
|
+
out = JSON.parse(`"${raw.replace(/"/g, '\\"')}"`);
|
|
11629
|
+
} catch {
|
|
11630
|
+
out = raw.replace(/\\u0025/g, "%").replace(/\\u0026/g, "&").replace(/\\u003D/g, "=").replace(/\\u003d/g, "=").replace(/\\"/g, '"').replace(/\\\//g, "/");
|
|
11631
|
+
}
|
|
11632
|
+
return htmlDecode(out);
|
|
11633
|
+
}
|
|
11634
|
+
function facebookVideoIdFromUrl(url) {
|
|
11635
|
+
try {
|
|
11636
|
+
const parsed = new URL(url);
|
|
11637
|
+
const path6 = parsed.pathname;
|
|
11638
|
+
const direct = path6.match(/\/(?:reel|videos|watch)\/(?:[^/]+\/)?(\d{8,})/i);
|
|
11639
|
+
if (direct) return direct[1];
|
|
11640
|
+
const queryVideoId = parsed.searchParams.get("v") ?? parsed.searchParams.get("video_id");
|
|
11641
|
+
if (queryVideoId && /^\d{8,}$/.test(queryVideoId)) return queryVideoId;
|
|
11642
|
+
const numericTail = path6.match(/\/(\d{8,})\/?$/);
|
|
11643
|
+
if (numericTail) return numericTail[1];
|
|
11644
|
+
} catch {
|
|
11645
|
+
}
|
|
11646
|
+
return null;
|
|
11647
|
+
}
|
|
11648
|
+
function parseEfg(url) {
|
|
11649
|
+
try {
|
|
11650
|
+
const efg = new URL(url).searchParams.get("efg");
|
|
11651
|
+
if (!efg) return null;
|
|
11652
|
+
return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
|
|
11653
|
+
} catch {
|
|
11654
|
+
return null;
|
|
11655
|
+
}
|
|
11656
|
+
}
|
|
11657
|
+
function manifestDurationSec(htmlWindow) {
|
|
11658
|
+
const decoded = decodeEscapedValue(htmlWindow);
|
|
11659
|
+
const match = decoded.match(/mediaPresentationDuration="PT([\d.]+)S"/);
|
|
11660
|
+
return match ? Number(match[1]) : null;
|
|
11661
|
+
}
|
|
11662
|
+
function metadataContent(html, key) {
|
|
11663
|
+
const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
11664
|
+
const re = new RegExp(`<meta\\s+[^>]*(?:property|name)=["']${escaped}["'][^>]*content=["']([^"']+)["'][^>]*>`, "i");
|
|
11665
|
+
const m = html.match(re);
|
|
11666
|
+
return m ? htmlDecode(m[1]).trim() : null;
|
|
11667
|
+
}
|
|
11668
|
+
function ownerNameFromHtml(html) {
|
|
11669
|
+
const ogTitle = metadataContent(html, "og:title");
|
|
11670
|
+
const pipe = ogTitle?.split("|").map((s) => s.trim()).filter(Boolean);
|
|
11671
|
+
if (pipe && pipe.length > 1) return pipe[pipe.length - 1] ?? null;
|
|
11672
|
+
return null;
|
|
11673
|
+
}
|
|
11674
|
+
function extractFacebookOrganicVideo(html, pageUrl, sourceUrl = pageUrl, quality = "best") {
|
|
11675
|
+
const videoId = facebookVideoIdFromUrl(pageUrl) ?? facebookVideoIdFromUrl(sourceUrl);
|
|
11676
|
+
const manifestIndex = videoId ? html.indexOf(`dash_mpd_debug.mpd?v=${videoId}`) : -1;
|
|
11677
|
+
const searchWindow = manifestIndex >= 0 ? html.slice(Math.max(0, manifestIndex - 5e4), Math.min(html.length, manifestIndex + 5e4)) : html;
|
|
11678
|
+
const targetDuration = manifestDurationSec(searchWindow);
|
|
11679
|
+
const candidates = [];
|
|
11680
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11681
|
+
const re = /browser_native_(sd|hd)_url\\?"\s*:\s*\\?"([^"<]+?)\\?"/g;
|
|
11682
|
+
let match;
|
|
11683
|
+
while ((match = re.exec(searchWindow)) !== null) {
|
|
11684
|
+
const url = decodeEscapedValue(match[2]);
|
|
11685
|
+
if (!url || seen.has(url)) continue;
|
|
11686
|
+
seen.add(url);
|
|
11687
|
+
const parsed = parseEfg(url);
|
|
11688
|
+
let bitrate = null;
|
|
11689
|
+
try {
|
|
11690
|
+
const value = new URL(url).searchParams.get("bitrate");
|
|
11691
|
+
bitrate = value ? Number(value) : null;
|
|
11692
|
+
} catch {
|
|
11693
|
+
}
|
|
11694
|
+
candidates.push({
|
|
11695
|
+
url,
|
|
11696
|
+
quality: match[1],
|
|
11697
|
+
bitrate: Number.isFinite(bitrate) ? bitrate : null,
|
|
11698
|
+
durationSec: typeof parsed?.duration_s === "number" ? parsed.duration_s : null,
|
|
11699
|
+
assetId: typeof parsed?.xpv_asset_id === "number" ? parsed.xpv_asset_id : null,
|
|
11700
|
+
vencodeTag: typeof parsed?.vencode_tag === "string" ? parsed.vencode_tag : null
|
|
11701
|
+
});
|
|
11702
|
+
}
|
|
11703
|
+
const targetCandidates = candidates.filter((candidate) => {
|
|
11704
|
+
if (!targetDuration || candidate.durationSec == null) return true;
|
|
11705
|
+
return Math.abs(candidate.durationSec - targetDuration) <= 2;
|
|
11706
|
+
});
|
|
11707
|
+
const selectable = targetCandidates.length ? targetCandidates : candidates;
|
|
11708
|
+
const selected = [...selectable].filter((candidate) => quality === "best" || candidate.quality === quality).sort((a, b) => {
|
|
11709
|
+
const aq = a.quality === "hd" ? 1 : 0;
|
|
11710
|
+
const bq = b.quality === "hd" ? 1 : 0;
|
|
11711
|
+
if (quality === "best" && aq !== bq) return bq - aq;
|
|
11712
|
+
return (b.bitrate ?? 0) - (a.bitrate ?? 0);
|
|
11713
|
+
})[0];
|
|
11714
|
+
if (!selected) {
|
|
11715
|
+
throw new Error("No progressive Facebook video URL was found in the rendered page state");
|
|
11716
|
+
}
|
|
11717
|
+
return {
|
|
11718
|
+
sourceUrl,
|
|
11719
|
+
pageUrl,
|
|
11720
|
+
videoId,
|
|
11721
|
+
title: metadataContent(html, "og:title") ?? metadataContent(html, "twitter:title"),
|
|
11722
|
+
description: metadataContent(html, "og:description") ?? metadataContent(html, "description") ?? metadataContent(html, "twitter:description"),
|
|
11723
|
+
ownerName: ownerNameFromHtml(html),
|
|
11724
|
+
videoUrl: selected.url,
|
|
11725
|
+
selectedQuality: selected.quality,
|
|
11726
|
+
bitrate: selected.bitrate,
|
|
11727
|
+
durationSec: targetDuration ?? selected.durationSec,
|
|
11728
|
+
assetId: selected.assetId,
|
|
11729
|
+
vencodeTag: selected.vencodeTag,
|
|
11730
|
+
candidates: selectable,
|
|
11731
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11732
|
+
};
|
|
11733
|
+
}
|
|
11734
|
+
async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "best") {
|
|
11735
|
+
await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
|
|
11736
|
+
});
|
|
11737
|
+
await page.waitForTimeout(1500);
|
|
11738
|
+
const html = await page.content();
|
|
11739
|
+
return extractFacebookOrganicVideo(html, page.url(), sourceUrl, quality);
|
|
11740
|
+
}
|
|
11741
|
+
var init_FacebookOrganicVideoExtractor = __esm({
|
|
11742
|
+
"src/extractor/FacebookOrganicVideoExtractor.ts"() {
|
|
11743
|
+
"use strict";
|
|
11744
|
+
}
|
|
11745
|
+
});
|
|
11746
|
+
|
|
11621
11747
|
// src/locations.ts
|
|
11622
11748
|
var LOCATIONS;
|
|
11623
11749
|
var init_locations = __esm({
|
|
@@ -12161,6 +12287,39 @@ var init_kernel_proxy_resolver = __esm({
|
|
|
12161
12287
|
function invalidRequest(message) {
|
|
12162
12288
|
return { error_code: "invalid_request", message };
|
|
12163
12289
|
}
|
|
12290
|
+
function isAllowedFacebookOrganicUrl(url) {
|
|
12291
|
+
const hostname = url.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
12292
|
+
return hostname === "facebook.com" || hostname === "fb.watch";
|
|
12293
|
+
}
|
|
12294
|
+
function facebookTranscriptMarkdown(title, text, chunks, durationMs) {
|
|
12295
|
+
const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
12296
|
+
const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
|
|
12297
|
+
if (chunks.length) {
|
|
12298
|
+
lines.push("## Timestamped Segments", "");
|
|
12299
|
+
for (const ch of chunks) {
|
|
12300
|
+
lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
|
|
12301
|
+
}
|
|
12302
|
+
}
|
|
12303
|
+
return lines.join("\n");
|
|
12304
|
+
}
|
|
12305
|
+
async function transcribeFacebookVideoUrl(videoUrl, markdownTitle = "# Facebook Video Transcript") {
|
|
12306
|
+
const startMs = Date.now();
|
|
12307
|
+
const result = await import_client3.fal.subscribe("fal-ai/wizper", {
|
|
12308
|
+
input: { audio_url: videoUrl, task: "transcribe", language: "en" },
|
|
12309
|
+
logs: false,
|
|
12310
|
+
pollInterval: 3e3
|
|
12311
|
+
});
|
|
12312
|
+
const data = result.data;
|
|
12313
|
+
const text = data.text ?? "";
|
|
12314
|
+
const chunks = data.chunks ?? [];
|
|
12315
|
+
const durationMs = Date.now() - startMs;
|
|
12316
|
+
return {
|
|
12317
|
+
text,
|
|
12318
|
+
chunks,
|
|
12319
|
+
durationMs,
|
|
12320
|
+
markdown: facebookTranscriptMarkdown(markdownTitle, text, chunks, durationMs)
|
|
12321
|
+
};
|
|
12322
|
+
}
|
|
12164
12323
|
async function detectSoftBlock(driver) {
|
|
12165
12324
|
const page = driver.getPage();
|
|
12166
12325
|
const bodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
|
|
@@ -12190,7 +12349,7 @@ async function kernelLaunchOptsResidential() {
|
|
|
12190
12349
|
}
|
|
12191
12350
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
12192
12351
|
}
|
|
12193
|
-
var import_hono4, import_zod15, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
|
|
12352
|
+
var import_hono4, import_zod15, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
|
|
12194
12353
|
var init_facebook_ad_routes = __esm({
|
|
12195
12354
|
"src/api/facebook-ad-routes.ts"() {
|
|
12196
12355
|
"use strict";
|
|
@@ -12202,6 +12361,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12202
12361
|
init_BrowserDriver();
|
|
12203
12362
|
init_FacebookAdExtractor();
|
|
12204
12363
|
init_FacebookAdGraphql();
|
|
12364
|
+
init_FacebookOrganicVideoExtractor();
|
|
12205
12365
|
init_kernel_proxy_resolver();
|
|
12206
12366
|
import_client3 = require("@fal-ai/client");
|
|
12207
12367
|
init_api_auth();
|
|
@@ -12224,6 +12384,10 @@ var init_facebook_ad_routes = __esm({
|
|
|
12224
12384
|
FacebookTranscribeBodySchema = import_zod15.z.object({
|
|
12225
12385
|
videoUrl: import_zod15.z.string().trim().min(1, "videoUrl is required")
|
|
12226
12386
|
});
|
|
12387
|
+
FacebookVideoTranscribeBodySchema = import_zod15.z.object({
|
|
12388
|
+
url: import_zod15.z.string().trim().min(1, "url is required"),
|
|
12389
|
+
quality: import_zod15.z.enum(["best", "hd", "sd"]).default("best")
|
|
12390
|
+
});
|
|
12227
12391
|
FacebookSearchBodySchema = import_zod15.z.object({
|
|
12228
12392
|
query: import_zod15.z.string().trim().min(1, "query is required"),
|
|
12229
12393
|
country: import_zod15.z.string().trim().toUpperCase().optional(),
|
|
@@ -12347,26 +12511,9 @@ var init_facebook_ad_routes = __esm({
|
|
|
12347
12511
|
const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, videoUrl);
|
|
12348
12512
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_transcribe), 402);
|
|
12349
12513
|
debited = true;
|
|
12350
|
-
const
|
|
12351
|
-
|
|
12352
|
-
|
|
12353
|
-
logs: false,
|
|
12354
|
-
pollInterval: 3e3
|
|
12355
|
-
});
|
|
12356
|
-
const data = result.data;
|
|
12357
|
-
const text = data.text ?? "";
|
|
12358
|
-
const chunks = data.chunks ?? [];
|
|
12359
|
-
const durationMs = Date.now() - startMs;
|
|
12360
|
-
const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
12361
|
-
const lines = ["# Facebook Ad Transcript", "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
|
|
12362
|
-
if (chunks.length) {
|
|
12363
|
-
lines.push("## Timestamped Segments", "");
|
|
12364
|
-
for (const ch of chunks) {
|
|
12365
|
-
lines.push(`**[${fmtTs2(ch.timestamp[0])} \u2192 ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
|
|
12366
|
-
}
|
|
12367
|
-
}
|
|
12368
|
-
await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: chunks.length, result: { text, chunks, durationMs } });
|
|
12369
|
-
return c.json({ text, chunks, durationMs, markdown: lines.join("\n") });
|
|
12514
|
+
const transcript = await transcribeFacebookVideoUrl(videoUrl, "# Facebook Ad Transcript");
|
|
12515
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
|
|
12516
|
+
return c.json(transcript);
|
|
12370
12517
|
} catch (err) {
|
|
12371
12518
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12372
12519
|
if (debited) await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
|
|
@@ -12376,6 +12523,62 @@ var init_facebook_ad_routes = __esm({
|
|
|
12376
12523
|
await releaseConcurrencyGate(gate.lockId);
|
|
12377
12524
|
}
|
|
12378
12525
|
});
|
|
12526
|
+
facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
|
|
12527
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
12528
|
+
const parsed = FacebookVideoTranscribeBodySchema.safeParse(raw);
|
|
12529
|
+
if (!parsed.success) {
|
|
12530
|
+
return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
12531
|
+
}
|
|
12532
|
+
const body = parsed.data;
|
|
12533
|
+
const urlCheck = await validatePublicHttpUrl(body.url, { field: "url", requireHttps: false });
|
|
12534
|
+
if (urlCheck.error) {
|
|
12535
|
+
return c.json(invalidRequest(urlCheck.error), 400);
|
|
12536
|
+
}
|
|
12537
|
+
const sourceUrl = urlCheck.parsed;
|
|
12538
|
+
if (!isAllowedFacebookOrganicUrl(sourceUrl)) {
|
|
12539
|
+
return c.json(invalidRequest("url must be a Facebook organic video, reel, or share URL"), 400);
|
|
12540
|
+
}
|
|
12541
|
+
const fbUser = c.get("user");
|
|
12542
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_video_transcribe", {
|
|
12543
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
12544
|
+
metadata: { url: sourceUrl.href }
|
|
12545
|
+
});
|
|
12546
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12547
|
+
import_client3.fal.config({ credentials: process.env.FAL_KEY });
|
|
12548
|
+
const driver = new BrowserDriver();
|
|
12549
|
+
let debited = false;
|
|
12550
|
+
try {
|
|
12551
|
+
const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
|
|
12552
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_transcribe), 402);
|
|
12553
|
+
debited = true;
|
|
12554
|
+
await driver.launch(await kernelLaunchOptsResidential());
|
|
12555
|
+
await driver.navigateTo(sourceUrl.href);
|
|
12556
|
+
const page = driver.getPage();
|
|
12557
|
+
const video = await extractFacebookOrganicVideoFromPage(page, sourceUrl.href, body.quality);
|
|
12558
|
+
const transcript = await transcribeFacebookVideoUrl(video.videoUrl, "# Facebook Organic Video Transcript");
|
|
12559
|
+
const result = {
|
|
12560
|
+
...video,
|
|
12561
|
+
videoDurationSec: video.durationSec,
|
|
12562
|
+
text: transcript.text,
|
|
12563
|
+
chunks: transcript.chunks,
|
|
12564
|
+
durationMs: transcript.durationMs,
|
|
12565
|
+
markdown: transcript.markdown
|
|
12566
|
+
};
|
|
12567
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_video_transcribe", status: "done", query: sourceUrl.href, resultCount: transcript.chunks.length, result });
|
|
12568
|
+
return c.json(result);
|
|
12569
|
+
} catch (err) {
|
|
12570
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
12571
|
+
if (debited) await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
|
|
12572
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_video_transcribe", status: "failed", query: sourceUrl.href, error: msg });
|
|
12573
|
+
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
12574
|
+
return c.json({ error: msg }, 503);
|
|
12575
|
+
}
|
|
12576
|
+
return c.json({ error: msg }, 500);
|
|
12577
|
+
} finally {
|
|
12578
|
+
await driver.close();
|
|
12579
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12580
|
+
}
|
|
12581
|
+
});
|
|
12379
12582
|
facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
12380
12583
|
const raw = await c.req.json().catch(() => ({}));
|
|
12381
12584
|
const parsed = FacebookSearchBodySchema.safeParse(raw);
|
|
@@ -14603,6 +14806,62 @@ ${chunkRows}` : "",
|
|
|
14603
14806
|
].filter(Boolean).join("\n");
|
|
14604
14807
|
return oneBlock(full);
|
|
14605
14808
|
}
|
|
14809
|
+
function formatFacebookVideoTranscribe(raw, input) {
|
|
14810
|
+
const parsed = parseData(raw);
|
|
14811
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
14812
|
+
const d = parsed.data;
|
|
14813
|
+
const text = d.text ?? "";
|
|
14814
|
+
const chunks = d.chunks ?? [];
|
|
14815
|
+
const wordCount = text.trim() ? text.trim().split(/\s+/).length : 0;
|
|
14816
|
+
const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
|
|
14817
|
+
const videoDuration = typeof d.videoDurationSec === "number" ? `${Math.round(d.videoDurationSec)}s` : "\u2014";
|
|
14818
|
+
const label = d.videoId ? `Facebook Organic Video \`${d.videoId}\`` : "Facebook Organic Video";
|
|
14819
|
+
const chunkRows = chunks.slice(0, 50).map((c) => {
|
|
14820
|
+
const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
|
|
14821
|
+
const mm = String(Math.floor(sec / 60)).padStart(2, "0");
|
|
14822
|
+
const ss = String(sec % 60).padStart(2, "0");
|
|
14823
|
+
return `| ${mm}:${ss} | ${cell(truncate(c.text, 120))} |`;
|
|
14824
|
+
}).join("\n");
|
|
14825
|
+
const full = [
|
|
14826
|
+
`# ${label} Transcript`,
|
|
14827
|
+
d.ownerName ? `**Owner:** ${d.ownerName}` : "",
|
|
14828
|
+
`**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${wordCount} words**`,
|
|
14829
|
+
d.pageUrl ? `**Page URL:** ${d.pageUrl}` : `**Page URL:** ${input.url}`,
|
|
14830
|
+
d.videoUrl ? `**Extracted MP4:** \`${d.videoUrl}\`` : "",
|
|
14831
|
+
`
|
|
14832
|
+
## Full Transcript
|
|
14833
|
+
${text}`,
|
|
14834
|
+
chunks.length ? `
|
|
14835
|
+
## Timestamped Chunks
|
|
14836
|
+
| Time | Text |
|
|
14837
|
+
|------|------|
|
|
14838
|
+
${chunkRows}` : "",
|
|
14839
|
+
`
|
|
14840
|
+
---
|
|
14841
|
+
\u{1F4A1} To download the extracted MP4, call the Facebook media endpoint with the extracted MP4 URL.`
|
|
14842
|
+
].filter(Boolean).join("\n");
|
|
14843
|
+
return {
|
|
14844
|
+
...oneBlock(full),
|
|
14845
|
+
structuredContent: {
|
|
14846
|
+
sourceUrl: d.sourceUrl ?? input.url,
|
|
14847
|
+
pageUrl: d.pageUrl ?? input.url,
|
|
14848
|
+
videoId: d.videoId ?? null,
|
|
14849
|
+
ownerName: d.ownerName ?? null,
|
|
14850
|
+
selectedQuality: d.selectedQuality ?? input.quality ?? "best",
|
|
14851
|
+
bitrate: typeof d.bitrate === "number" ? d.bitrate : null,
|
|
14852
|
+
videoDurationSec: typeof d.videoDurationSec === "number" ? d.videoDurationSec : null,
|
|
14853
|
+
videoUrl: d.videoUrl ?? "",
|
|
14854
|
+
wordCount,
|
|
14855
|
+
chunkCount: chunks.length,
|
|
14856
|
+
transcriptText: text,
|
|
14857
|
+
chunks: chunks.map((c) => ({
|
|
14858
|
+
startSec: Number.isFinite(c.timestamp[0]) ? c.timestamp[0] : 0,
|
|
14859
|
+
endSec: Number.isFinite(c.timestamp[1]) ? c.timestamp[1] : 0,
|
|
14860
|
+
text: c.text
|
|
14861
|
+
}))
|
|
14862
|
+
}
|
|
14863
|
+
};
|
|
14864
|
+
}
|
|
14606
14865
|
var import_node_fs3, import_node_os3, import_node_path5, reportSavingEnabled;
|
|
14607
14866
|
var init_mcp_response_formatter = __esm({
|
|
14608
14867
|
"src/mcp/mcp-response-formatter.ts"() {
|
|
@@ -19886,12 +20145,12 @@ var PACKAGE_VERSION;
|
|
|
19886
20145
|
var init_version = __esm({
|
|
19887
20146
|
"src/version.ts"() {
|
|
19888
20147
|
"use strict";
|
|
19889
|
-
PACKAGE_VERSION = "0.2.
|
|
20148
|
+
PACKAGE_VERSION = "0.2.15";
|
|
19890
20149
|
}
|
|
19891
20150
|
});
|
|
19892
20151
|
|
|
19893
20152
|
// src/mcp/mcp-tool-schemas.ts
|
|
19894
|
-
var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, CreditsInfoInputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
20153
|
+
var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, CreditsInfoInputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
19895
20154
|
var init_mcp_tool_schemas = __esm({
|
|
19896
20155
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
19897
20156
|
"use strict";
|
|
@@ -19948,6 +20207,10 @@ var init_mcp_tool_schemas = __esm({
|
|
|
19948
20207
|
FacebookAdTranscribeInputSchema = {
|
|
19949
20208
|
videoUrl: import_zod26.z.string().url().describe("Facebook CDN video URL from a facebook_page_intel result")
|
|
19950
20209
|
};
|
|
20210
|
+
FacebookVideoTranscribeInputSchema = {
|
|
20211
|
+
url: import_zod26.z.string().url().describe("Organic Facebook reel, video, watch, or share URL. The tool renders the page, extracts the best public Facebook CDN MP4 URL, then transcribes it."),
|
|
20212
|
+
quality: import_zod26.z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
|
|
20213
|
+
};
|
|
19951
20214
|
MapsPlaceIntelInputSchema = {
|
|
19952
20215
|
businessName: import_zod26.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
19953
20216
|
location: import_zod26.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
|
|
@@ -20313,6 +20576,24 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20313
20576
|
variations: import_zod26.z.number().int().nullable()
|
|
20314
20577
|
}))
|
|
20315
20578
|
};
|
|
20579
|
+
FacebookVideoTranscribeOutputSchema = {
|
|
20580
|
+
sourceUrl: import_zod26.z.string().url(),
|
|
20581
|
+
pageUrl: import_zod26.z.string().url(),
|
|
20582
|
+
videoId: NullableString,
|
|
20583
|
+
ownerName: NullableString,
|
|
20584
|
+
selectedQuality: import_zod26.z.string(),
|
|
20585
|
+
bitrate: import_zod26.z.number().int().nullable(),
|
|
20586
|
+
videoDurationSec: import_zod26.z.number().nullable(),
|
|
20587
|
+
videoUrl: import_zod26.z.string().url(),
|
|
20588
|
+
wordCount: import_zod26.z.number().int().min(0),
|
|
20589
|
+
chunkCount: import_zod26.z.number().int().min(0),
|
|
20590
|
+
transcriptText: import_zod26.z.string(),
|
|
20591
|
+
chunks: import_zod26.z.array(import_zod26.z.object({
|
|
20592
|
+
startSec: import_zod26.z.number(),
|
|
20593
|
+
endSec: import_zod26.z.number(),
|
|
20594
|
+
text: import_zod26.z.string()
|
|
20595
|
+
}))
|
|
20596
|
+
};
|
|
20316
20597
|
CreditsInfoInputSchema = {
|
|
20317
20598
|
item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
|
|
20318
20599
|
includeLedger: import_zod26.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
@@ -20820,6 +21101,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
20820
21101
|
inputSchema: FacebookAdTranscribeInputSchema,
|
|
20821
21102
|
annotations: liveWebToolAnnotations("Facebook Ad Transcription")
|
|
20822
21103
|
}, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
|
|
21104
|
+
server.registerTool("facebook_video_transcribe", {
|
|
21105
|
+
title: "Facebook Organic Video Transcription",
|
|
21106
|
+
description: withReportNote("Transcribe audio from an organic Facebook reel, video, watch, or share URL. Renders the Facebook page, extracts the best public Facebook CDN MP4 URL from the page state, then returns the full transcript, timestamped chunks, and extracted MP4 URL."),
|
|
21107
|
+
inputSchema: FacebookVideoTranscribeInputSchema,
|
|
21108
|
+
outputSchema: FacebookVideoTranscribeOutputSchema,
|
|
21109
|
+
annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
|
|
21110
|
+
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
20823
21111
|
server.registerTool("maps_place_intel", {
|
|
20824
21112
|
title: "Google Maps Business Profile Details",
|
|
20825
21113
|
description: withReportNote('Extract Google Maps business intelligence for one known/named business: rating, review count, category, address, phone, website, hours, booking URL, review histogram, review topics, about attributes, entity IDs, and optional review cards. Do not use this for category searches, local market prospect lists, or requests for multiple GMB/GBP profiles; use maps_search first for those. Split business name from location (e.g. "Elite Roofing Denver CO" => businessName "Elite Roofing", location "Denver, CO"). Pass includeReviews true when the user asks for reviews/customer pain.'),
|
|
@@ -20968,6 +21256,9 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
20968
21256
|
facebookAdTranscribe(input) {
|
|
20969
21257
|
return this.call("/facebook/transcribe", input);
|
|
20970
21258
|
}
|
|
21259
|
+
facebookVideoTranscribe(input) {
|
|
21260
|
+
return this.call("/facebook/video-transcribe", input, this.httpTimeoutOverrideMs ?? 24e4);
|
|
21261
|
+
}
|
|
20971
21262
|
mapsPlaceIntel(input) {
|
|
20972
21263
|
return this.call("/maps/place", input);
|
|
20973
21264
|
}
|