mcp-scraper 0.2.14 → 0.2.16

package/dist/{server-7QP6HQJJ.js → server-KSEQLZNP.js}

@@ -100,7 +100,7 @@ import {
   harvestTimeoutBudget,
   liveWebToolAnnotations,
   outputBaseDir
-} from "./chunk-IWQTNY2E.js";
+} from "./chunk-FMC3V54I.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -114,7 +114,7 @@ import {
   workflowDefinition,
   workflowOutputBaseDir
 } from "./chunk-F44RBOJ5.js";
-import "./chunk-ROCXJOVY.js";
+import "./chunk-KE4JZDLV.js";
 
 // src/api/outbound-sanitize.ts
 var KEY_RENAMES = {
@@ -9189,6 +9189,127 @@ function advertisersFromResults(results, maxResults) {
   return [...byPage.values()].map((e) => ({ pageName: e.pageName, pageId: e.pageId, sampleLibraryId: e.sampleLibraryId, adCount: Math.max(e.maxCollation, e.resultCount) })).sort((a, b) => b.adCount - a.adCount).slice(0, maxResults);
 }
 
+// src/extractor/FacebookOrganicVideoExtractor.ts
+function htmlDecode(value) {
+  return value.replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/&#039;/g, "'").replace(/&lt;/g, "<").replace(/&gt;/g, ">");
+}
+function decodeEscapedValue(raw) {
+  let out = raw;
+  try {
+    out = JSON.parse(`"${raw.replace(/"/g, '\\"')}"`);
+  } catch {
+    out = raw.replace(/\\u0025/g, "%").replace(/\\u0026/g, "&").replace(/\\u003D/g, "=").replace(/\\u003d/g, "=").replace(/\\"/g, '"').replace(/\\\//g, "/");
+  }
+  return htmlDecode(out);
+}
+function facebookVideoIdFromUrl(url) {
+  try {
+    const parsed = new URL(url);
+    const path5 = parsed.pathname;
+    const direct = path5.match(/\/(?:reel|videos|watch)\/(?:[^/]+\/)?(\d{8,})/i);
+    if (direct) return direct[1];
+    const queryVideoId = parsed.searchParams.get("v") ?? parsed.searchParams.get("video_id");
+    if (queryVideoId && /^\d{8,}$/.test(queryVideoId)) return queryVideoId;
+    const numericTail = path5.match(/\/(\d{8,})\/?$/);
+    if (numericTail) return numericTail[1];
+  } catch {
+  }
+  return null;
+}
+function parseEfg(url) {
+  try {
+    const efg = new URL(url).searchParams.get("efg");
+    if (!efg) return null;
+    return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
+  } catch {
+    return null;
+  }
+}
+function manifestDurationSec(htmlWindow) {
+  const decoded = decodeEscapedValue(htmlWindow);
+  const match = decoded.match(/mediaPresentationDuration="PT([\d.]+)S"/);
+  return match ? Number(match[1]) : null;
+}
+function metadataContent(html, key) {
+  const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const re = new RegExp(`<meta\\s+[^>]*(?:property|name)=["']${escaped}["'][^>]*content=["']([^"']+)["'][^>]*>`, "i");
+  const m = html.match(re);
+  return m ? htmlDecode(m[1]).trim() : null;
+}
+function ownerNameFromHtml(html) {
+  const ogTitle = metadataContent(html, "og:title");
+  const pipe = ogTitle?.split("|").map((s) => s.trim()).filter(Boolean);
+  if (pipe && pipe.length > 1) return pipe[pipe.length - 1] ?? null;
+  return null;
+}
+function extractFacebookOrganicVideo(html, pageUrl, sourceUrl = pageUrl, quality = "best") {
+  const videoId = facebookVideoIdFromUrl(pageUrl) ?? facebookVideoIdFromUrl(sourceUrl);
+  const manifestIndex = videoId ? html.indexOf(`dash_mpd_debug.mpd?v=${videoId}`) : -1;
+  const searchWindow = manifestIndex >= 0 ? html.slice(Math.max(0, manifestIndex - 5e4), Math.min(html.length, manifestIndex + 5e4)) : html;
+  const targetDuration = manifestDurationSec(searchWindow);
+  const candidates = [];
+  const seen = /* @__PURE__ */ new Set();
+  const re = /browser_native_(sd|hd)_url\\?"\s*:\s*\\?"([^"<]+?)\\?"/g;
+  let match;
+  while ((match = re.exec(searchWindow)) !== null) {
+    const url = decodeEscapedValue(match[2]);
+    if (!url || seen.has(url)) continue;
+    seen.add(url);
+    const parsed = parseEfg(url);
+    let bitrate = null;
+    try {
+      const value = new URL(url).searchParams.get("bitrate");
+      bitrate = value ? Number(value) : null;
+    } catch {
+    }
+    candidates.push({
+      url,
+      quality: match[1],
+      bitrate: Number.isFinite(bitrate) ? bitrate : null,
+      durationSec: typeof parsed?.duration_s === "number" ? parsed.duration_s : null,
+      assetId: typeof parsed?.xpv_asset_id === "number" ? parsed.xpv_asset_id : null,
+      vencodeTag: typeof parsed?.vencode_tag === "string" ? parsed.vencode_tag : null
+    });
+  }
+  const targetCandidates = candidates.filter((candidate) => {
+    if (!targetDuration || candidate.durationSec == null) return true;
+    return Math.abs(candidate.durationSec - targetDuration) <= 2;
+  });
+  const selectable = targetCandidates.length ? targetCandidates : candidates;
+  const selected = [...selectable].filter((candidate) => quality === "best" || candidate.quality === quality).sort((a, b) => {
+    const aq = a.quality === "hd" ? 1 : 0;
+    const bq = b.quality === "hd" ? 1 : 0;
+    if (quality === "best" && aq !== bq) return bq - aq;
+    return (b.bitrate ?? 0) - (a.bitrate ?? 0);
+  })[0];
+  if (!selected) {
+    throw new Error("No progressive Facebook video URL was found in the rendered page state");
+  }
+  return {
+    sourceUrl,
+    pageUrl,
+    videoId,
+    title: metadataContent(html, "og:title") ?? metadataContent(html, "twitter:title"),
+    description: metadataContent(html, "og:description") ?? metadataContent(html, "description") ?? metadataContent(html, "twitter:description"),
+    ownerName: ownerNameFromHtml(html),
+    videoUrl: selected.url,
+    selectedQuality: selected.quality,
+    bitrate: selected.bitrate,
+    durationSec: targetDuration ?? selected.durationSec,
+    assetId: selected.assetId,
+    vencodeTag: selected.vencodeTag,
+    candidates: selectable,
+    extractedAt: (/* @__PURE__ */ new Date()).toISOString()
+  };
+}
+async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "best") {
+  await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
+  });
+  await page.waitForTimeout(1500);
+  const html = await page.content();
+  return extractFacebookOrganicVideo(html, page.url(), sourceUrl, quality);
+}
+
 // src/api/facebook-ad-routes.ts
 import { fal as fal2 } from "@fal-ai/client";
 var FacebookAdBodySchema = z13.object({
@@ -9208,6 +9329,10 @@ var FacebookPageIntelBodySchema = z13.object({
 var FacebookTranscribeBodySchema = z13.object({
   videoUrl: z13.string().trim().min(1, "videoUrl is required")
 });
+var FacebookVideoTranscribeBodySchema = z13.object({
+  url: z13.string().trim().min(1, "url is required"),
+  quality: z13.enum(["best", "hd", "sd"]).default("best")
+});
 var FacebookSearchBodySchema = z13.object({
   query: z13.string().trim().min(1, "query is required"),
   country: z13.string().trim().toUpperCase().optional(),
@@ -9220,6 +9345,39 @@ var FacebookMediaBodySchema = z13.object({
 function invalidRequest(message) {
   return { error_code: "invalid_request", message };
 }
+function isAllowedFacebookOrganicUrl(url) {
+  const hostname = url.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
+  return hostname === "facebook.com" || hostname === "fb.watch";
+}
+function facebookTranscriptMarkdown(title, text, chunks, durationMs) {
+  const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
+  const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
+  if (chunks.length) {
+    lines.push("## Timestamped Segments", "");
+    for (const ch of chunks) {
+      lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
+    }
+  }
+  return lines.join("\n");
+}
+async function transcribeFacebookVideoUrl(videoUrl, markdownTitle = "# Facebook Video Transcript") {
+  const startMs = Date.now();
+  const result = await fal2.subscribe("fal-ai/wizper", {
+    input: { audio_url: videoUrl, task: "transcribe", language: "en" },
+    logs: false,
+    pollInterval: 3e3
+  });
+  const data = result.data;
+  const text = data.text ?? "";
+  const chunks = data.chunks ?? [];
+  const durationMs = Date.now() - startMs;
+  return {
+    text,
+    chunks,
+    durationMs,
+    markdown: facebookTranscriptMarkdown(markdownTitle, text, chunks, durationMs)
+  };
+}
 async function detectSoftBlock(driver) {
   const page = driver.getPage();
   const bodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
@@ -9363,26 +9521,9 @@ facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
     const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, videoUrl);
     if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_transcribe), 402);
     debited = true;
-    const startMs = Date.now();
-    const result = await fal2.subscribe("fal-ai/wizper", {
-      input: { audio_url: videoUrl, task: "transcribe", language: "en" },
-      logs: false,
-      pollInterval: 3e3
-    });
-    const data = result.data;
-    const text = data.text ?? "";
-    const chunks = data.chunks ?? [];
-    const durationMs = Date.now() - startMs;
-    const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
-    const lines = ["# Facebook Ad Transcript", "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
-    if (chunks.length) {
-      lines.push("## Timestamped Segments", "");
-      for (const ch of chunks) {
-        lines.push(`**[${fmtTs2(ch.timestamp[0])} \u2192 ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
-      }
-    }
-    await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: chunks.length, result: { text, chunks, durationMs } });
-    return c.json({ text, chunks, durationMs, markdown: lines.join("\n") });
+    const transcript = await transcribeFacebookVideoUrl(videoUrl, "# Facebook Ad Transcript");
+    await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
+    return c.json(transcript);
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
     if (debited) await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
@@ -9392,6 +9533,62 @@ facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
     await releaseConcurrencyGate(gate.lockId);
   }
 });
+facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
+  const raw = await c.req.json().catch(() => ({}));
+  const parsed = FacebookVideoTranscribeBodySchema.safeParse(raw);
+  if (!parsed.success) {
+    return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
+  }
+  const body = parsed.data;
+  const urlCheck = await validatePublicHttpUrl(body.url, { field: "url", requireHttps: false });
+  if (urlCheck.error) {
+    return c.json(invalidRequest(urlCheck.error), 400);
+  }
+  const sourceUrl = urlCheck.parsed;
+  if (!isAllowedFacebookOrganicUrl(sourceUrl)) {
+    return c.json(invalidRequest("url must be a Facebook organic video, reel, or share URL"), 400);
+  }
+  const fbUser = c.get("user");
+  const gate = await acquireConcurrencyGate(fbUser, "facebook_video_transcribe", {
+    reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
+    metadata: { url: sourceUrl.href }
+  });
+  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
+  fal2.config({ credentials: process.env.FAL_KEY });
+  const driver = new BrowserDriver();
+  let debited = false;
+  try {
+    const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
+    if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_transcribe), 402);
+    debited = true;
+    await driver.launch(await kernelLaunchOptsResidential());
+    await driver.navigateTo(sourceUrl.href);
+    const page = driver.getPage();
+    const video = await extractFacebookOrganicVideoFromPage(page, sourceUrl.href, body.quality);
+    const transcript = await transcribeFacebookVideoUrl(video.videoUrl, "# Facebook Organic Video Transcript");
+    const result = {
+      ...video,
+      videoDurationSec: video.durationSec,
+      text: transcript.text,
+      chunks: transcript.chunks,
+      durationMs: transcript.durationMs,
+      markdown: transcript.markdown
+    };
+    await logRequestEvent({ userId: fbUser.id, source: "facebook_video_transcribe", status: "done", query: sourceUrl.href, resultCount: transcript.chunks.length, result });
+    return c.json(result);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    if (debited) await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
+    await logRequestEvent({ userId: fbUser.id, source: "facebook_video_transcribe", status: "failed", query: sourceUrl.href, error: msg });
+    if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
+      return c.json({ error: msg }, 503);
+    }
+    return c.json({ error: msg }, 500);
+  } finally {
+    await driver.close();
+    await releaseConcurrencyGate(gate.lockId);
+  }
+});
 facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
   const raw = await c.req.json().catch(() => ({}));
   const parsed = FacebookSearchBodySchema.safeParse(raw);
@@ -13602,7 +13799,7 @@ var sessionCookieOptions = {
   sameSite: "Strict"
 };
 function configuredOrigins() {
-  const origins = /* @__PURE__ */ new Set(["https://mcpscraper.dev"]);
+  const origins = /* @__PURE__ */ new Set(["https://mcpscraper.dev", "https://www.mcpscraper.dev"]);
   for (const raw of (process.env.ALLOWED_ORIGINS ?? process.env.APP_ORIGIN ?? "").split(",")) {
     const trimmed = raw.trim();
     if (trimmed) origins.add(trimmed);
@@ -14536,4 +14733,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-7QP6HQJJ.js.map
+//# sourceMappingURL=server-KSEQLZNP.js.map