mcp-scraper 0.4.13 → 0.4.14

package/dist/bin/api-server.cjs

@@ -13705,13 +13705,12 @@ function parseTimedtextXml(xml) {
   }
   return results;
 }
-function isBotBlockStatus(status) {
-  return status === "LOGIN_REQUIRED" || status === "AGE_VERIFICATION_REQUIRED";
-}
 async function probePlayer(page, videoId) {
   return page.evaluate(
     async ({ vid, clients }) => {
       let lastStatus = null;
+      let sawBotBlock = false;
+      let sawAccessible = false;
       for (const client2 of clients) {
         const resp = await fetch("/youtubei/v1/player?prettyPrint=false", {
           method: "POST",
@@ -13727,97 +13726,24 @@ async function probePlayer(page, videoId) {
         }
         const data = await resp.json();
         lastStatus = data?.playabilityStatus?.status ?? null;
-        const audioFormats = (data?.streamingData?.adaptiveFormats ?? []).filter((f) => typeof f.url === "string" && f.url && typeof f.mimeType === "string" && f.mimeType.startsWith("audio/")).sort((a, b) => (Number(a.contentLength) || Infinity) - (Number(b.contentLength) || Infinity));
-        const pick = audioFormats[0] ?? null;
-        let captions = null;
+        if (lastStatus === "LOGIN_REQUIRED" || lastStatus === "AGE_VERIFICATION_REQUIRED") sawBotBlock = true;
+        else if (lastStatus === "OK") sawAccessible = true;
         const tracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
         if (tracks.length > 0) {
           const track = tracks.find((t) => t.languageCode === "en") ?? tracks[0];
           const sep = track.baseUrl.includes("?") ? "&" : "?";
           const xmlResp = await fetch(`${track.baseUrl}${sep}fmt=json3`).catch(() => null);
-          if (xmlResp?.ok) captions = await xmlResp.text();
+          if (xmlResp?.ok) {
+            const captions = await xmlResp.text();
+            if (captions) return { captions, status: lastStatus, escalate: false };
+          }
         }
-        if (!captions && !pick) continue;
-        return {
-          captions,
-          status: lastStatus,
-          audio: pick ? {
-            url: pick.url,
-            mimeType: pick.mimeType,
-            bitrate: pick.bitrate ?? null,
-            contentLength: pick.contentLength ? Number(pick.contentLength) || null : null,
-            approxDurationMs: pick.approxDurationMs ? Number(pick.approxDurationMs) || null : null
-          } : null
-        };
       }
-      return { captions: null, audio: null, status: lastStatus };
+      return { captions: null, status: lastStatus, escalate: sawBotBlock && !sawAccessible };
     },
     { vid: videoId, clients: INNERTUBE_CLIENTS }
   ).catch(() => null);
 }
-async function downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, audio) {
-  if (audio.contentLength && audio.contentLength > MAX_AUDIO_BYTES) {
-    console.warn(`[CaptionFetcher] audio stream ${audio.contentLength} bytes exceeds ${MAX_AUDIO_BYTES} byte cap`);
-    return null;
-  }
-  const sep = audio.url.includes("?") ? "&" : "?";
-  const url = audio.contentLength ? `${audio.url}${sep}range=0-${audio.contentLength - 1}` : audio.url;
-  const result = await page.evaluate(async (u) => {
-    try {
-      const resp = await fetch(u);
-      if (!resp.ok) return { ok: false, error: `http_${resp.status}` };
-      const blob = await resp.blob();
-      const dataUrl = await new Promise((resolve, reject) => {
-        const reader = new FileReader();
-        reader.onload = () => resolve(reader.result);
-        reader.onerror = () => reject(reader.error);
-        reader.readAsDataURL(blob);
-      });
-      return { ok: true, base64: dataUrl.slice(dataUrl.indexOf(",") + 1) };
-    } catch (e) {
-      return { ok: false, error: String(e) };
-    }
-  }, url).catch((e) => ({ ok: false, error: String(e) }));
-  if (!result.ok || !result.base64) {
-    console.warn(`[CaptionFetcher] audio fetch failed (${result.error ?? "unknown"}) for a ${audio.contentLength ?? 0} byte stream`);
-    return null;
-  }
-  const bytes = Buffer.from(result.base64, "base64");
-  if (audio.contentLength && bytes.length < audio.contentLength) {
-    console.warn(`[CaptionFetcher] partial audio: ${bytes.length} of ${audio.contentLength} bytes`);
-    return null;
-  }
-  return { bytes, mimeType: audio.mimeType.split(";")[0].trim() };
-}
-async function transcribeWithWizper(videoId, audio, startMs) {
-  const falKey = process.env.FAL_KEY;
-  if (!falKey) return null;
-  try {
-    import_client4.fal.config({ credentials: falKey });
-    const ext = audio.mimeType.includes("webm") ? "webm" : audio.mimeType.includes("mp4") ? "m4a" : "audio";
-    const audioFile = new File([new Uint8Array(audio.bytes)], `audio.${ext}`, { type: audio.mimeType });
-    const uploadedUrl = await import_client4.fal.storage.upload(audioFile);
-    const subscribeStartMs = Date.now();
-    const result = await import_client4.fal.subscribe("fal-ai/wizper", {
-      input: { audio_url: uploadedUrl, task: "transcribe", language: "en", chunk_level: "segment" },
-      logs: false,
-      pollInterval: 3e3
-    });
-    const subscribeDurationMs = Date.now() - subscribeStartMs;
-    const data = result.data;
-    const chunks = (data.chunks ?? []).map((c) => ({
-      timestamp: c.timestamp,
-      text: c.text.trim()
-    }));
-    void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: subscribeDurationMs / 1e3, unitType: "compute_sec" });
-    const text = data.text ?? chunks.map((c) => c.text).join(" ");
-    if (!text) return null;
-    return { videoId, text, chunks, durationMs: Date.now() - startMs, method: "browser-whisper" };
-  } catch (err) {
-    console.warn(`[CaptionFetcher] wizper transcription failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
-    return null;
-  }
-}
 async function resolveResidentialProxy(kernelApiKey) {
   try {
     const resolved = await resolveKernelProxyId({
@@ -13833,14 +13759,13 @@ async function resolveResidentialProxy(kernelApiKey) {
     return {};
   }
 }
-async function probeAndExtract(videoId, opts) {
+async function probeCaptions(videoId, kernelProxyIdOverride) {
   const kernelApiKey = browserServiceApiKey();
   if (!kernelApiKey) return null;
-  const kernelProxyId = opts?.kernelProxyId ?? browserServiceProxyId();
+  const kernelProxyId = kernelProxyIdOverride ?? browserServiceProxyId();
   const driver = new BrowserDriver();
   return runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: "browser-innertube" }, async () => {
     let entries = [];
-    let audio = null;
     let retryWithFreshIp = false;
     try {
       await driver.launch({ kernelApiKey, kernelProxyId, headless: true, headlessMode: "headless", viewport: { width: 1280, height: 800 }, locale: "en-US" });
@@ -13852,69 +13777,51 @@ async function probeAndExtract(videoId, opts) {
       );
       const probe = await probePlayer(page, videoId);
       if (!probe) return null;
-      if (probe.captions && !opts?.skipCaptions) {
+      if (probe.captions) {
         entries = parseTimedtextJson3(probe.captions);
         if (entries.length === 0) entries = parseTimedtextXml(probe.captions);
       }
       if (entries.length === 0) {
-        if (!probe.audio) {
-          retryWithFreshIp = isBotBlockStatus(probe.status);
-          console.warn(`[CaptionFetcher] no captions and no audio stream for ${videoId} (playability=${probe.status ?? "unknown"}, proxied=${Boolean(kernelProxyId)})`);
-          return { entries, audio: null, retryWithFreshIp };
-        }
-        audio = await downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, probe.audio);
-        if (!audio) return { entries, audio: null, retryWithFreshIp: true };
+        retryWithFreshIp = probe.escalate;
+        console.warn(`[CaptionFetcher] no caption track for ${videoId} (playability=${probe.status ?? "unknown"}, proxied=${Boolean(kernelProxyId)}, escalate=${probe.escalate})`);
       }
     } catch (err) {
       console.warn(`[CaptionFetcher] kernel session failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
       return null;
     } finally {
-      const servedSubOp = audio ? "browser-whisper" : "browser-innertube";
-      await runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: servedSubOp }, () => driver.close());
+      await driver.close();
     }
-    return { entries, audio, retryWithFreshIp };
+    return { entries, retryWithFreshIp };
   });
 }
-async function extractWithProxyEscalation(videoId, opts) {
-  const direct = await probeAndExtract(videoId, opts);
-  if (direct && (direct.entries.length > 0 || direct.audio)) return direct;
-  const shouldEscalate = direct === null || direct.retryWithFreshIp;
-  if (!shouldEscalate) return direct;
-  const kernelApiKey = browserServiceApiKey();
-  if (!kernelApiKey) return direct;
-  const { kernelProxyId, disposableProxyId } = await resolveResidentialProxy(kernelApiKey);
-  if (!kernelProxyId) return direct;
-  try {
-    console.warn(`[CaptionFetcher] ${videoId}: retrying through residential proxy after bot block`);
-    const viaProxy = await probeAndExtract(videoId, { ...opts, kernelProxyId });
-    return viaProxy ?? direct;
-  } finally {
-    if (disposableProxyId) {
-      await deleteKernelProxyId(kernelApiKey, disposableProxyId).catch(
-        (err) => console.warn(`[CaptionFetcher] failed to delete disposable proxy: ${err instanceof Error ? err.message : String(err)}`)
-      );
-    }
-  }
-}
 async function fetchViaKernelSession(videoId) {
   const start = Date.now();
-  const outcome = await extractWithProxyEscalation(videoId);
-  if (!outcome) return null;
-  if (outcome.entries.length > 0) {
-    const chunks = outcome.entries.map((e) => ({
-      timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
-      text: e.text
-    }));
-    const text = chunks.map((c) => c.text).join(" ");
-    return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
-  }
-  if (outcome.audio) {
-    return runWithCostContext(
-      { ...currentCostContext(), op: "yt_transcription", subOp: "browser-whisper" },
-      () => transcribeWithWizper(videoId, outcome.audio, start)
-    );
+  let outcome = await probeCaptions(videoId);
+  if (!outcome || outcome.entries.length === 0 && outcome.retryWithFreshIp) {
+    const kernelApiKey = browserServiceApiKey();
+    if (kernelApiKey) {
+      const { kernelProxyId, disposableProxyId } = await resolveResidentialProxy(kernelApiKey);
+      if (kernelProxyId) {
+        try {
+          console.warn(`[CaptionFetcher] ${videoId}: retrying captions through residential proxy after bot block`);
+          outcome = await probeCaptions(videoId, kernelProxyId) ?? outcome;
+        } finally {
+          if (disposableProxyId) {
+            await deleteKernelProxyId(kernelApiKey, disposableProxyId).catch(
+              (err) => console.warn(`[CaptionFetcher] failed to delete disposable proxy: ${err instanceof Error ? err.message : String(err)}`)
+            );
+          }
+        }
+      }
+    }
   }
-  return null;
+  if (!outcome || outcome.entries.length === 0) return null;
+  const chunks = outcome.entries.map((e) => ({
+    timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
+    text: e.text
+  }));
+  const text = chunks.map((c) => c.text).join(" ");
+  return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
 }
 async function fetchViaResolvedAudio(videoId) {
   if (!mediaResolverConfigured()) return null;
@@ -13964,7 +13871,7 @@ async function fetchCaptions(videoId) {
     `Could not transcribe ${videoId}: it has no caption track and its audio could not be retrieved.`
   );
 }
-var import_client4, INNERTUBE_CLIENTS, MAX_AUDIO_BYTES, YT_RESIDENTIAL_LOCATION;
+var INNERTUBE_CLIENTS, YT_RESIDENTIAL_LOCATION;
 var init_CaptionFetcher = __esm({
   "src/youtube/CaptionFetcher.ts"() {
     "use strict";
@@ -13973,7 +13880,6 @@ var init_CaptionFetcher = __esm({
     init_kernel_proxy_resolver();
     init_audio_resolver();
     init_media_transcription();
-    import_client4 = require("@fal-ai/client");
     init_cost_telemetry();
     init_cost_context();
     INNERTUBE_CLIENTS = [
@@ -13981,7 +13887,6 @@ var init_CaptionFetcher = __esm({
       { name: "IOS", version: "20.10.4", userAgent: "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)" },
       { name: "ANDROID_VR", version: "1.62.27", userAgent: "com.google.android.apps.youtube.vr.oculus/1.62.27 (Linux; U; Android 12L) gzip" }
     ];
-    MAX_AUDIO_BYTES = 96 * 1024 * 1024;
     YT_RESIDENTIAL_LOCATION = process.env.YOUTUBE_PROXY_LOCATION ?? "Austin, Texas";
   }
 });
@@ -15560,7 +15465,7 @@ async function kernelLaunchOptsResidential() {
   }
   return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
 }
-var import_hono5, import_zod16, import_client5, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
+var import_hono5, import_zod16, import_client4, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
 var init_facebook_ad_routes = __esm({
   "src/api/facebook-ad-routes.ts"() {
     "use strict";
@@ -15576,7 +15481,7 @@ var init_facebook_ad_routes = __esm({
     init_FacebookOrganicVideoExtractor();
     init_kernel_proxy_resolver();
     init_schemas3();
-    import_client5 = require("@fal-ai/client");
+    import_client4 = require("@fal-ai/client");
     init_api_auth();
     init_url_utils();
     init_concurrency_gates();
@@ -15719,7 +15624,7 @@ var init_facebook_ad_routes = __esm({
         metadata: { videoUrl }
       });
       if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-      import_client5.fal.config({ credentials: process.env.FAL_KEY });
+      import_client4.fal.config({ credentials: process.env.FAL_KEY });
       const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
       let debited = false;
       try {
@@ -15763,7 +15668,7 @@ var init_facebook_ad_routes = __esm({
         metadata: { url: sourceUrl.href }
       });
       if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-      import_client5.fal.config({ credentials: process.env.FAL_KEY });
+      import_client4.fal.config({ credentials: process.env.FAL_KEY });
       const driver = new BrowserDriver();
       const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
       let debited = false;
@@ -16144,7 +16049,7 @@ async function kernelLaunchOptsResidential2() {
 function isBlockedMessage(msg) {
   return msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha");
 }
-var import_hono6, import_zod17, import_client6, GoogleAdsSearchBodySchema, GoogleAdsPageIntelBodySchema, GoogleAdsTranscribeBodySchema, googleAdsApp;
+var import_hono6, import_zod17, import_client5, GoogleAdsSearchBodySchema, GoogleAdsPageIntelBodySchema, GoogleAdsTranscribeBodySchema, googleAdsApp;
 var init_google_ads_routes = __esm({
   "src/api/google-ads-routes.ts"() {
     "use strict";
@@ -16157,7 +16062,7 @@ var init_google_ads_routes = __esm({
     init_GoogleAdsExtractor();
     init_kernel_proxy_resolver();
     init_schemas3();
-    import_client6 = require("@fal-ai/client");
+    import_client5 = require("@fal-ai/client");
     init_api_auth();
     init_url_utils();
     init_concurrency_gates();
@@ -16268,7 +16173,7 @@ var init_google_ads_routes = __esm({
       const user = c.get("user");
       const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
       if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-      import_client6.fal.config({ credentials: process.env.FAL_KEY });
+      import_client5.fal.config({ credentials: process.env.FAL_KEY });
       const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
       let debited = false;
       try {
@@ -27399,7 +27304,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.4.13";
+    PACKAGE_VERSION = "0.4.14";
   }
 });