mcp-scraper 0.4.13 → 0.5.0

package/dist/bin/api-server.cjs

@@ -8013,16 +8013,16 @@ var init_rates = __esm({
       browser_minute: 12e3,
       reddit_thread: 3e3,
       reddit_comment: 200,
-      video_analysis: 666667,
+      video_analysis: 666700,
       trustpilot_reviews: 500,
       trustpilot_review: 100,
       g2_reviews: 500,
-      g2_review: 150,
+      g2_review: 200,
       diff_page: 100
     };
     MC_PER_BROWSER_MS = MC_COSTS.browser_minute / 6e4;
     BROWSER_OPEN_MIN_BALANCE_MC = 1e3;
-    MC_PER_CREDIT = 1e3;
+    MC_PER_CREDIT = 100;
     CREDIT_COST_CATALOG = [
       {
         key: "serp",
@@ -8242,8 +8242,8 @@ var init_rates = __esm({
     CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
     SUBSCRIPTION_TIERS = {
       "price_1TmiHRS8aAcsk3TGwmSNfNIa": { tier: "starter", label: "Starter", price_id: "price_1TmiHRS8aAcsk3TGwmSNfNIa", monthly_usd: 12, credits_mc: 8e6, concurrency: 3, intro_coupon: "mcp-starter-1dollar-intro-12" },
-      "price_1Tmg1nS8aAcsk3TGe8zcnGTM": { tier: "growth", label: "Growth", price_id: "price_1Tmg1nS8aAcsk3TGe8zcnGTM", monthly_usd: 100, credits_mc: 8e7, concurrency: 10, intro_coupon: null },
-      "price_1Tmg1nS8aAcsk3TGsZw34iXS": { tier: "scale", label: "Scale", price_id: "price_1Tmg1nS8aAcsk3TGsZw34iXS", monthly_usd: 250, credits_mc: 24e7, concurrency: 20, intro_coupon: null }
+      "price_1TrgihS8aAcsk3TG5cglrq4D": { tier: "growth", label: "Growth", price_id: "price_1TrgihS8aAcsk3TG5cglrq4D", monthly_usd: 40, credits_mc: 26666700, concurrency: 10, intro_coupon: null },
+      "price_1TrgihS8aAcsk3TG4HnG4gbY": { tier: "scale", label: "Scale", price_id: "price_1TrgihS8aAcsk3TG4HnG4gbY", monthly_usd: 100, credits_mc: 8e7, concurrency: 20, intro_coupon: null }
     };
     SUBSCRIPTION_TIER_BY_KEY = Object.fromEntries(
       Object.values(SUBSCRIPTION_TIERS).map((t) => [t.tier, t])
@@ -13705,13 +13705,12 @@ function parseTimedtextXml(xml) {
   }
   return results;
 }
-function isBotBlockStatus(status) {
-  return status === "LOGIN_REQUIRED" || status === "AGE_VERIFICATION_REQUIRED";
-}
 async function probePlayer(page, videoId) {
   return page.evaluate(
     async ({ vid, clients }) => {
       let lastStatus = null;
+      let sawBotBlock = false;
+      let sawAccessible = false;
       for (const client2 of clients) {
         const resp = await fetch("/youtubei/v1/player?prettyPrint=false", {
           method: "POST",
@@ -13727,97 +13726,24 @@ async function probePlayer(page, videoId) {
         }
         const data = await resp.json();
         lastStatus = data?.playabilityStatus?.status ?? null;
-        const audioFormats = (data?.streamingData?.adaptiveFormats ?? []).filter((f) => typeof f.url === "string" && f.url && typeof f.mimeType === "string" && f.mimeType.startsWith("audio/")).sort((a, b) => (Number(a.contentLength) || Infinity) - (Number(b.contentLength) || Infinity));
-        const pick = audioFormats[0] ?? null;
-        let captions = null;
+        if (lastStatus === "LOGIN_REQUIRED" || lastStatus === "AGE_VERIFICATION_REQUIRED") sawBotBlock = true;
+        else if (lastStatus === "OK") sawAccessible = true;
         const tracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
         if (tracks.length > 0) {
           const track = tracks.find((t) => t.languageCode === "en") ?? tracks[0];
           const sep = track.baseUrl.includes("?") ? "&" : "?";
           const xmlResp = await fetch(`${track.baseUrl}${sep}fmt=json3`).catch(() => null);
-          if (xmlResp?.ok) captions = await xmlResp.text();
+          if (xmlResp?.ok) {
+            const captions = await xmlResp.text();
+            if (captions) return { captions, status: lastStatus, escalate: false };
+          }
         }
-        if (!captions && !pick) continue;
-        return {
-          captions,
-          status: lastStatus,
-          audio: pick ? {
-            url: pick.url,
-            mimeType: pick.mimeType,
-            bitrate: pick.bitrate ?? null,
-            contentLength: pick.contentLength ? Number(pick.contentLength) || null : null,
-            approxDurationMs: pick.approxDurationMs ? Number(pick.approxDurationMs) || null : null
-          } : null
-        };
       }
-      return { captions: null, audio: null, status: lastStatus };
+      return { captions: null, status: lastStatus, escalate: sawBotBlock && !sawAccessible };
     },
     { vid: videoId, clients: INNERTUBE_CLIENTS }
   ).catch(() => null);
 }
-async function downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, audio) {
-  if (audio.contentLength && audio.contentLength > MAX_AUDIO_BYTES) {
-    console.warn(`[CaptionFetcher] audio stream ${audio.contentLength} bytes exceeds ${MAX_AUDIO_BYTES} byte cap`);
-    return null;
-  }
-  const sep = audio.url.includes("?") ? "&" : "?";
-  const url = audio.contentLength ? `${audio.url}${sep}range=0-${audio.contentLength - 1}` : audio.url;
-  const result = await page.evaluate(async (u) => {
-    try {
-      const resp = await fetch(u);
-      if (!resp.ok) return { ok: false, error: `http_${resp.status}` };
-      const blob = await resp.blob();
-      const dataUrl = await new Promise((resolve, reject) => {
-        const reader = new FileReader();
-        reader.onload = () => resolve(reader.result);
-        reader.onerror = () => reject(reader.error);
-        reader.readAsDataURL(blob);
-      });
-      return { ok: true, base64: dataUrl.slice(dataUrl.indexOf(",") + 1) };
-    } catch (e) {
-      return { ok: false, error: String(e) };
-    }
-  }, url).catch((e) => ({ ok: false, error: String(e) }));
-  if (!result.ok || !result.base64) {
-    console.warn(`[CaptionFetcher] audio fetch failed (${result.error ?? "unknown"}) for a ${audio.contentLength ?? 0} byte stream`);
-    return null;
-  }
-  const bytes = Buffer.from(result.base64, "base64");
-  if (audio.contentLength && bytes.length < audio.contentLength) {
-    console.warn(`[CaptionFetcher] partial audio: ${bytes.length} of ${audio.contentLength} bytes`);
-    return null;
-  }
-  return { bytes, mimeType: audio.mimeType.split(";")[0].trim() };
-}
-async function transcribeWithWizper(videoId, audio, startMs) {
-  const falKey = process.env.FAL_KEY;
-  if (!falKey) return null;
-  try {
-    import_client4.fal.config({ credentials: falKey });
-    const ext = audio.mimeType.includes("webm") ? "webm" : audio.mimeType.includes("mp4") ? "m4a" : "audio";
-    const audioFile = new File([new Uint8Array(audio.bytes)], `audio.${ext}`, { type: audio.mimeType });
-    const uploadedUrl = await import_client4.fal.storage.upload(audioFile);
-    const subscribeStartMs = Date.now();
-    const result = await import_client4.fal.subscribe("fal-ai/wizper", {
-      input: { audio_url: uploadedUrl, task: "transcribe", language: "en", chunk_level: "segment" },
-      logs: false,
-      pollInterval: 3e3
-    });
-    const subscribeDurationMs = Date.now() - subscribeStartMs;
-    const data = result.data;
-    const chunks = (data.chunks ?? []).map((c) => ({
-      timestamp: c.timestamp,
-      text: c.text.trim()
-    }));
-    void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: subscribeDurationMs / 1e3, unitType: "compute_sec" });
-    const text = data.text ?? chunks.map((c) => c.text).join(" ");
-    if (!text) return null;
-    return { videoId, text, chunks, durationMs: Date.now() - startMs, method: "browser-whisper" };
-  } catch (err) {
-    console.warn(`[CaptionFetcher] wizper transcription failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
-    return null;
-  }
-}
 async function resolveResidentialProxy(kernelApiKey) {
   try {
     const resolved = await resolveKernelProxyId({
@@ -13833,14 +13759,13 @@ async function resolveResidentialProxy(kernelApiKey) {
     return {};
   }
 }
-async function probeAndExtract(videoId, opts) {
+async function probeCaptions(videoId, kernelProxyIdOverride) {
   const kernelApiKey = browserServiceApiKey();
   if (!kernelApiKey) return null;
-  const kernelProxyId = opts?.kernelProxyId ?? browserServiceProxyId();
+  const kernelProxyId = kernelProxyIdOverride ?? browserServiceProxyId();
   const driver = new BrowserDriver();
   return runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: "browser-innertube" }, async () => {
     let entries = [];
-    let audio = null;
     let retryWithFreshIp = false;
     try {
       await driver.launch({ kernelApiKey, kernelProxyId, headless: true, headlessMode: "headless", viewport: { width: 1280, height: 800 }, locale: "en-US" });
@@ -13852,69 +13777,51 @@ async function probeAndExtract(videoId, opts) {
       );
       const probe = await probePlayer(page, videoId);
       if (!probe) return null;
-      if (probe.captions && !opts?.skipCaptions) {
+      if (probe.captions) {
         entries = parseTimedtextJson3(probe.captions);
         if (entries.length === 0) entries = parseTimedtextXml(probe.captions);
       }
       if (entries.length === 0) {
-        if (!probe.audio) {
-          retryWithFreshIp = isBotBlockStatus(probe.status);
-          console.warn(`[CaptionFetcher] no captions and no audio stream for ${videoId} (playability=${probe.status ?? "unknown"}, proxied=${Boolean(kernelProxyId)})`);
-          return { entries, audio: null, retryWithFreshIp };
-        }
-        audio = await downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, probe.audio);
-        if (!audio) return { entries, audio: null, retryWithFreshIp: true };
+        retryWithFreshIp = probe.escalate;
+        console.warn(`[CaptionFetcher] no caption track for ${videoId} (playability=${probe.status ?? "unknown"}, proxied=${Boolean(kernelProxyId)}, escalate=${probe.escalate})`);
       }
     } catch (err) {
       console.warn(`[CaptionFetcher] kernel session failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
       return null;
     } finally {
-      const servedSubOp = audio ? "browser-whisper" : "browser-innertube";
-      await runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: servedSubOp }, () => driver.close());
+      await driver.close();
     }
-    return { entries, audio, retryWithFreshIp };
+    return { entries, retryWithFreshIp };
   });
 }
-async function extractWithProxyEscalation(videoId, opts) {
-  const direct = await probeAndExtract(videoId, opts);
-  if (direct && (direct.entries.length > 0 || direct.audio)) return direct;
-  const shouldEscalate = direct === null || direct.retryWithFreshIp;
-  if (!shouldEscalate) return direct;
-  const kernelApiKey = browserServiceApiKey();
-  if (!kernelApiKey) return direct;
-  const { kernelProxyId, disposableProxyId } = await resolveResidentialProxy(kernelApiKey);
-  if (!kernelProxyId) return direct;
-  try {
-    console.warn(`[CaptionFetcher] ${videoId}: retrying through residential proxy after bot block`);
-    const viaProxy = await probeAndExtract(videoId, { ...opts, kernelProxyId });
-    return viaProxy ?? direct;
-  } finally {
-    if (disposableProxyId) {
-      await deleteKernelProxyId(kernelApiKey, disposableProxyId).catch(
-        (err) => console.warn(`[CaptionFetcher] failed to delete disposable proxy: ${err instanceof Error ? err.message : String(err)}`)
-      );
-    }
-  }
-}
 async function fetchViaKernelSession(videoId) {
   const start = Date.now();
-  const outcome = await extractWithProxyEscalation(videoId);
-  if (!outcome) return null;
-  if (outcome.entries.length > 0) {
-    const chunks = outcome.entries.map((e) => ({
-      timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
-      text: e.text
-    }));
-    const text = chunks.map((c) => c.text).join(" ");
-    return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
-  }
-  if (outcome.audio) {
-    return runWithCostContext(
-      { ...currentCostContext(), op: "yt_transcription", subOp: "browser-whisper" },
-      () => transcribeWithWizper(videoId, outcome.audio, start)
-    );
+  let outcome = await probeCaptions(videoId);
+  if (!outcome || outcome.entries.length === 0 && outcome.retryWithFreshIp) {
+    const kernelApiKey = browserServiceApiKey();
+    if (kernelApiKey) {
+      const { kernelProxyId, disposableProxyId } = await resolveResidentialProxy(kernelApiKey);
+      if (kernelProxyId) {
+        try {
+          console.warn(`[CaptionFetcher] ${videoId}: retrying captions through residential proxy after bot block`);
+          outcome = await probeCaptions(videoId, kernelProxyId) ?? outcome;
+        } finally {
+          if (disposableProxyId) {
+            await deleteKernelProxyId(kernelApiKey, disposableProxyId).catch(
+              (err) => console.warn(`[CaptionFetcher] failed to delete disposable proxy: ${err instanceof Error ? err.message : String(err)}`)
+            );
+          }
+        }
+      }
+    }
   }
-  return null;
+  if (!outcome || outcome.entries.length === 0) return null;
+  const chunks = outcome.entries.map((e) => ({
+    timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
+    text: e.text
+  }));
+  const text = chunks.map((c) => c.text).join(" ");
+  return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
 }
 async function fetchViaResolvedAudio(videoId) {
   if (!mediaResolverConfigured()) return null;
@@ -13964,7 +13871,7 @@ async function fetchCaptions(videoId) {
     `Could not transcribe ${videoId}: it has no caption track and its audio could not be retrieved.`
   );
 }
-var import_client4, INNERTUBE_CLIENTS, MAX_AUDIO_BYTES, YT_RESIDENTIAL_LOCATION;
+var INNERTUBE_CLIENTS, YT_RESIDENTIAL_LOCATION;
 var init_CaptionFetcher = __esm({
   "src/youtube/CaptionFetcher.ts"() {
     "use strict";
@@ -13973,7 +13880,6 @@ var init_CaptionFetcher = __esm({
     init_kernel_proxy_resolver();
     init_audio_resolver();
     init_media_transcription();
-    import_client4 = require("@fal-ai/client");
     init_cost_telemetry();
     init_cost_context();
     INNERTUBE_CLIENTS = [
@@ -13981,7 +13887,6 @@ var init_CaptionFetcher = __esm({
       { name: "IOS", version: "20.10.4", userAgent: "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)" },
       { name: "ANDROID_VR", version: "1.62.27", userAgent: "com.google.android.apps.youtube.vr.oculus/1.62.27 (Linux; U; Android 12L) gzip" }
     ];
-    MAX_AUDIO_BYTES = 96 * 1024 * 1024;
     YT_RESIDENTIAL_LOCATION = process.env.YOUTUBE_PROXY_LOCATION ?? "Austin, Texas";
   }
 });
@@ -15560,7 +15465,7 @@ async function kernelLaunchOptsResidential() {
   }
   return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
 }
-var import_hono5, import_zod16, import_client5, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
+var import_hono5, import_zod16, import_client4, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
 var init_facebook_ad_routes = __esm({
   "src/api/facebook-ad-routes.ts"() {
     "use strict";
@@ -15576,7 +15481,7 @@ var init_facebook_ad_routes = __esm({
     init_FacebookOrganicVideoExtractor();
     init_kernel_proxy_resolver();
     init_schemas3();
-    import_client5 = require("@fal-ai/client");
+    import_client4 = require("@fal-ai/client");
     init_api_auth();
     init_url_utils();
     init_concurrency_gates();
@@ -15719,7 +15624,7 @@ var init_facebook_ad_routes = __esm({
         metadata: { videoUrl }
       });
       if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-      import_client5.fal.config({ credentials: process.env.FAL_KEY });
+      import_client4.fal.config({ credentials: process.env.FAL_KEY });
       const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
       let debited = false;
       try {
@@ -15763,7 +15668,7 @@ var init_facebook_ad_routes = __esm({
         metadata: { url: sourceUrl.href }
       });
       if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-      import_client5.fal.config({ credentials: process.env.FAL_KEY });
+      import_client4.fal.config({ credentials: process.env.FAL_KEY });
       const driver = new BrowserDriver();
       const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
       let debited = false;
@@ -16144,7 +16049,7 @@ async function kernelLaunchOptsResidential2() {
 function isBlockedMessage(msg) {
   return msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha");
 }
-var import_hono6, import_zod17, import_client6, GoogleAdsSearchBodySchema, GoogleAdsPageIntelBodySchema, GoogleAdsTranscribeBodySchema, googleAdsApp;
+var import_hono6, import_zod17, import_client5, GoogleAdsSearchBodySchema, GoogleAdsPageIntelBodySchema, GoogleAdsTranscribeBodySchema, googleAdsApp;
 var init_google_ads_routes = __esm({
   "src/api/google-ads-routes.ts"() {
     "use strict";
@@ -16157,7 +16062,7 @@ var init_google_ads_routes = __esm({
     init_GoogleAdsExtractor();
     init_kernel_proxy_resolver();
     init_schemas3();
-    import_client6 = require("@fal-ai/client");
+    import_client5 = require("@fal-ai/client");
     init_api_auth();
     init_url_utils();
     init_concurrency_gates();
@@ -16268,7 +16173,7 @@ var init_google_ads_routes = __esm({
       const user = c.get("user");
       const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
       if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-      import_client6.fal.config({ credentials: process.env.FAL_KEY });
+      import_client5.fal.config({ credentials: process.env.FAL_KEY });
       const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
       let debited = false;
       try {
@@ -20614,7 +20519,7 @@ function formatCreditsInfo(raw, input) {
     return `| ${c.label} | ${c.credits} | ${c.unit}${notes} |`;
   }).join("\n");
   const ledgerRows = ledger.map((row) => {
-    const credits = row.amount_mc / 1e3;
+    const credits = row.amount_mc / MC_PER_CREDIT;
     return `| ${row.created_at} | ${row.operation} | ${credits} | ${row.description ?? ""} |`;
   }).join("\n");
   const matchedSection = matched ? `
@@ -20664,7 +20569,7 @@ ${ledgerRows}` : ""
       ledger: ledger.map((row) => ({
         createdAt: String(row.created_at ?? ""),
         operation: String(row.operation ?? ""),
-        credits: row.amount_mc / 1e3,
+        credits: row.amount_mc / MC_PER_CREDIT,
         description: row.description ?? null
       })),
       concurrency: concurrencyRaw && upgradeRaw ? {
@@ -21463,6 +21368,7 @@ var init_mcp_response_formatter = __esm({
     init_seo_issues();
     init_image_audit();
     init_report_artifact_offload();
+    init_rates();
     INLINE_BUDGET_BYTES = Number(process.env.MCP_SCRAPER_INLINE_BUDGET ?? 5e4);
     STRUCTURED_ARRAY_CAP = 25;
     reportSavingEnabled = true;
@@ -27297,7 +27203,7 @@ var init_serp_intelligence_routes = __esm({
           kernelApiKey: browserServiceApiKey(),
           kernelProxyId: browserServiceProxyId(),
           signal: c.req.raw.signal,
-          billing: { creditsUsed: cost / 1e3 }
+          billing: { creditsUsed: cost / MC_PER_CREDIT }
         });
         await logRequestEvent({
           userId: user.id,
@@ -27399,7 +27305,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.4.13";
+    PACKAGE_VERSION = "0.5.0";
   }
 });
 
@@ -38839,7 +38745,7 @@ var init_server = __esm({
         api_key: user.key_active ? user.api_key : null,
         created_at: user.created_at,
         balance_mc: user.balance_mc,
-        balance_credits: user.balance_mc / 1e3,
+        balance_credits: user.balance_mc / MC_PER_CREDIT,
         extra_concurrency_slots: user.extra_concurrency_slots,
         concurrency_limit: concurrencyLimitForUser(user),
         has_concurrency_sub: !!user.concurrency_stripe_sub_id,
@@ -39951,7 +39857,7 @@ var init_server = __esm({
           tier: tier.tier,
           label: tier.label,
           monthly_usd: tier.monthly_usd,
-          credits_per_month: tier.credits_mc / 1e3,
+          credits_per_month: tier.credits_mc / MC_PER_CREDIT,
           concurrency: tier.concurrency,
           intro: tier.intro_coupon ? "$1 first month" : null,
           next_step: "Open checkout_url in a browser to complete payment."
@@ -39981,7 +39887,7 @@ var init_server = __esm({
       const freeCredits = await getFreeCreditBreakdown(user.id);
       return c.json({
         balance_mc: balanceMc,
-        balance_credits: balanceMc / 1e3,
+        balance_credits: balanceMc / MC_PER_CREDIT,
         free_credits: freeCredits,
         ledger
       });
@@ -40003,7 +39909,7 @@ var init_server = __esm({
       })) : void 0;
       return c.json({
         balance_mc: balanceMc,
-        balance_credits: balanceMc / 1e3,
+        balance_credits: balanceMc / MC_PER_CREDIT,
         item: body.item ?? null,
         matched_cost: matchedCost ? (({ aliases, ...cost }) => cost)(matchedCost) : null,
         costs,