mcp-scraper 0.4.13 → 0.5.0

package/dist/{server-HNZG36IP.js → server-Z4MEISH5.js}

@@ -33,7 +33,7 @@ import {
   registerSerpIntelligenceCaptureTools,
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-EA2AANFI.js";
+} from "./chunk-J4T5OSCF.js";
 import {
   auditImages,
   getBlobStore
@@ -67,7 +67,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-XGIPATLV.js";
-import "./chunk-O2D5HGKF.js";
+import "./chunk-3RTPEQJC.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -78,7 +78,7 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-VQV7MB2S.js";
+} from "./chunk-BL7BBSYF.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -101,7 +101,7 @@ import {
   browserActiveCostMc,
   concurrencySlotBillingInfo,
   insufficientBalanceResponse
-} from "./chunk-SIPXEFTR.js";
+} from "./chunk-APJO2XV5.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -8788,7 +8788,6 @@ async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript"
 }
 
 // src/youtube/CaptionFetcher.ts
-import { fal as fal2 } from "@fal-ai/client";
 async function fetchViaYoutubeTranscript(videoId) {
   try {
     const { YoutubeTranscript } = await import("youtube-transcript");
@@ -8861,13 +8860,12 @@ var INNERTUBE_CLIENTS = [
   { name: "IOS", version: "20.10.4", userAgent: "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)" },
   { name: "ANDROID_VR", version: "1.62.27", userAgent: "com.google.android.apps.youtube.vr.oculus/1.62.27 (Linux; U; Android 12L) gzip" }
 ];
-function isBotBlockStatus(status) {
-  return status === "LOGIN_REQUIRED" || status === "AGE_VERIFICATION_REQUIRED";
-}
 async function probePlayer(page, videoId) {
   return page.evaluate(
     async ({ vid, clients }) => {
       let lastStatus = null;
+      let sawBotBlock = false;
+      let sawAccessible = false;
       for (const client2 of clients) {
         const resp = await fetch("/youtubei/v1/player?prettyPrint=false", {
           method: "POST",
@@ -8883,98 +8881,24 @@ async function probePlayer(page, videoId) {
         }
         const data = await resp.json();
         lastStatus = data?.playabilityStatus?.status ?? null;
-        const audioFormats = (data?.streamingData?.adaptiveFormats ?? []).filter((f) => typeof f.url === "string" && f.url && typeof f.mimeType === "string" && f.mimeType.startsWith("audio/")).sort((a, b) => (Number(a.contentLength) || Infinity) - (Number(b.contentLength) || Infinity));
-        const pick = audioFormats[0] ?? null;
-        let captions = null;
+        if (lastStatus === "LOGIN_REQUIRED" || lastStatus === "AGE_VERIFICATION_REQUIRED") sawBotBlock = true;
+        else if (lastStatus === "OK") sawAccessible = true;
         const tracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
         if (tracks.length > 0) {
           const track = tracks.find((t) => t.languageCode === "en") ?? tracks[0];
           const sep = track.baseUrl.includes("?") ? "&" : "?";
           const xmlResp = await fetch(`${track.baseUrl}${sep}fmt=json3`).catch(() => null);
-          if (xmlResp?.ok) captions = await xmlResp.text();
+          if (xmlResp?.ok) {
+            const captions = await xmlResp.text();
+            if (captions) return { captions, status: lastStatus, escalate: false };
+          }
         }
-        if (!captions && !pick) continue;
-        return {
-          captions,
-          status: lastStatus,
-          audio: pick ? {
-            url: pick.url,
-            mimeType: pick.mimeType,
-            bitrate: pick.bitrate ?? null,
-            contentLength: pick.contentLength ? Number(pick.contentLength) || null : null,
-            approxDurationMs: pick.approxDurationMs ? Number(pick.approxDurationMs) || null : null
-          } : null
-        };
       }
-      return { captions: null, audio: null, status: lastStatus };
+      return { captions: null, status: lastStatus, escalate: sawBotBlock && !sawAccessible };
     },
     { vid: videoId, clients: INNERTUBE_CLIENTS }
   ).catch(() => null);
 }
-var MAX_AUDIO_BYTES = 96 * 1024 * 1024;
-async function downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, audio) {
-  if (audio.contentLength && audio.contentLength > MAX_AUDIO_BYTES) {
-    console.warn(`[CaptionFetcher] audio stream ${audio.contentLength} bytes exceeds ${MAX_AUDIO_BYTES} byte cap`);
-    return null;
-  }
-  const sep = audio.url.includes("?") ? "&" : "?";
-  const url = audio.contentLength ? `${audio.url}${sep}range=0-${audio.contentLength - 1}` : audio.url;
-  const result = await page.evaluate(async (u) => {
-    try {
-      const resp = await fetch(u);
-      if (!resp.ok) return { ok: false, error: `http_${resp.status}` };
-      const blob = await resp.blob();
-      const dataUrl = await new Promise((resolve, reject) => {
-        const reader = new FileReader();
-        reader.onload = () => resolve(reader.result);
-        reader.onerror = () => reject(reader.error);
-        reader.readAsDataURL(blob);
-      });
-      return { ok: true, base64: dataUrl.slice(dataUrl.indexOf(",") + 1) };
-    } catch (e) {
-      return { ok: false, error: String(e) };
-    }
-  }, url).catch((e) => ({ ok: false, error: String(e) }));
-  if (!result.ok || !result.base64) {
-    console.warn(`[CaptionFetcher] audio fetch failed (${result.error ?? "unknown"}) for a ${audio.contentLength ?? 0} byte stream`);
-    return null;
-  }
-  const bytes = Buffer.from(result.base64, "base64");
-  if (audio.contentLength && bytes.length < audio.contentLength) {
-    console.warn(`[CaptionFetcher] partial audio: ${bytes.length} of ${audio.contentLength} bytes`);
-    return null;
-  }
-  return { bytes, mimeType: audio.mimeType.split(";")[0].trim() };
-}
-async function transcribeWithWizper(videoId, audio, startMs) {
-  const falKey = process.env.FAL_KEY;
-  if (!falKey) return null;
-  try {
-    fal2.config({ credentials: falKey });
-    const ext = audio.mimeType.includes("webm") ? "webm" : audio.mimeType.includes("mp4") ? "m4a" : "audio";
-    const audioFile = new File([new Uint8Array(audio.bytes)], `audio.${ext}`, { type: audio.mimeType });
-    const uploadedUrl = await fal2.storage.upload(audioFile);
-    const subscribeStartMs = Date.now();
-    const result = await fal2.subscribe("fal-ai/wizper", {
-      input: { audio_url: uploadedUrl, task: "transcribe", language: "en", chunk_level: "segment" },
-      logs: false,
-      pollInterval: 3e3
-    });
-    const subscribeDurationMs = Date.now() - subscribeStartMs;
-    const data = result.data;
-    const chunks = (data.chunks ?? []).map((c) => ({
-      timestamp: c.timestamp,
-      text: c.text.trim()
-    }));
-    void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: subscribeDurationMs / 1e3, unitType: "compute_sec" });
-    const text = data.text ?? chunks.map((c) => c.text).join(" ");
-    if (!text) return null;
-    return { videoId, text, chunks, durationMs: Date.now() - startMs, method: "browser-whisper" };
-  } catch (err) {
-    console.warn(`[CaptionFetcher] wizper transcription failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
-    return null;
-  }
-}
 var YT_RESIDENTIAL_LOCATION = process.env.YOUTUBE_PROXY_LOCATION ?? "Austin, Texas";
 async function resolveResidentialProxy(kernelApiKey) {
   try {
@@ -8991,14 +8915,13 @@ async function resolveResidentialProxy(kernelApiKey) {
     return {};
   }
 }
-async function probeAndExtract(videoId, opts) {
+async function probeCaptions(videoId, kernelProxyIdOverride) {
   const kernelApiKey = browserServiceApiKey();
   if (!kernelApiKey) return null;
-  const kernelProxyId = opts?.kernelProxyId ?? browserServiceProxyId();
+  const kernelProxyId = kernelProxyIdOverride ?? browserServiceProxyId();
   const driver = new BrowserDriver();
   return runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: "browser-innertube" }, async () => {
     let entries = [];
-    let audio = null;
     let retryWithFreshIp = false;
     try {
       await driver.launch({ kernelApiKey, kernelProxyId, headless: true, headlessMode: "headless", viewport: { width: 1280, height: 800 }, locale: "en-US" });
@@ -9010,69 +8933,51 @@ async function probeAndExtract(videoId, opts) {
       );
       const probe = await probePlayer(page, videoId);
       if (!probe) return null;
-      if (probe.captions && !opts?.skipCaptions) {
+      if (probe.captions) {
         entries = parseTimedtextJson3(probe.captions);
         if (entries.length === 0) entries = parseTimedtextXml(probe.captions);
       }
       if (entries.length === 0) {
-        if (!probe.audio) {
-          retryWithFreshIp = isBotBlockStatus(probe.status);
-          console.warn(`[CaptionFetcher] no captions and no audio stream for ${videoId} (playability=${probe.status ?? "unknown"}, proxied=${Boolean(kernelProxyId)})`);
-          return { entries, audio: null, retryWithFreshIp };
-        }
-        audio = await downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, probe.audio);
-        if (!audio) return { entries, audio: null, retryWithFreshIp: true };
+        retryWithFreshIp = probe.escalate;
+        console.warn(`[CaptionFetcher] no caption track for ${videoId} (playability=${probe.status ?? "unknown"}, proxied=${Boolean(kernelProxyId)}, escalate=${probe.escalate})`);
       }
     } catch (err) {
       console.warn(`[CaptionFetcher] kernel session failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
       return null;
     } finally {
-      const servedSubOp = audio ? "browser-whisper" : "browser-innertube";
-      await runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: servedSubOp }, () => driver.close());
+      await driver.close();
     }
-    return { entries, audio, retryWithFreshIp };
+    return { entries, retryWithFreshIp };
   });
 }
-async function extractWithProxyEscalation(videoId, opts) {
-  const direct = await probeAndExtract(videoId, opts);
-  if (direct && (direct.entries.length > 0 || direct.audio)) return direct;
-  const shouldEscalate = direct === null || direct.retryWithFreshIp;
-  if (!shouldEscalate) return direct;
-  const kernelApiKey = browserServiceApiKey();
-  if (!kernelApiKey) return direct;
-  const { kernelProxyId, disposableProxyId } = await resolveResidentialProxy(kernelApiKey);
-  if (!kernelProxyId) return direct;
-  try {
-    console.warn(`[CaptionFetcher] ${videoId}: retrying through residential proxy after bot block`);
-    const viaProxy = await probeAndExtract(videoId, { ...opts, kernelProxyId });
-    return viaProxy ?? direct;
-  } finally {
-    if (disposableProxyId) {
-      await deleteKernelProxyId(kernelApiKey, disposableProxyId).catch(
-        (err) => console.warn(`[CaptionFetcher] failed to delete disposable proxy: ${err instanceof Error ? err.message : String(err)}`)
-      );
-    }
-  }
-}
 async function fetchViaKernelSession(videoId) {
   const start = Date.now();
-  const outcome = await extractWithProxyEscalation(videoId);
-  if (!outcome) return null;
-  if (outcome.entries.length > 0) {
-    const chunks = outcome.entries.map((e) => ({
-      timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
-      text: e.text
-    }));
-    const text = chunks.map((c) => c.text).join(" ");
-    return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
-  }
-  if (outcome.audio) {
-    return runWithCostContext(
-      { ...currentCostContext(), op: "yt_transcription", subOp: "browser-whisper" },
-      () => transcribeWithWizper(videoId, outcome.audio, start)
-    );
+  let outcome = await probeCaptions(videoId);
+  if (!outcome || outcome.entries.length === 0 && outcome.retryWithFreshIp) {
+    const kernelApiKey = browserServiceApiKey();
+    if (kernelApiKey) {
+      const { kernelProxyId, disposableProxyId } = await resolveResidentialProxy(kernelApiKey);
+      if (kernelProxyId) {
+        try {
+          console.warn(`[CaptionFetcher] ${videoId}: retrying captions through residential proxy after bot block`);
+          outcome = await probeCaptions(videoId, kernelProxyId) ?? outcome;
+        } finally {
+          if (disposableProxyId) {
+            await deleteKernelProxyId(kernelApiKey, disposableProxyId).catch(
+              (err) => console.warn(`[CaptionFetcher] failed to delete disposable proxy: ${err instanceof Error ? err.message : String(err)}`)
+            );
+          }
+        }
+      }
+    }
   }
-  return null;
+  if (!outcome || outcome.entries.length === 0) return null;
+  const chunks = outcome.entries.map((e) => ({
+    timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
+    text: e.text
+  }));
+  const text = chunks.map((c) => c.text).join(" ");
+  return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
 }
 async function fetchViaResolvedAudio(videoId) {
   if (!mediaResolverConfigured()) return null;
@@ -10485,7 +10390,7 @@ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "b
 }
 
 // src/api/facebook-ad-routes.ts
-import { fal as fal3 } from "@fal-ai/client";
+import { fal as fal2 } from "@fal-ai/client";
 var FacebookAdBodySchema = z13.object({
   url: z13.string().trim().optional(),
   libraryId: z13.string().trim().optional(),
@@ -10665,7 +10570,7 @@ facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
     metadata: { videoUrl }
   });
   if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-  fal3.config({ credentials: process.env.FAL_KEY });
+  fal2.config({ credentials: process.env.FAL_KEY });
   const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
   let debited = false;
   try {
@@ -10709,7 +10614,7 @@ facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
     metadata: { url: sourceUrl.href }
   });
   if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-  fal3.config({ credentials: process.env.FAL_KEY });
+  fal2.config({ credentials: process.env.FAL_KEY });
   const driver = new BrowserDriver();
   const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
   let debited = false;
@@ -11062,7 +10967,7 @@ var GoogleAdsExtractor = class {
 };
 
 // src/api/google-ads-routes.ts
-import { fal as fal4 } from "@fal-ai/client";
+import { fal as fal3 } from "@fal-ai/client";
 var GoogleAdsSearchBodySchema = z14.object({
   query: z14.string().trim().min(1, "query is required"),
   region: z14.string().trim().toUpperCase().length(2).optional(),
@@ -11192,7 +11097,7 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
   const user = c.get("user");
   const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
   if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-  fal4.config({ credentials: process.env.FAL_KEY });
+  fal3.config({ credentials: process.env.FAL_KEY });
   const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
   let debited = false;
   try {
@@ -15532,7 +15437,7 @@ serpIntelligenceApp.post("/capture", async (c) => {
       kernelApiKey: browserServiceApiKey(),
       kernelProxyId: browserServiceProxyId(),
       signal: c.req.raw.signal,
-      billing: { creditsUsed: cost / 1e3 }
+      billing: { creditsUsed: cost / MC_PER_CREDIT }
     });
     await logRequestEvent({
       userId: user.id,
@@ -20692,7 +20597,7 @@ app.get("/me", async (c) => {
     api_key: user.key_active ? user.api_key : null,
     created_at: user.created_at,
     balance_mc: user.balance_mc,
-    balance_credits: user.balance_mc / 1e3,
+    balance_credits: user.balance_mc / MC_PER_CREDIT,
     extra_concurrency_slots: user.extra_concurrency_slots,
     concurrency_limit: concurrencyLimitForUser(user),
     has_concurrency_sub: !!user.concurrency_stripe_sub_id,
@@ -21854,7 +21759,7 @@ app.post("/billing/subscribe/terminal-checkout", auth2, async (c) => {
       tier: tier.tier,
       label: tier.label,
       monthly_usd: tier.monthly_usd,
-      credits_per_month: tier.credits_mc / 1e3,
+      credits_per_month: tier.credits_mc / MC_PER_CREDIT,
       concurrency: tier.concurrency,
       intro: tier.intro_coupon ? "$1 first month" : null,
       next_step: "Open checkout_url in a browser to complete payment."
@@ -21884,7 +21789,7 @@ app.get("/billing/balance", auth2, async (c) => {
   const freeCredits = await getFreeCreditBreakdown(user.id);
   return c.json({
     balance_mc: balanceMc,
-    balance_credits: balanceMc / 1e3,
+    balance_credits: balanceMc / MC_PER_CREDIT,
     free_credits: freeCredits,
     ledger
   });
@@ -21906,7 +21811,7 @@ app.post("/billing/credits", auth2, async (c) => {
   })) : void 0;
   return c.json({
     balance_mc: balanceMc,
-    balance_credits: balanceMc / 1e3,
+    balance_credits: balanceMc / MC_PER_CREDIT,
     item: body.item ?? null,
     matched_cost: matchedCost ? (({ aliases, ...cost }) => cost)(matchedCost) : null,
     costs,
@@ -21924,7 +21829,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-WCSKUAOT.js");
+  const { drainQueue } = await import("./worker-KJZ3ZN2N.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
@@ -21942,7 +21847,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const jobId = c.req.param("id");
-  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-6X2C7RLF.js");
+  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-U476J44K.js");
   const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
   const job = await getExtractJob2(jobId);
   if (!job) return c.json({ error: "job not found" }, 404);
@@ -21950,7 +21855,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
   await completeExtractJob2(jobId, stored);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-6X2C7RLF.js");
+    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-U476J44K.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -22114,4 +22019,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-HNZG36IP.js.map
+//# sourceMappingURL=server-Z4MEISH5.js.map