mcp-scraper 0.18.0 → 0.20.0

package/dist/{server-MQDCAR6I.js → server-VDWIPV7F.js}

@@ -1,27 +1,24 @@
+import {
+  harvestPageMedia
+} from "./chunk-JWIE5NCR.js";
 import {
   classifyHarvestProblem,
   createHarvestAttemptRecorder,
   harvestProblemResponse,
   serializeHarvestProblem
-} from "./chunk-HE2LQPJ2.js";
+} from "./chunk-ZRKFW5FB.js";
 import {
   BrowserDriver,
   MapsSelectors,
   buildYouTubeChannelVideosUrl,
-  currentCostContext,
   deleteKernelProxyId,
   harvest,
-  kernelCostUsd,
-  migrateCostTelemetry,
-  recordKernelSession,
-  recordVendorUsage,
-  resolveKernelProxyId,
-  runWithCostContext,
-  vendorCostUsd
-} from "./chunk-RMPPYKUV.js";
+  resolveKernelProxyId
+} from "./chunk-XDFSLSSH.js";
 import {
   HttpMcpToolExecutor,
   MemoryMcpToolExecutor,
+  assessTranscriptSignal,
   buildLinkGraph,
   buildPaaExtractorMcpServer,
   cleanupExpiredConnectedDataArtifacts,
@@ -36,8 +33,9 @@ import {
   registerSerpIntelligenceCaptureTools,
   renewConnectedDataArtifactDownload,
   sanitizeAttempts,
-  sanitizeHarvestResult
-} from "./chunk-FMEDPBIV.js";
+  sanitizeHarvestResult,
+  transcribeMediaUrl
+} from "./chunk-KE7KE2Q2.js";
 import {
   auditImages,
   buildLinkReport,
@@ -48,8 +46,14 @@ import {
   browserServiceApiKey,
   browserServiceProfileName,
   browserServiceProfileSaveChanges,
-  browserServiceProxyId
-} from "./chunk-HPV4VOQX.js";
+  browserServiceProxyId,
+  currentCostContext,
+  kernelCostUsd,
+  migrateCostTelemetry,
+  recordKernelSession,
+  recordVendorUsage,
+  runWithCostContext
+} from "./chunk-UN7VMHZL.js";
 import {
   csvRecords,
   listWorkflowDefinitions,
@@ -73,7 +77,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-XGIPATLV.js";
-import "./chunk-RLWCHLV3.js";
+import "./chunk-PZB3TJWK.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -84,7 +88,7 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-RUGJE5EB.js";
+} from "./chunk-EJK25QOW.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -190,7 +194,7 @@ import {
   setUserMemoryCreds,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-GL4BW4CP.js";
+} from "./chunk-22VEGGTW.js";
 
 // src/blog/registry.ts
 var posts = [
@@ -4225,285 +4229,6 @@ async function capturePageData(url, opts = {}) {
   }
 }
 
-// src/lib/media-extractor.ts
-import { createWriteStream, mkdirSync } from "fs";
-import { homedir } from "os";
-import { join as join2, extname, basename } from "path";
-import { pipeline } from "stream/promises";
-import { Readable } from "stream";
-var AD_PATTERNS = [
-  "doubleclick.net",
-  "googlesyndication.com",
-  "googletagmanager.com",
-  "google-analytics.com",
-  "googletagservices.com",
-  "adservice.google",
-  "googletag.",
-  "pagead2.googlesyndication",
-  "facebook.net/tr",
-  "connect.facebook.net",
-  "fbcdn.net/rsrc",
-  "analytics.twitter.com",
-  "static.ads-twitter.com",
-  "ads.twitter.com",
-  "t.co/i/adsct",
-  "pixel.advertising.com",
-  "hotjar.com",
-  "clarity.ms",
-  "quantserve.com",
-  "scorecardresearch.com",
-  "newrelic.com",
-  "nr-data.net",
-  "segment.io",
-  "segment.com",
-  "amplitude.com",
-  "mixpanel.com",
-  "heap.io",
-  "fullstory.com",
-  "moatads.com",
-  "criteo.com",
-  "adsrvr.org",
-  "rubiconproject.com",
-  "pubmatic.com",
-  "openx.net",
-  "appnexus.com",
-  "amazon-adsystem.com",
-  "media.net",
-  "yieldmo.com",
-  "triplelift.com",
-  "sharethrough.com",
-  "prebid.",
-  "smaato.net",
-  "indexworm.com",
-  "casalemedia.com",
-  "outbrain.com",
-  "taboola.com",
-  "revcontent.com",
-  "mgid.com",
-  "tawk.to",
-  "intercom.io",
-  "drift.com",
-  "hs-scripts.com",
-  "zopim.com",
-  "livechatinc.com",
-  "userlike.com",
-  "onetrust.com",
-  "cookielaw.org",
-  "cookieinformation.com",
-  "trustarc.com",
-  "/ads/",
-  "/ad/",
-  "/banner/",
-  "/banners/",
-  "/pixel/",
-  "/beacon/",
-  "/tracking/",
-  "/tracker/",
-  "/remarketing/",
-  "/conversion/",
-  "1x1.gif",
-  "spacer.gif",
-  "blank.gif",
-  "transparent.gif"
-];
-var IMAGE_EXTS = /* @__PURE__ */ new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif", ".svg", ".tiff"]);
-var VIDEO_EXTS = /* @__PURE__ */ new Set([".mp4", ".webm", ".mov", ".avi", ".m4v", ".ogv", ".mkv"]);
-var AUDIO_EXTS = /* @__PURE__ */ new Set([".mp3", ".wav", ".ogg", ".aac", ".m4a", ".flac", ".opus"]);
-function isAdUrl(url) {
-  const lower = url.toLowerCase();
-  return AD_PATTERNS.some((p) => lower.includes(p));
-}
-function isDataUri(url) {
-  return url.startsWith("data:");
-}
-function typeFromUrl(url) {
-  try {
-    const ext = extname(new URL(url).pathname).toLowerCase();
-    if (IMAGE_EXTS.has(ext)) return "image";
-    if (VIDEO_EXTS.has(ext)) return "video";
-    if (AUDIO_EXTS.has(ext)) return "audio";
-  } catch {
-  }
-  return null;
-}
-function typeFromMime(mime) {
-  const lower = mime.toLowerCase();
-  if (lower.startsWith("image/")) return "image";
-  if (lower.startsWith("video/")) return "video";
-  if (lower.startsWith("audio/")) return "audio";
-  return null;
-}
-function resolveUrl(raw, base) {
-  if (!raw || isDataUri(raw)) return null;
-  try {
-    return new URL(raw, base).href;
-  } catch {
-    return null;
-  }
-}
-function safeFilename(url, index) {
-  try {
-    const u = new URL(url);
-    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
-    return base || `asset-${index}`;
-  } catch {
-    return `asset-${index}`;
-  }
-}
-function extractMediaUrls(html, baseUrl) {
-  const seen = /* @__PURE__ */ new Set();
-  const urls = [];
-  const add = (raw) => {
-    const resolved = resolveUrl(raw.trim(), baseUrl);
-    if (!resolved || seen.has(resolved) || isAdUrl(resolved)) return;
-    seen.add(resolved);
-    urls.push(resolved);
-  };
-  for (const m of html.matchAll(/<img\s[^>]*>/gi)) {
-    const tag = m[0];
-    for (const attr of ["src", "data-src", "data-lazy-src", "data-original"]) {
-      const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
-      if (v) add(v);
-    }
-    const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
-    if (srcset) {
-      for (const part of srcset.split(",")) {
-        const u = part.trim().split(/\s+/)[0];
-        if (u) add(u);
-      }
-    }
-  }
-  for (const m of html.matchAll(/<source\s[^>]*>/gi)) {
-    const tag = m[0];
-    const src = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
-    if (src) add(src);
-    const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
-    if (srcset) {
-      for (const part of srcset.split(",")) {
-        const u = part.trim().split(/\s+/)[0];
-        if (u) add(u);
-      }
-    }
-  }
-  for (const m of html.matchAll(/<video\s[^>]*>/gi)) {
-    const tag = m[0];
-    for (const attr of ["src", "poster"]) {
-      const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
-      if (v) add(v);
-    }
-  }
-  for (const m of html.matchAll(/<audio\s[^>]*>/gi)) {
-    const tag = m[0];
-    const v = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
-    if (v) add(v);
-  }
-  for (const m of html.matchAll(/<meta\s[^>]*>/gi)) {
-    const tag = m[0];
-    const prop = (tag.match(/\b(?:property|name)\s*=\s*["']([^"']+)["']/i) ?? [])[1]?.toLowerCase();
-    if (prop === "og:image" || prop === "twitter:image") {
-      const content = (tag.match(/\bcontent\s*=\s*["']([^"']+)["']/i) ?? [])[1];
-      if (content) add(content);
-    }
-  }
-  for (const m of html.matchAll(/background(?:-image)?\s*:\s*url\(\s*["']?([^"')]+)["']?\s*\)/gi)) {
-    add(m[1]);
-  }
-  return urls;
-}
-async function downloadAsset(url, destDir, filename) {
-  const res = await fetch(url, {
-    headers: { "User-Agent": "Mozilla/5.0 (compatible; ThorbitBot/1.0)" },
-    signal: AbortSignal.timeout(15e3)
-  });
-  if (!res.ok) throw new Error(`HTTP ${res.status}`);
-  if (!res.body) throw new Error("Empty response body");
-  const mimeType = res.headers.get("content-type")?.split(";")[0].trim() ?? null;
-  let dest = join2(destDir, filename);
-  if (mimeType && !extname(filename)) {
-    const mimeExt = {
-      "image/jpeg": ".jpg",
-      "image/png": ".png",
-      "image/webp": ".webp",
-      "image/gif": ".gif",
-      "image/svg+xml": ".svg",
-      "image/avif": ".avif",
-      "video/mp4": ".mp4",
-      "video/webm": ".webm",
-      "audio/mpeg": ".mp3",
-      "audio/ogg": ".ogg",
-      "audio/wav": ".wav"
-    };
-    const ext = mimeExt[mimeType];
-    if (ext) dest = dest + ext;
-  }
-  const writer = createWriteStream(dest);
-  await pipeline(Readable.fromWeb(res.body), writer);
-  const { statSync } = await import("fs");
-  const sizeBytes = statSync(dest).size;
-  return { savedPath: dest, sizeBytes, mimeType };
-}
-async function harvestPageMedia(html, pageUrl, options = {}) {
-  const types = options.types ?? ["image", "video", "audio"];
-  const typesSet = new Set(types);
-  const rawUrls = extractMediaUrls(html, pageUrl);
-  const totalFound = rawUrls.length;
-  const filteredUrls = [];
-  const kept = [];
-  for (const url of rawUrls) {
-    const type = typeFromUrl(url);
-    if (!type || !typesSet.has(type)) {
-      filteredUrls.push(url);
-      continue;
-    }
-    kept.push({ url, type });
-  }
-  if (options.outputDir === null) {
-    return {
-      pageUrl,
-      outputDir: null,
-      assets: kept.map(({ url, type }, i) => ({
-        url,
-        type,
-        mimeType: null,
-        filename: safeFilename(url, i),
-        savedPath: null,
-        sizeBytes: null
-      })),
-      filteredCount: filteredUrls.length,
-      totalFound
-    };
-  }
-  const domain = (() => {
-    try {
-      return new URL(pageUrl).hostname.replace(/^www\./, "");
-    } catch {
-      return "unknown";
-    }
-  })();
-  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
-  const outDir = options.outputDir ?? join2(homedir(), "Downloads", "mcp-scraper", "media", `${stamp}-${domain}`);
-  mkdirSync(outDir, { recursive: true });
-  const assets = [];
-  await Promise.allSettled(
-    kept.map(async ({ url, type }, i) => {
-      const filename = safeFilename(url, i);
-      try {
-        const { savedPath, sizeBytes, mimeType } = await downloadAsset(url, outDir, filename);
-        const resolvedType = mimeType ? typeFromMime(mimeType) ?? type : type;
-        assets.push({ url, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes });
-      } catch {
-        assets.push({ url, type, mimeType: null, filename, savedPath: null, sizeBytes: null });
-      }
-    })
-  );
-  assets.sort((a, b) => {
-    if (a.savedPath && !b.savedPath) return -1;
-    if (!a.savedPath && b.savedPath) return 1;
-    return a.url.localeCompare(b.url);
-  });
-  return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound };
-}
-
 // src/api/site-mapper.ts
 var SKIP_EXTENSIONS = /* @__PURE__ */ new Set([
   "jpg",
@@ -7627,7 +7352,7 @@ var siteExtractFn = inngest.createFunction(
       return buildSeoAudit(pages, job.startUrl);
     }) : null;
     const artifacts = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-U4D5LW5W.js");
+      const { assembleExtractArtifacts } = await import("./extract-bundle-OSUPAHCE.js");
       const stored = await assembleExtractArtifacts(job, { branding, imageAudit, seoAudit });
       await completeExtractJob(jobId, stored);
       return stored;
@@ -8797,41 +8522,6 @@ async function resolveYouTubeAudio(videoId) {
   }
 }
 
-// src/services/media-transcription.ts
-import { fal } from "@fal-ai/client";
-function transcriptMarkdown(title, text, chunks, durationMs) {
-  const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
-  const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
-  if (chunks.length) {
-    lines.push("## Timestamped Segments", "");
-    for (const ch of chunks) {
-      lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
-    }
-  }
-  return lines.join("\n");
-}
-async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
-  const startMs = Date.now();
-  const result = await fal.subscribe("fal-ai/wizper", {
-    input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
-    logs: false,
-    pollInterval: 3e3
-  });
-  const data = result.data;
-  const text = data.text ?? "";
-  const chunks = data.chunks ?? [];
-  const durationMs = Date.now() - startMs;
-  void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
-  const costUsd = vendorCostUsd("fal_wizper", durationMs / 1e3);
-  return {
-    text,
-    chunks,
-    durationMs,
-    costUsd,
-    markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
-  };
-}
-
 // src/youtube/CaptionFetcher.ts
 async function fetchViaYoutubeTranscript(videoId) {
   try {
@@ -9134,6 +8824,7 @@ var ExtractSiteBodySchema = z11.object({
   rotateProxies: z11.boolean().optional(),
   rotateProxyEvery: z11.number().int().min(1).max(100).optional(),
   background: z11.boolean().optional(),
+  downloadImages: z11.boolean().optional(),
   formats: z11.array(z11.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
 });
 var YoutubeHarvestBodySchema = z11.object({
@@ -10047,11 +9738,13 @@ var FacebookAdExtractor = class {
       if (n) nameFreq.set(n, (nameFreq.get(n) ?? 0) + 1);
     }
     const advertiserName = nameFreq.size ? [...nameFreq.entries()].sort((a, b) => b[1] - a[1])[0][0] : null;
+    const matchedAdvertisers = [...nameFreq.entries()].map(([name, adCount]) => ({ name, adCount })).sort((a, b) => b.adCount - a.adCount).slice(0, 20);
     return {
       listingUrl,
       scannedAt: (/* @__PURE__ */ new Date()).toISOString(),
       advertiserPageId,
       advertiserName,
+      matchedAdvertisers,
       summary: {
         totalAds,
         activeCount,
@@ -10435,7 +10128,7 @@ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "b
 }
 
 // src/api/facebook-ad-routes.ts
-import { fal as fal2 } from "@fal-ai/client";
+import { fal } from "@fal-ai/client";
 var FacebookAdBodySchema = z13.object({
   url: z13.string().trim().optional(),
   libraryId: z13.string().trim().optional(),
@@ -10486,6 +10179,45 @@ function buildPageIntelUrl(body, country) {
   if (body.pageId?.trim()) return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&is_targeted_country=false&media_type=all&search_type=page&view_all_page_id=${body.pageId.trim()}`;
   return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
 }
+function normalizeAdvertiserName(value) {
+  return value.toLowerCase().normalize("NFKD").replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
+}
+function advertiserTokenOverlap(query, candidate) {
+  const queryTokens = new Set(normalizeAdvertiserName(query).split(" ").filter(Boolean));
+  const candidateTokens = new Set(normalizeAdvertiserName(candidate).split(" ").filter(Boolean));
+  if (queryTokens.size === 0 || candidateTokens.size === 0) return 0;
+  let overlap = 0;
+  for (const token of queryTokens) if (candidateTokens.has(token)) overlap++;
+  return overlap / queryTokens.size;
+}
+function assessPageIntelMatch(body, result) {
+  if (body.pageId) return { inputMode: "pageId", matchConfidence: "high", matchReason: "Exact Facebook page ID filter.", warnings: [] };
+  if (body.libraryId) return { inputMode: "libraryId", matchConfidence: "high", matchReason: "Exact Ad Library archive ID filter.", warnings: [] };
+  const query = body.query?.trim() ?? "";
+  const normalizedQuery = normalizeAdvertiserName(query);
+  const matches = result.matchedAdvertisers ?? [];
+  const exact = matches.find((match) => normalizeAdvertiserName(match.name) === normalizedQuery);
+  const top = matches[0];
+  const topShare = top && result.summary.totalAds > 0 ? top.adCount / result.summary.totalAds : 0;
+  const overlap = top ? advertiserTokenOverlap(query, top.name) : 0;
+  const warnings = [];
+  let matchConfidence;
+  let matchReason;
+  if (exact && exact.adCount / Math.max(1, result.summary.totalAds) >= 0.7) {
+    matchConfidence = "high";
+    matchReason = "The exact advertiser name accounts for most returned ads.";
+  } else if (top && overlap >= 0.75 && topShare >= 0.5) {
+    matchConfidence = "medium";
+    matchReason = "The leading advertiser overlaps strongly with the query, but keyword search is not page-scoped.";
+  } else {
+    matchConfidence = "low";
+    matchReason = "Keyword search did not identify one dominant exact advertiser match.";
+  }
+  if (matches.length > 1) warnings.push(`Keyword search mixed ${matches.length} advertiser names. Confirm the intended pageId before using these ads for performance or creative analysis.`);
+  if (matchConfidence === "low") warnings.push("Low-specificity Ad Library match: treat these results as discovery candidates, not as verified ads from the requested brand.");
+  warnings.push("Paused, cancelled, or otherwise inactive non-political ads may be absent from Ad Library. Use meta_ad_creative_media for connected ad-account creatives.");
+  return { inputMode: "query", matchConfidence, matchReason, warnings };
+}
 function configuredKernelProxy() {
   return (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim() || void 0;
 }
@@ -10582,8 +10314,10 @@ facebookAdApp.post("/page-intel", createApiKeyAuth(), async (c) => {
       await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: body.pageId ?? body.query ?? body.libraryId ?? "", error: "soft-block: empty result refunded" });
       return c.json({ error: "soft-block: no ads returned (refunded)" }, 503);
     }
-    await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "done", query: body.pageId ?? body.query ?? body.libraryId ?? "", resultCount: result.ads.length, result });
-    return c.json(result);
+    const match = assessPageIntelMatch(body, result);
+    const output = { ...result, ...match };
+    await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "done", query: body.pageId ?? body.query ?? body.libraryId ?? "", resultCount: result.ads.length, result: output });
+    return c.json(output);
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
     if (debited && !refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
@@ -10615,7 +10349,7 @@ facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
     metadata: { videoUrl }
   });
   if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-  fal2.config({ credentials: process.env.FAL_KEY });
+  fal.config({ credentials: process.env.FAL_KEY });
   const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
   let debited = false;
   try {
@@ -10659,7 +10393,7 @@ facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
     metadata: { url: sourceUrl.href }
   });
   if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-  fal2.config({ credentials: process.env.FAL_KEY });
+  fal.config({ credentials: process.env.FAL_KEY });
   const driver = new BrowserDriver();
   const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
   let debited = false;
@@ -10684,7 +10418,8 @@ facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
       text: transcript.text,
       chunks: transcript.chunks,
       durationMs: transcript.durationMs,
-      markdown: transcript.markdown
+      markdown: transcript.markdown,
+      transcriptSignal: assessTranscriptSignal(transcript.text, transcript.chunks, video.durationSec)
     };
     await logRequestEvent({ userId: fbUser.id, source: "facebook_video_transcribe", status: "done", query: sourceUrl.href, resultCount: transcript.chunks.length, result });
     return c.json(result);
@@ -11018,7 +10753,7 @@ var GoogleAdsExtractor = class {
 };
 
 // src/api/google-ads-routes.ts
-import { fal as fal3 } from "@fal-ai/client";
+import { fal as fal2 } from "@fal-ai/client";
 var GoogleAdsSearchBodySchema = z14.object({
   query: z14.string().trim().min(1, "query is required"),
   region: z14.string().trim().toUpperCase().length(2).optional(),
@@ -11148,7 +10883,7 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
   const user = c.get("user");
   const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
   if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-  fal3.config({ credentials: process.env.FAL_KEY });
+  fal2.config({ credentials: process.env.FAL_KEY });
   const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
   let debited = false;
   try {
@@ -11177,7 +10912,7 @@ import { Hono as Hono7 } from "hono";
 import { z as z15 } from "zod";
 import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
 import { tmpdir } from "os";
-import { join as join4 } from "path";
+import { join as join3 } from "path";
 import { spawn } from "child_process";
 
 // src/extractor/InstagramContentExtractor.ts
@@ -11775,8 +11510,8 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
             downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
             const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
             if (isMuxInput) {
-              if (!muxWorkDir) muxWorkDir = mkdtempSync(join4(tmpdir(), "ig-mux-"));
-              const localPath = join4(muxWorkDir, trackFilename(track, index, extraction.shortcode));
+              if (!muxWorkDir) muxWorkDir = mkdtempSync(join3(tmpdir(), "ig-mux-"));
+              const localPath = join3(muxWorkDir, trackFilename(track, index, extraction.shortcode));
               writeFileSync(localPath, bytes);
               muxInputs.set(track.url, localPath);
             }
@@ -11785,7 +11520,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
           }
         }
         if (muxEligible && muxWorkDir && muxInputs.has(extraction.selectedVideoTrack.url) && muxInputs.has(extraction.selectedAudioTrack.url)) {
-          const outPath = join4(muxWorkDir, "muxed.mp4");
+          const outPath = join3(muxWorkDir, "muxed.mp4");
           const muxed = await runFfmpegMux(muxInputs.get(extraction.selectedVideoTrack.url), muxInputs.get(extraction.selectedAudioTrack.url), outPath);
           if (muxed.ok) {
             const stored = await store.put(`${prefix}/muxed.mp4`, readFileSync2(outPath), "video/mp4");
@@ -13696,7 +13431,7 @@ import { Hono as Hono13 } from "hono";
 
 // src/directory/directory-workflow.ts
 import { mkdir as mkdir2, writeFile } from "fs/promises";
-import { join as join5 } from "path";
+import { join as join4 } from "path";
 import { z as z21 } from "zod";
 
 // src/directory/location-db.ts
@@ -14032,11 +13767,11 @@ function csvRowsFor(result) {
   return rows;
 }
 async function saveDirectoryCsv(result) {
-  const outDir = join5(outputBaseDir(), "directory-workflows");
+  const outDir = join4(outputBaseDir(), "directory-workflows");
   await mkdir2(outDir, { recursive: true });
   const stamp = result.extractedAt.replace(/[:.]/g, "-");
   const slug = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
-  const path5 = join5(outDir, `${stamp}-${slug}-directory-workflow.csv`);
+  const path5 = join4(outDir, `${stamp}-${slug}-directory-workflow.csv`);
   const headers = [
     "source_query",
     "source_location",
@@ -20608,8 +20343,8 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
 
 // src/api/scrape-blob-cleanup.ts
 import { readdir, stat, unlink } from "fs/promises";
-import { homedir as homedir2 } from "os";
-import { join as join6 } from "path";
+import { homedir } from "os";
+import { join as join5 } from "path";
 async function cleanupVercel(token, cutoff) {
   const { list, del } = await import("@vercel/blob");
   let cursor;
@@ -20626,8 +20361,8 @@ async function cleanupVercel(token, cutoff) {
   return { deleted, store: "vercel-blob" };
 }
 async function cleanupLocal(cutoff) {
-  const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir2(), "Downloads", "mcp-scraper");
-  const dir = join6(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
+  const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join5(homedir(), "Downloads", "mcp-scraper");
+  const dir = join5(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
   let deleted = 0;
   let entries;
   try {
@@ -20636,7 +20371,7 @@ async function cleanupLocal(cutoff) {
     return { deleted: 0, store: "local" };
   }
   for (const name of entries) {
-    const path5 = join6(dir, name);
+    const path5 = join5(dir, name);
     try {
       const s = await stat(path5);
       if (s.isFile() && s.mtimeMs < cutoff) {
@@ -22209,7 +21944,7 @@ var auth2 = createMiddleware3(async (c, next) => {
   const user = await getUserByApiKey(key);
   if (!user) return c.json({ error: "Invalid or inactive API key" }, 401);
   const refreshed = await applyMonthlyFreeRefresh(user);
-  const balanceMc = await reconcileBalanceMc(refreshed.id);
+  const balanceMc = await reconcileBalanceMc(refreshed.id, refreshed.balance_mc);
   c.set("user", { ...refreshed, balance_mc: balanceMc });
   return next();
 });
@@ -22228,7 +21963,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
   const user = await getUserById(userId);
   if (!user) return c.json({ error: "User not found" }, 401);
   const refreshed = await applyMonthlyFreeRefresh(user);
-  const balanceMc = await reconcileBalanceMc(refreshed.id);
+  const balanceMc = await reconcileBalanceMc(refreshed.id, refreshed.balance_mc);
   c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
   return next();
 });
@@ -22468,7 +22203,7 @@ app.get("/me", async (c) => {
   }
   const refreshed = await applyMonthlyFreeRefresh(foundUser);
   const [balanceMc, stats] = await Promise.all([
-    reconcileBalanceMc(refreshed.id),
+    reconcileBalanceMc(refreshed.id, refreshed.balance_mc),
     getUserStats(refreshed.id)
   ]);
   const user = { ...refreshed, balance_mc: balanceMc };
@@ -23892,7 +23627,8 @@ app.post("/extract-site", auth2, async (c) => {
   if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
   const parsed = checked.parsed;
   const user = c.get("user");
-  if (body.background === true) {
+  const downloadImages = body.downloadImages === true;
+  if (body.background === true || downloadImages) {
     const affordablePages = Math.floor(user.balance_mc / MC_COSTS.page_scrape);
     if (affordablePages < 1) return c.json(insufficientBalanceResponse(user.balance_mc, MC_COSTS.page_scrape), 402);
     const maxPages = Math.min(Math.min(1e4, Math.max(1, body.maxPages ?? 1e4)), affordablePages);
@@ -23907,6 +23643,7 @@ app.post("/extract-site", auth2, async (c) => {
         concurrency: concurrencyLimitForUser(user),
         urlsPerBrowser: body.rotateProxyEvery ?? 10,
         formats: body.formats,
+        downloadImages,
         heldMc
       });
       jobCreated = true;
@@ -24215,7 +23952,7 @@ app.post("/billing/concurrency/cancel", requireAllowedOrigin, sessionAuth, async
 });
 app.get("/billing/balance", auth2, async (c) => {
   const user = c.get("user");
-  const balanceMc = await reconcileBalanceMc(user.id);
+  const balanceMc = user.balance_mc;
   const ledger = await getLedger(user.id, 20);
   const freeCredits = await getFreeCreditBreakdown(user.id);
   return c.json({
@@ -24227,7 +23964,7 @@ app.get("/billing/balance", auth2, async (c) => {
 });
 app.post("/billing/credits", auth2, async (c) => {
   const user = c.get("user");
-  const balanceMc = await reconcileBalanceMc(user.id);
+  const balanceMc = user.balance_mc;
   const body = await c.req.json().catch(() => ({}));
   const query = body.item?.trim().toLowerCase();
   const costs = CREDIT_COST_CATALOG.map(({ aliases, ...cost }) => cost);
@@ -24260,7 +23997,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-JQTS437L.js");
+  const { drainQueue } = await import("./worker-O5PZTTPY.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
@@ -24288,15 +24025,15 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const jobId = c.req.param("id");
-  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-NVSZH35Y.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-U4D5LW5W.js");
+  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-GWKGK46Z.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-OSUPAHCE.js");
   const job = await getExtractJob2(jobId);
   if (!job) return c.json({ error: "job not found" }, 404);
   const stored = await assembleExtractArtifacts(job, {});
   await completeExtractJob2(jobId, stored);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-NVSZH35Y.js");
+    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-GWKGK46Z.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -24461,4 +24198,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-MQDCAR6I.js.map
+//# sourceMappingURL=server-VDWIPV7F.js.map