mcp-scraper 0.1.3 → 0.1.5

package/dist/{server-6CHHLOII.js → server-V5XMVRYE.js}

@@ -2,8 +2,9 @@ import {
   CaptureSerpPageSnapshotsInputSchema,
   CaptureSerpSnapshotInputSchema,
   HttpMcpToolExecutor,
-  buildPaaExtractorMcpServer
-} from "./chunk-JQKZWEON.js";
+  buildPaaExtractorMcpServer,
+  harvestTimeoutBudget
+} from "./chunk-4OHPDEZM.js";
 import {
   BALANCE_PACK_LABELS,
   BALANCE_PRICE_IDS,
@@ -19,7 +20,7 @@ import {
   harvestProblemResponse,
   insufficientBalanceResponse,
   serializeHarvestProblem
-} from "./chunk-Y74EXABN.js";
+} from "./chunk-7HB7NDOY.js";
 import {
   BrowserDriver,
   CaptchaError,
@@ -30,8 +31,9 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema,
   buildYouTubeChannelVideosUrl,
-  harvest
-} from "./chunk-HERFK7W6.js";
+  harvest,
+  resolveKernelProxyId
+} from "./chunk-W4P2U5VF.js";
 import {
   SiteAuditJobRowSchema,
   cancelJob,
@@ -3430,11 +3432,6 @@ async function validatePublicHttpUrl(raw, opts) {
   return { parsed };
 }
 
-// src/api/server.ts
-import { mkdirSync as mkdirSync2, writeFileSync } from "fs";
-import { homedir as homedir2 } from "os";
-import { join as join4 } from "path";
-
 // src/api/kpo-extractor.ts
 import TurndownService from "turndown";
 
@@ -3849,25 +3846,73 @@ function firstFont(fontFamily) {
   const first = fontFamily.split(",")[0].trim().replace(/['"]/g, "");
   return first || null;
 }
+function dominantColor(freq) {
+  return Object.entries(freq).filter(([hex]) => !isTransparentOrWhite(hex) && hex !== "#000000" && hex !== "#020101").sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
+}
 async function extractBrandingFromPage(page) {
   const evalScript = `
     (function() {
       function cs(el) { return el ? window.getComputedStyle(el) : null; }
+      function toHex(rgb) {
+        var m = rgb && rgb.match(/rgba?\\((\\d+),\\s*(\\d+),\\s*(\\d+)/);
+        if (!m) return null;
+        return '#' + [m[1],m[2],m[3]].map(function(v){ return ('0'+parseInt(v).toString(16)).slice(-2); }).join('');
+      }
+      function isUsable(hex) {
+        if (!hex) return false;
+        if (hex === '#000000' || hex === '#020101' || hex === '#ffffff' || hex === '#fffffe') return false;
+        var r=parseInt(hex.slice(1,3),16), g=parseInt(hex.slice(3,5),16), b=parseInt(hex.slice(5,7),16);
+        return (0.2126*r + 0.7152*g + 0.0722*b) <= 230;
+      }
+
       var navEl  = document.querySelector('nav, header, [role="banner"]');
       var bodyEl = document.body;
       var h1El   = document.querySelector('h1');
       var btnEl  = document.querySelector(
         'a.btn-primary, button.btn-primary, .btn-primary, .cta-btn,' +
         'a.button--primary, button.button--primary, [class*="btn-cta"],' +
-        '[class*="cta-button"], .wp-block-button__link, [class*="hero"] a'
+        '[class*="cta-button"], .wp-block-button__link, [class*="hero"] a,' +
+        '.elementor-button, .elementor-button-link,' +
+        '.et_pb_button,' +
+        '.fl-button,' +
+        '.vc_btn,' +
+        '[class*="cta"][href], [class*="get-started"], [class*="contact-btn"]'
       );
       var navStyle  = cs(navEl);
       var bodyStyle = cs(bodyEl);
       var h1Style   = cs(h1El);
       var btnStyle  = cs(btnEl);
-      var pageHost = window.location.hostname.replace(/^www./, '');
+
+      var svgFreq = {};
+      var svgScope = navEl || document.querySelector('header, [role="banner"]');
+      if (svgScope) {
+        var svgEls = svgScope.querySelectorAll('svg *, [fill], path, circle, rect, polygon, polyline');
+        for (var si = 0; si < svgEls.length; si++) {
+          var svgEl = svgEls[si];
+          var fillComp = cs(svgEl) ? cs(svgEl).fill : null;
+          var fillAttr = svgEl.getAttribute('fill');
+          var fillHex = null;
+          if (fillComp && fillComp !== 'none') { fillHex = toHex(fillComp); }
+          else if (fillAttr && fillAttr !== 'none' && fillAttr.startsWith('#')) { fillHex = fillAttr; }
+          if (fillHex && isUsable(fillHex)) { svgFreq[fillHex] = (svgFreq[fillHex] || 0) + 1; }
+        }
+      }
+
+      var navChildBgFreq = {};
+      if (navEl) {
+        var navChildren = navEl.querySelectorAll('li, a, button, [class*="menu-item"]');
+        for (var ni = 0; ni < navChildren.length; ni++) {
+          var nbg = cs(navChildren[ni]);
+          if (nbg) {
+            var bghex = toHex(nbg.backgroundColor);
+            if (bghex && isUsable(bghex)) { navChildBgFreq[bghex] = (navChildBgFreq[bghex] || 0) + 1; }
+          }
+        }
+      }
+
+      var pageHost = window.location.hostname.replace(/^www\\./, '');
       function isSameDomain(src) {
-        try { return new URL(src).hostname.replace(/^www./, '').endsWith(pageHost); } catch { return false; }
+        try { return new URL(src).hostname.replace(/^www\\./, '').endsWith(pageHost); } catch { return false; }
       }
       var logoSelectors = [
         'header img[class*="logo"]', 'nav img[class*="logo"]',
@@ -3890,22 +3935,27 @@ async function extractBrandingFromPage(page) {
         'link[rel~="icon"], link[rel="shortcut icon"], link[rel="apple-touch-icon"]'
       );
       return {
-        navBg:       navStyle  ? navStyle.backgroundColor  : null,
-        bodyBg:      bodyStyle ? bodyStyle.backgroundColor : null,
-        bodyColor:   bodyStyle ? bodyStyle.color           : null,
-        h1Color:     h1Style   ? h1Style.color             : null,
-        btnBg:       btnStyle  ? btnStyle.backgroundColor  : null,
-        bodyFont:    bodyStyle ? bodyStyle.fontFamily      : null,
-        h1Font:      h1Style   ? h1Style.fontFamily        : null,
-        logoSrc:     logoSrc,
-        faviconHref: faviconEl ? faviconEl.href            : null,
+        navBg:          navStyle  ? navStyle.backgroundColor  : null,
+        bodyBg:         bodyStyle ? bodyStyle.backgroundColor : null,
+        bodyColor:      bodyStyle ? bodyStyle.color           : null,
+        h1Color:        h1Style   ? h1Style.color             : null,
+        btnBg:          btnStyle  ? btnStyle.backgroundColor  : null,
+        bodyFont:       bodyStyle ? bodyStyle.fontFamily      : null,
+        h1Font:         h1Style   ? h1Style.fontFamily        : null,
+        logoSrc:        logoSrc,
+        faviconHref:    faviconEl ? faviconEl.href            : null,
+        svgFreq:        svgFreq,
+        navChildBgFreq: navChildBgFreq,
       };
     })()
   `;
   const raw = await page.evaluate(evalScript);
   const navBgHex = rgbToHex(raw.navBg ?? "");
   const bodyBgHex = rgbToHex(raw.bodyBg ?? "");
-  const primary = !isTransparentOrWhite(navBgHex) ? navBgHex : bodyBgHex;
+  const navBgUsable = navBgHex && !isTransparentOrWhite(navBgHex) && navBgHex !== "#000000" && navBgHex !== "#020101" ? navBgHex : null;
+  const svgPrimary = dominantColor(raw.svgFreq ?? {});
+  const navChildBg = dominantColor(raw.navChildBgFreq ?? {});
+  const primary = navBgUsable ?? svgPrimary ?? navChildBg ?? bodyBgHex;
   const accent = rgbToHex(raw.btnBg ?? "");
   const text = rgbToHex(raw.bodyColor ?? "");
   const heading = rgbToHex(raw.h1Color ?? "");
@@ -8767,6 +8817,106 @@ var FacebookAdExtractor = class {
   }
 };
 
+// src/extractor/FacebookAdGraphql.ts
+var AD_LIBRARY_QUERY = "AdLibrarySearchPaginationQuery";
+function parseFbGraphqlJson(text) {
+  const out = [];
+  const body = text.replace(/^for\s*\(;;\);/, "").trim();
+  try {
+    out.push(JSON.parse(body));
+    return out;
+  } catch {
+    for (const line of body.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      try {
+        out.push(JSON.parse(trimmed));
+      } catch {
+        continue;
+      }
+    }
+    return out;
+  }
+}
+function extractCollatedResults(payload) {
+  const root = payload;
+  const edges = root?.data?.ad_library_main?.search_results_connection?.edges ?? [];
+  const results = [];
+  for (const edge of edges) {
+    const node = edge?.node;
+    for (const raw of node?.collated_results ?? []) {
+      const r = raw;
+      const id = r.ad_archive_id;
+      if (id === void 0 || id === null) continue;
+      const snapshot = r.snapshot ?? null;
+      results.push({
+        ad_archive_id: String(id),
+        page_id: r.page_id != null ? String(r.page_id) : "",
+        page_name: r.page_name ?? snapshot?.page_name ?? "",
+        is_active: Boolean(r.is_active),
+        collation_count: typeof r.collation_count === "number" ? r.collation_count : null,
+        snapshot
+      });
+    }
+  }
+  return results;
+}
+async function collectAdLibraryResults(page, url, maxResults, opts = {}) {
+  const captureMs = opts.captureMs ?? 3e4;
+  const collected = [];
+  const seen = /* @__PURE__ */ new Set();
+  const handler = (resp) => {
+    if (!resp.url().includes("/api/graphql")) return;
+    const friendlyName = (resp.request().postData() ?? "").match(/fb_api_req_friendly_name=([^&]+)/)?.[1];
+    if (friendlyName !== AD_LIBRARY_QUERY) return;
+    void resp.text().then((text) => {
+      for (const payload of parseFbGraphqlJson(text)) {
+        for (const result of extractCollatedResults(payload)) {
+          if (seen.has(result.ad_archive_id)) continue;
+          seen.add(result.ad_archive_id);
+          collected.push(result);
+        }
+      }
+    }).catch(() => void 0);
+  };
+  page.on("response", handler);
+  try {
+    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+    const deadline = Date.now() + captureMs;
+    let lastCount = -1;
+    let stableRounds = 0;
+    while (Date.now() < deadline && collected.length < maxResults) {
+      await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)).catch(() => void 0);
+      await page.waitForTimeout(2e3);
+      if (collected.length === lastCount) {
+        stableRounds++;
+        if (stableRounds >= 2 && collected.length > 0) break;
+      } else {
+        stableRounds = 0;
+      }
+      lastCount = collected.length;
+    }
+  } finally {
+    page.off("response", handler);
+  }
+  return collected.slice(0, maxResults);
+}
+function advertisersFromResults(results, maxResults) {
+  const byPage = /* @__PURE__ */ new Map();
+  for (const r of results) {
+    if (!r.page_id || !r.page_name) continue;
+    const collation = typeof r.collation_count === "number" && r.collation_count > 0 ? r.collation_count : 0;
+    const existing = byPage.get(r.page_id);
+    if (existing) {
+      existing.resultCount++;
+      existing.maxCollation = Math.max(existing.maxCollation, collation);
+    } else {
+      byPage.set(r.page_id, { pageName: r.page_name, pageId: r.page_id, sampleLibraryId: r.ad_archive_id, maxCollation: collation, resultCount: 1 });
+    }
+  }
+  return [...byPage.values()].map((e) => ({ pageName: e.pageName, pageId: e.pageId, sampleLibraryId: e.sampleLibraryId, adCount: Math.max(e.maxCollation, e.resultCount) })).sort((a, b) => b.adCount - a.adCount).slice(0, maxResults);
+}
+
 // src/api/facebook-ad-routes.ts
 import { fal as fal2 } from "@fal-ai/client";
 var FacebookAdBodySchema = z13.object({
@@ -8811,6 +8961,22 @@ function buildPageIntelUrl(body, country) {
 function kernelLaunchOpts() {
   return { headless: true, kernelApiKey: process.env.KERNEL_API_KEY?.trim(), kernelProxyId: process.env.KERNEL_PROXY_ID?.trim(), viewport: { width: 1280, height: 900 }, locale: "en-US" };
 }
+async function kernelLaunchOptsResidential() {
+  let proxyId = process.env.KERNEL_PROXY_ID?.trim();
+  try {
+    const resolution = await resolveKernelProxyId({
+      kernelApiKey: process.env.KERNEL_API_KEY?.trim(),
+      proxyMode: "location",
+      configuredKernelProxyId: process.env.KERNEL_PROXY_ID?.trim(),
+      location: "New York, NY",
+      gl: "us"
+    });
+    if (resolution.kernelProxyId) proxyId = resolution.kernelProxyId;
+  } catch {
+    proxyId = process.env.KERNEL_PROXY_ID?.trim();
+  }
+  return { headless: true, kernelApiKey: process.env.KERNEL_API_KEY?.trim(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
+}
 var facebookAdApp = new Hono4();
 facebookAdApp.post("/ad", createApiKeyAuth(), async (c) => {
   const raw = await c.req.json().catch(() => ({}));
@@ -8867,7 +9033,7 @@ facebookAdApp.post("/page-intel", createApiKeyAuth(), async (c) => {
   const driver = new BrowserDriver();
   let refunded = false;
   try {
-    await driver.launch(kernelLaunchOpts());
+    await driver.launch(await kernelLaunchOptsResidential());
     await driver.navigateTo(listingUrl);
     const extractor = new FacebookAdExtractor(driver);
     const result = await extractor.extractPageIntel(listingUrl, maxAds);
@@ -8951,18 +9117,15 @@ facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
   const driver = new BrowserDriver();
   let searchRefunded = false;
   try {
-    await driver.launch(kernelLaunchOpts());
+    await driver.launch(await kernelLaunchOptsResidential());
     const page = driver.getPage();
-    await driver.navigateTo(searchUrl);
-    try {
-      await page.waitForFunction(
-        () => {
-          const bt = document.body ? document.body.innerText ?? "" : "";
-          return bt.includes("Library ID") || bt.includes("No results");
-        },
-        { timeout: 2e4, polling: 500 }
-      );
-    } catch {
+    const collated = await collectAdLibraryResults(page, searchUrl, Math.max(maxResults * 4, 40));
+    const gqlAdvertisers = advertisersFromResults(collated, maxResults);
+    if (gqlAdvertisers.length > 0) {
+      const results2 = gqlAdvertisers.map((a) => ({ name: a.pageName, pageName: a.pageName, pageId: a.pageId, libraryId: a.sampleLibraryId, sampleLibraryId: a.sampleLibraryId, adCount: a.adCount }));
+      const searchResult2 = { query: body.query.trim(), searchUrl, results: results2, via: "graphql" };
+      await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "done", query: body.query.trim(), resultCount: results2.length, result: searchResult2 });
+      return c.json(searchResult2);
     }
     await page.waitForTimeout(1500);
     for (let scroll = 0; scroll < 3; scroll++) {
@@ -9008,7 +9171,7 @@ facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
         advertiserMap.set(pageName, { pageName, sampleLibraryId: libraryId, adCount: 1 });
       }
     }
-    const results = [...advertiserMap.values()].sort((a, b) => b.adCount - a.adCount).slice(0, maxResults);
+    const results = [...advertiserMap.values()].sort((a, b) => b.adCount - a.adCount).slice(0, maxResults).map((a) => ({ name: a.pageName, pageName: a.pageName, libraryId: a.sampleLibraryId, sampleLibraryId: a.sampleLibraryId, adCount: a.adCount }));
     const searchResult = { query: body.query.trim(), searchUrl, results };
     await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "done", query: body.query.trim(), resultCount: results.length, result: searchResult });
     return c.json(searchResult);
@@ -11029,7 +11192,11 @@ app.delete("/api-key", requireAllowedOrigin, sessionAuth, async (c) => {
 var BYPASS_EMAILS = new Set(
   (process.env.HARVEST_LIMIT_BYPASS_EMAILS ?? "").split(",").map((e) => e.trim()).filter(Boolean)
 );
-var SYNC_HARVEST_TIMEOUT_MS = Number(process.env.SYNC_HARVEST_TIMEOUT_MS ?? 105e3);
+var SYNC_HARVEST_TIMEOUT_OVERRIDE_MS = (() => {
+  const raw = process.env.SYNC_HARVEST_TIMEOUT_MS;
+  const parsed = raw === void 0 ? NaN : Number(raw);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
+})();
 function combineAbortSignals(signals) {
   const controller = new AbortController();
   const abortFrom = (signal) => {
@@ -11129,9 +11296,10 @@ app.post("/harvest/sync", auth, async (c) => {
   if (!syncOk) return c.json(insufficientBalanceResponse(syncBal, syncCost), 402);
   const jobId = await createRunningJob(user.id, options.query, options);
   const recordAttempt = createHarvestAttemptRecorder(jobId, user.id);
+  const syncTimeoutMs = SYNC_HARVEST_TIMEOUT_OVERRIDE_MS ?? harvestTimeoutBudget(options.maxQuestions, options.serpOnly).serverMs;
   const syncSignal = combineAbortSignals([
     c.req.raw.signal,
-    AbortSignal.timeout(Number.isFinite(SYNC_HARVEST_TIMEOUT_MS) && SYNC_HARVEST_TIMEOUT_MS > 0 ? SYNC_HARVEST_TIMEOUT_MS : 105e3)
+    AbortSignal.timeout(syncTimeoutMs)
   ]);
   try {
     const result = await harvest({
@@ -11279,13 +11447,7 @@ app.post("/extract-url", auth, async (c) => {
     const brandingData = pageData?.branding ?? null;
     let screenshotMeta = null;
     if (screenshotBuf) {
-      const outDir = join4(homedir2(), "Downloads", "mcp-scraper", "screenshots");
-      mkdirSync2(outDir, { recursive: true });
-      const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-      const slug = canonicalUrl.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
-      const filePath = join4(outDir, `${stamp}-${slug}.png`);
-      writeFileSync(filePath, screenshotBuf);
-      screenshotMeta = { savedPath: filePath, sizeBytes: screenshotBuf.length, device };
+      screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
     }
     const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"] }) : null;
     await logRequestEvent({ userId: user.id, source: "extract_url", status: "done", query: canonicalUrl, resultCount: result.headings.length, result });
@@ -11500,7 +11662,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-D4D2YQTA.js");
+  const { drainQueue } = await import("./worker-UT4ZQU2T.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const [results, sweepResult] = await Promise.all([
     drainQueue(budget),
@@ -11622,4 +11784,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-6CHHLOII.js.map
+//# sourceMappingURL=server-V5XMVRYE.js.map