mcp-scraper 0.1.3 → 0.1.4

package/dist/{server-6CHHLOII.js → server-KUF3QJC7.js}

@@ -3,7 +3,7 @@ import {
   CaptureSerpSnapshotInputSchema,
   HttpMcpToolExecutor,
   buildPaaExtractorMcpServer
-} from "./chunk-JQKZWEON.js";
+} from "./chunk-DZY3XO3M.js";
 import {
   BALANCE_PACK_LABELS,
   BALANCE_PRICE_IDS,
@@ -19,7 +19,7 @@ import {
   harvestProblemResponse,
   insufficientBalanceResponse,
   serializeHarvestProblem
-} from "./chunk-Y74EXABN.js";
+} from "./chunk-7HB7NDOY.js";
 import {
   BrowserDriver,
   CaptchaError,
@@ -30,8 +30,9 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema,
   buildYouTubeChannelVideosUrl,
-  harvest
-} from "./chunk-HERFK7W6.js";
+  harvest,
+  resolveKernelProxyId
+} from "./chunk-W4P2U5VF.js";
 import {
   SiteAuditJobRowSchema,
   cancelJob,
@@ -8767,6 +8768,106 @@ var FacebookAdExtractor = class {
   }
 };
 
+// src/extractor/FacebookAdGraphql.ts
+var AD_LIBRARY_QUERY = "AdLibrarySearchPaginationQuery";
+function parseFbGraphqlJson(text) {
+  const out = [];
+  const body = text.replace(/^for\s*\(;;\);/, "").trim();
+  try {
+    out.push(JSON.parse(body));
+    return out;
+  } catch {
+    for (const line of body.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      try {
+        out.push(JSON.parse(trimmed));
+      } catch {
+        continue;
+      }
+    }
+    return out;
+  }
+}
+function extractCollatedResults(payload) {
+  const root = payload;
+  const edges = root?.data?.ad_library_main?.search_results_connection?.edges ?? [];
+  const results = [];
+  for (const edge of edges) {
+    const node = edge?.node;
+    for (const raw of node?.collated_results ?? []) {
+      const r = raw;
+      const id = r.ad_archive_id;
+      if (id === void 0 || id === null) continue;
+      const snapshot = r.snapshot ?? null;
+      results.push({
+        ad_archive_id: String(id),
+        page_id: r.page_id != null ? String(r.page_id) : "",
+        page_name: r.page_name ?? snapshot?.page_name ?? "",
+        is_active: Boolean(r.is_active),
+        collation_count: typeof r.collation_count === "number" ? r.collation_count : null,
+        snapshot
+      });
+    }
+  }
+  return results;
+}
+async function collectAdLibraryResults(page, url, maxResults, opts = {}) {
+  const captureMs = opts.captureMs ?? 3e4;
+  const collected = [];
+  const seen = /* @__PURE__ */ new Set();
+  const handler = (resp) => {
+    if (!resp.url().includes("/api/graphql")) return;
+    const friendlyName = (resp.request().postData() ?? "").match(/fb_api_req_friendly_name=([^&]+)/)?.[1];
+    if (friendlyName !== AD_LIBRARY_QUERY) return;
+    void resp.text().then((text) => {
+      for (const payload of parseFbGraphqlJson(text)) {
+        for (const result of extractCollatedResults(payload)) {
+          if (seen.has(result.ad_archive_id)) continue;
+          seen.add(result.ad_archive_id);
+          collected.push(result);
+        }
+      }
+    }).catch(() => void 0);
+  };
+  page.on("response", handler);
+  try {
+    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+    const deadline = Date.now() + captureMs;
+    let lastCount = -1;
+    let stableRounds = 0;
+    while (Date.now() < deadline && collected.length < maxResults) {
+      await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)).catch(() => void 0);
+      await page.waitForTimeout(2e3);
+      if (collected.length === lastCount) {
+        stableRounds++;
+        if (stableRounds >= 2 && collected.length > 0) break;
+      } else {
+        stableRounds = 0;
+      }
+      lastCount = collected.length;
+    }
+  } finally {
+    page.off("response", handler);
+  }
+  return collected.slice(0, maxResults);
+}
+function advertisersFromResults(results, maxResults) {
+  const byPage = /* @__PURE__ */ new Map();
+  for (const r of results) {
+    if (!r.page_id || !r.page_name) continue;
+    const collation = typeof r.collation_count === "number" && r.collation_count > 0 ? r.collation_count : 0;
+    const existing = byPage.get(r.page_id);
+    if (existing) {
+      existing.resultCount++;
+      existing.maxCollation = Math.max(existing.maxCollation, collation);
+    } else {
+      byPage.set(r.page_id, { pageName: r.page_name, pageId: r.page_id, sampleLibraryId: r.ad_archive_id, maxCollation: collation, resultCount: 1 });
+    }
+  }
+  return [...byPage.values()].map((e) => ({ pageName: e.pageName, pageId: e.pageId, sampleLibraryId: e.sampleLibraryId, adCount: Math.max(e.maxCollation, e.resultCount) })).sort((a, b) => b.adCount - a.adCount).slice(0, maxResults);
+}
+
 // src/api/facebook-ad-routes.ts
 import { fal as fal2 } from "@fal-ai/client";
 var FacebookAdBodySchema = z13.object({
@@ -8811,6 +8912,22 @@ function buildPageIntelUrl(body, country) {
 function kernelLaunchOpts() {
   return { headless: true, kernelApiKey: process.env.KERNEL_API_KEY?.trim(), kernelProxyId: process.env.KERNEL_PROXY_ID?.trim(), viewport: { width: 1280, height: 900 }, locale: "en-US" };
 }
+async function kernelLaunchOptsResidential() {
+  let proxyId = process.env.KERNEL_PROXY_ID?.trim();
+  try {
+    const resolution = await resolveKernelProxyId({
+      kernelApiKey: process.env.KERNEL_API_KEY?.trim(),
+      proxyMode: "location",
+      configuredKernelProxyId: process.env.KERNEL_PROXY_ID?.trim(),
+      location: "New York, NY",
+      gl: "us"
+    });
+    if (resolution.kernelProxyId) proxyId = resolution.kernelProxyId;
+  } catch {
+    proxyId = process.env.KERNEL_PROXY_ID?.trim();
+  }
+  return { headless: true, kernelApiKey: process.env.KERNEL_API_KEY?.trim(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
+}
 var facebookAdApp = new Hono4();
 facebookAdApp.post("/ad", createApiKeyAuth(), async (c) => {
   const raw = await c.req.json().catch(() => ({}));
@@ -8867,7 +8984,7 @@ facebookAdApp.post("/page-intel", createApiKeyAuth(), async (c) => {
   const driver = new BrowserDriver();
   let refunded = false;
   try {
-    await driver.launch(kernelLaunchOpts());
+    await driver.launch(await kernelLaunchOptsResidential());
     await driver.navigateTo(listingUrl);
     const extractor = new FacebookAdExtractor(driver);
     const result = await extractor.extractPageIntel(listingUrl, maxAds);
@@ -8951,18 +9068,15 @@ facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
   const driver = new BrowserDriver();
   let searchRefunded = false;
   try {
-    await driver.launch(kernelLaunchOpts());
+    await driver.launch(await kernelLaunchOptsResidential());
     const page = driver.getPage();
-    await driver.navigateTo(searchUrl);
-    try {
-      await page.waitForFunction(
-        () => {
-          const bt = document.body ? document.body.innerText ?? "" : "";
-          return bt.includes("Library ID") || bt.includes("No results");
-        },
-        { timeout: 2e4, polling: 500 }
-      );
-    } catch {
+    const collated = await collectAdLibraryResults(page, searchUrl, Math.max(maxResults * 4, 40));
+    const gqlAdvertisers = advertisersFromResults(collated, maxResults);
+    if (gqlAdvertisers.length > 0) {
+      const results2 = gqlAdvertisers.map((a) => ({ name: a.pageName, pageName: a.pageName, pageId: a.pageId, libraryId: a.sampleLibraryId, sampleLibraryId: a.sampleLibraryId, adCount: a.adCount }));
+      const searchResult2 = { query: body.query.trim(), searchUrl, results: results2, via: "graphql" };
+      await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "done", query: body.query.trim(), resultCount: results2.length, result: searchResult2 });
+      return c.json(searchResult2);
     }
     await page.waitForTimeout(1500);
     for (let scroll = 0; scroll < 3; scroll++) {
@@ -9008,7 +9122,7 @@ facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
         advertiserMap.set(pageName, { pageName, sampleLibraryId: libraryId, adCount: 1 });
       }
     }
-    const results = [...advertiserMap.values()].sort((a, b) => b.adCount - a.adCount).slice(0, maxResults);
+    const results = [...advertiserMap.values()].sort((a, b) => b.adCount - a.adCount).slice(0, maxResults).map((a) => ({ name: a.pageName, pageName: a.pageName, libraryId: a.sampleLibraryId, sampleLibraryId: a.sampleLibraryId, adCount: a.adCount }));
     const searchResult = { query: body.query.trim(), searchUrl, results };
     await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "done", query: body.query.trim(), resultCount: results.length, result: searchResult });
     return c.json(searchResult);
@@ -11500,7 +11614,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-D4D2YQTA.js");
+  const { drainQueue } = await import("./worker-UT4ZQU2T.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const [results, sweepResult] = await Promise.all([
     drainQueue(budget),
@@ -11622,4 +11736,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-6CHHLOII.js.map
+//# sourceMappingURL=server-KUF3QJC7.js.map