mcp-scraper 0.3.28 → 0.3.40

package/dist/{db-FP2ABUU4.js → db-LYQENFPW.js}

@@ -97,7 +97,7 @@ import {
   stripeEventAlreadyProcessed,
   updateKpoJobState,
   verifyPassword
-} from "./chunk-TRNSXR46.js";
+} from "./chunk-DVRPXPYH.js";
 export {
   CREDIT_LOT_TTL,
   SiteAuditJobRowSchema,
@@ -198,4 +198,4 @@ export {
   updateKpoJobState,
   verifyPassword
 };
-//# sourceMappingURL=db-FP2ABUU4.js.map
+//# sourceMappingURL=db-LYQENFPW.js.map

package/dist/index.cjs

@@ -1597,6 +1597,13 @@ var PAAExtractor = class {
   }
   driver;
   reporter;
+  completeness = {
+    paaWithoutAnswer: 0,
+    paaWithoutSource: 0,
+    paaAnswersRecovered: 0,
+    aioShareCaptured: null
+  };
+  aioShareUrlEarly = null;
   normalizeQuestion(q) {
     return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
   }
@@ -1725,6 +1732,7 @@ var PAAExtractor = class {
       }
     }
     const itemMap = new Map((await this.extractVisibleItems(page)).map((i) => [i.question, i]));
+    await this.fillIncompleteItems(page, orderedQs, itemMap, options);
     for (const q of orderedQs) {
       if (results.length >= options.maxQuestions) break;
       const key = this.normalizeQuestion(q);
@@ -1738,8 +1746,52 @@ var PAAExtractor = class {
         results.push(this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, 1, null, options.query));
       }
     }
+    this.completeness.paaWithoutAnswer = results.filter((r) => !r.answer).length;
+    this.completeness.paaWithoutSource = results.filter((r) => !r.source_title && !r.source_site && !r.source_cite).length;
     return results;
   }
+  async fillIncompleteItems(page, orderedQs, itemMap, options) {
+    const kept = new Set(orderedQs.slice(0, options.maxQuestions));
+    const missingBefore = new Set([...kept].filter((q) => !itemMap.get(q)?.answer));
+    if (missingBefore.size === 0) return;
+    for (let round = 0; round < 2; round++) {
+      if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
+      const stillMissing = [...missingBefore].filter((q) => !itemMap.get(q)?.answer);
+      if (stillMissing.length === 0) break;
+      const reExpanded = await this.reExpandCollapsedItems(page).catch(() => false);
+      await page.waitForTimeout(reExpanded ? 900 : 500);
+      let improved = false;
+      for (const item of await this.extractVisibleItems(page)) {
+        if (!kept.has(item.question)) continue;
+        const existing = itemMap.get(item.question);
+        if (!item.answer || existing?.answer) continue;
+        itemMap.set(item.question, {
+          ...item,
+          sourceTitle: item.sourceTitle ?? existing?.sourceTitle,
+          sourceSite: item.sourceSite ?? existing?.sourceSite,
+          sourceCite: item.sourceCite ?? existing?.sourceCite
+        });
+        improved = true;
+      }
+      if (!improved) break;
+    }
+    this.completeness.paaAnswersRecovered = [...missingBefore].filter((q) => itemMap.get(q)?.answer).length;
+  }
+  async reExpandCollapsedItems(page) {
+    const unexpandedSel = `${PAASelectors.item}:not(.${PAASelectors.expandedClass}) ${PAASelectors.clickTarget}`;
+    const count = await page.locator(unexpandedSel).count().catch(() => 0);
+    if (count === 0) return false;
+    for (let i = 0; i < count; i++) {
+      try {
+        const btn = page.locator(unexpandedSel).first();
+        await btn.scrollIntoViewIfNeeded();
+        await btn.click({ force: true });
+        await page.waitForTimeout(300);
+      } catch {
+      }
+    }
+    return true;
+  }
   async extractVideos(page) {
     const vsels = VideoSelectors;
     return page.evaluate((sels) => {
@@ -2074,12 +2126,58 @@ var PAAExtractor = class {
     }
     return { ...entityIds, entities: records, cids: [...cidSet] };
   }
-  async extractAISurfaces(page) {
-    return page.evaluate(extractAISurfacesFromDocument, {
+  async extractAISurfaces(page, options) {
+    const surfaces = await page.evaluate(extractAISurfacesFromDocument, {
       aio: AIOverviewSelectors,
       aim: AIModeSelectors,
       expandWaitMs: 1500
     });
+    if (!surfaces.aiOverview.detected) {
+      this.completeness.aioShareCaptured = null;
+      return surfaces;
+    }
+    const shareUrl = this.aioShareUrlEarly ?? await this.captureAIOverviewShareUrl(page, options).catch(() => null);
+    this.completeness.aioShareCaptured = shareUrl !== null;
+    return {
+      ...surfaces,
+      aiOverview: { ...surfaces.aiOverview, shareUrl }
+    };
+  }
+  async captureAIOverviewShareUrl(page, options) {
+    for (let attempt = 0; attempt < 3; attempt++) {
+      if (options?.softDeadlineMs && Date.now() >= options.softDeadlineMs) return null;
+      const clicked = await page.evaluate(() => {
+        const heads = Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"],div')).filter((h) => (h.textContent || "").trim() === "AI Overview");
+        for (const head of heads) {
+          let sec = head;
+          for (let up = 0; up < 8 && sec; up++) {
+            const btn = sec.querySelector('[aria-label="Share"]');
+            if (btn) {
+              btn.scrollIntoView({ block: "center" });
+              btn.click();
+              return true;
+            }
+            sec = sec.parentElement;
+          }
+        }
+        return false;
+      }).catch(() => false);
+      if (!clicked) return null;
+      await page.waitForTimeout(4500);
+      const url = await page.evaluate(() => {
+        const w = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
+        while (w.nextNode()) {
+          const m = (w.currentNode.textContent || "").match(/https:\/\/share\.google\/aimode\/\S+/);
+          if (m) return m[0];
+        }
+        return null;
+      }).catch(() => null);
+      await page.keyboard.press("Escape").catch(() => {
+      });
+      if (url) return url;
+      await page.waitForTimeout(600);
+    }
+    return null;
   }
   buildTree(flat, _seed) {
     const roots = [];
@@ -2132,6 +2230,8 @@ var PAAExtractor = class {
   }
   async extract(options, signal) {
     const startMs = Date.now();
+    this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
+    this.aioShareUrlEarly = null;
     const isMobile = options.device === "mobile";
     const config = {
       headless: options.headless,
@@ -2170,6 +2270,7 @@ var PAAExtractor = class {
       this.throwIfAborted(signal);
       const page = this.driver.getPage();
       await this.throwIfCaptcha(page, "Google SERP");
+      this.aioShareUrlEarly = await this.captureAIOverviewShareUrl(page, options).catch(() => null);
       if (options.serpOnly) {
         const [organicResults2, localPack2, rawEntityIds2] = await Promise.all([
           this.extractOrganicResults(page),
@@ -2177,7 +2278,7 @@ var PAAExtractor = class {
           this.extractEntityIds(page)
         ]);
         const entityIds2 = this.mergeLocalPackIntoEntities(rawEntityIds2, localPack2);
-        const aiSurfaces2 = await this.extractAISurfaces(page);
+        const aiSurfaces2 = await this.extractAISurfaces(page, options);
         let locationEvidence2 = options.debug ? inferSerpLocationEvidence(canonicalLocation, organicResults2, localPack2) : void 0;
         let allOrganic2 = organicResults2;
         if ((options.pages ?? 1) >= 2) {
@@ -2206,6 +2307,7 @@ var PAAExtractor = class {
           diagnostics: {
             completionStatus: "serp_only",
             problem: null,
+            completeness: { ...this.completeness },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence2) } : {}
           },
           totalQuestions: 0,
@@ -2249,7 +2351,7 @@ var PAAExtractor = class {
             locationEvidence2 = inferSerpLocationEvidence(canonicalLocation, noPaaOrganic, localPack);
           }
         }
-        const aiSurfaces2 = await this.extractAISurfaces(page);
+        const aiSurfaces2 = await this.extractAISurfaces(page, options);
         const stats2 = {
           seed: options.query,
           totalQuestions: 0,
@@ -2265,6 +2367,7 @@ var PAAExtractor = class {
           diagnostics: {
             completionStatus: "no_paa",
             problem: null,
+            completeness: { ...this.completeness },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence2) } : {}
           },
           totalQuestions: 0,
@@ -2284,7 +2387,7 @@ var PAAExtractor = class {
       }
       const flat = await this.runBFS(page, options, signal);
       this.throwIfAborted(signal);
-      const aiSurfaces = await this.extractAISurfaces(page);
+      const aiSurfaces = await this.extractAISurfaces(page, options);
       const shortVidsParams = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", udm: ShortVideoSelectors.udm });
       if (uule) shortVidsParams.set("uule", uule);
       let shortVideos = [];
@@ -2331,6 +2434,7 @@ var PAAExtractor = class {
         diagnostics: {
           completionStatus: "paa_found",
           problem: null,
+          completeness: { ...this.completeness },
           ...diagnosticWarnings.length > 0 ? { warnings: diagnosticWarnings } : {},
           ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence) } : {}
         },