mcp-scraper 0.5.0 → 0.7.0

package/dist/index.cjs

@@ -1688,13 +1688,41 @@ var PAAExtractor = class {
         }
         return parts.join(" ").replace(/\s+/g, " ").trim();
       }
-      return Array.from(document.querySelectorAll(selectors.item)).map((pair) => ({
-        question: pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "",
-        answer: cleanText(pair.querySelector(selectors.answerContainer)) || void 0,
-        sourceTitle: pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0,
-        sourceSite: pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0,
-        sourceCite: pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0
-      }));
+      return Array.from(document.querySelectorAll(selectors.item)).map((pair) => {
+        let sourceTitle = pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0;
+        let sourceSite = pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0;
+        let sourceCite = pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0;
+        if (!sourceCite) {
+          const seenHrefs = /* @__PURE__ */ new Set();
+          const anchors = [];
+          for (const a of Array.from(pair.querySelectorAll("a[href]"))) {
+            const href = a.href;
+            if (!/^https?:\/\//.test(href) || seenHrefs.has(href)) continue;
+            let host = "";
+            try {
+              host = new URL(href).hostname;
+            } catch {
+              continue;
+            }
+            if (/(^|\.)google(usercontent)?\.[a-z.]+$/.test(host)) continue;
+            seenHrefs.add(href);
+            anchors.push({ text: (a.textContent || "").trim(), href, host });
+          }
+          const primary = anchors.find((x) => x.text.length > 2) ?? anchors[0];
+          if (primary) {
+            sourceTitle = sourceTitle || primary.text || void 0;
+            sourceSite = sourceSite || primary.host.replace(/^www\./, "");
+            sourceCite = primary.href;
+          }
+        }
+        return {
+          question: pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "",
+          answer: cleanText(pair.querySelector(selectors.answerContainer)) || void 0,
+          sourceTitle,
+          sourceSite,
+          sourceCite
+        };
+      });
     }, sels);
     return raw.flatMap((item) => {
       const cleaned = {
@@ -1736,54 +1764,115 @@ var PAAExtractor = class {
     const seenQs = /* @__PURE__ */ new Set();
     const orderedQs = [];
     const results = [];
-    const readAllQs = () => page.evaluate(
-      ({ sel, dataQ, dataInitQ, questionEl }) => Array.from(document.querySelectorAll(sel)).map(
-        (el) => el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || ""
-      ).filter(Boolean),
-      { sel: PAASelectors.item, dataQ: PAASelectors.itemDataQ, dataInitQ: PAASelectors.itemDataInitQ, questionEl: PAASelectors.itemQuestionEl }
+    const abandonedNeverReclick = /* @__PURE__ */ new Set();
+    const clickedOnceEver = /* @__PURE__ */ new Set();
+    const itemArgs = {
+      sel: PAASelectors.item,
+      dataQ: PAASelectors.itemDataQ,
+      dataInitQ: PAASelectors.itemDataInitQ,
+      questionEl: PAASelectors.itemQuestionEl,
+      answerSel: PAASelectors.answerContainer,
+      expandedClass: PAASelectors.expandedClass,
+      clickTarget: PAASelectors.clickTarget
+    };
+    const readItemStates = () => page.evaluate(
+      ({ sel, dataQ, dataInitQ, questionEl, expandedClass, answerSel }) => {
+        const answerText = (root) => {
+          if (!root) return "";
+          let out = "";
+          const walk = (n) => {
+            for (const c of Array.from(n.childNodes)) {
+              if (c.nodeType === Node.TEXT_NODE) out += c.textContent || "";
+              else if (c.nodeType === Node.ELEMENT_NODE && c.tagName !== "STYLE" && c.tagName !== "SCRIPT") walk(c);
+            }
+          };
+          walk(root);
+          return out.replace(/\s+/g, " ").trim();
+        };
+        return Array.from(document.querySelectorAll(sel)).map((el) => ({
+          q: el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || "",
+          done: el.classList.contains(expandedClass) || answerText(el.querySelector(answerSel)).length > 30
+        })).filter((s) => s.q);
+      },
+      itemArgs
     );
+    const expandOneItemSerially = async (q) => {
+      const clicked = await page.evaluate(
+        ({ sel, dataQ, dataInitQ, clickTarget, q: target }) => {
+          const item = Array.from(document.querySelectorAll(sel)).find((el) => (el.getAttribute(dataQ) || el.getAttribute(dataInitQ)) === target);
+          const btn = item?.querySelector(clickTarget);
+          if (!btn) return false;
+          btn.scrollIntoView({ block: "center" });
+          btn.click();
+          return true;
+        },
+        { ...itemArgs, q }
+      ).catch(() => false);
+      if (!clicked) return "failed";
+      try {
+        const outcome = await page.waitForFunction(
+          ({ sel, dataQ, dataInitQ, answerSel, q: target }) => {
+            const answerText = (root) => {
+              if (!root) return "";
+              let out = "";
+              const walk = (n) => {
+                for (const c of Array.from(n.childNodes)) {
+                  if (c.nodeType === Node.TEXT_NODE) out += c.textContent || "";
+                  else if (c.nodeType === Node.ELEMENT_NODE && c.tagName !== "STYLE" && c.tagName !== "SCRIPT") walk(c);
+                }
+              };
+              walk(root);
+              return out.replace(/\s+/g, " ").trim();
+            };
+            const item = Array.from(document.querySelectorAll(sel)).find((el) => (el.getAttribute(dataQ) || el.getAttribute(dataInitQ)) === target);
+            if (!item) return "gone";
+            const text = answerText(item.querySelector(answerSel));
+            if (/an error has occurred/i.test(text.slice(0, 80))) return "error";
+            if (text.length > 30) return "ok";
+            return false;
+          },
+          { ...itemArgs, q },
+          { timeout: 6500, polling: 200 }
+        );
+        return await outcome.jsonValue() === "ok" ? "ok" : "failed";
+      } catch {
+        return "failed";
+      }
+    };
     let round = 0;
-    while (seenQs.size < options.maxQuestions) {
+    let growthWaits = 0;
+    while (true) {
       if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
       this.throwIfAborted(signal);
-      await this.throwIfCaptcha(page, "Google PAA expansion");
-      const beforeQs = await readAllQs();
-      if (beforeQs.length >= options.maxQuestions) break;
-      const unexpandedSel = `${PAASelectors.item}:not(.${PAASelectors.expandedClass}) ${PAASelectors.clickTarget}`;
-      const unexpandedCount = await page.locator(unexpandedSel).count();
-      if (unexpandedCount === 0) break;
-      this.reporter.onDepth(++round);
-      for (let ci = 0; ci < unexpandedCount; ci++) {
-        if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
-        this.throwIfAborted(signal);
-        try {
-          const btn = page.locator(unexpandedSel).first();
-          await btn.scrollIntoViewIfNeeded();
-          await btn.hover({ force: true });
-          await page.waitForTimeout(100);
-          await btn.click({ force: true });
-          await page.waitForTimeout(500);
-        } catch {
+      const states = await readItemStates();
+      for (const s of states) {
+        if (!seenQs.has(s.q)) {
+          seenQs.add(s.q);
+          orderedQs.push(s.q);
         }
       }
-      await page.waitForFunction(
-        ({ sel, min }) => document.querySelectorAll(sel).length > min,
-        { sel: PAASelectors.item, min: beforeQs.length },
-        { timeout: 5e3 }
-      ).catch(() => {
-      });
-      await this.throwIfCaptcha(page, "Google PAA expansion");
-      const afterQs = await readAllQs();
-      if (afterQs.length === beforeQs.length) break;
-      for (const q of afterQs) {
-        if (!seenQs.has(q)) {
-          seenQs.add(q);
-          orderedQs.push(q);
-        }
+      const kept = new Set(orderedQs.slice(0, options.maxQuestions));
+      const clickable = states.filter((s) => kept.has(s.q) && !clickedOnceEver.has(s.q) && !abandonedNeverReclick.has(s.q));
+      const target = clickable.find((s) => !s.done) ?? (seenQs.size < options.maxQuestions ? clickable[0] : void 0);
+      if (!target) {
+        if (seenQs.size >= options.maxQuestions) break;
+        if (++growthWaits > 1) break;
+        const grew = await page.waitForFunction(
+          ({ sel, min }) => document.querySelectorAll(sel).length > min,
+          { sel: PAASelectors.item, min: states.length },
+          { timeout: 3e3 }
+        ).catch(() => null);
+        if (!grew) break;
+        continue;
       }
+      growthWaits = 0;
+      this.reporter.onDepth(++round);
+      await this.throwIfCaptcha(page, "Google PAA expansion");
+      clickedOnceEver.add(target.q);
+      if (await expandOneItemSerially(target.q) === "failed") abandonedNeverReclick.add(target.q);
     }
     const itemMap = new Map((await this.extractVisibleItems(page)).map((i) => [i.question, i]));
-    await this.fillIncompleteItems(page, orderedQs, itemMap, options);
+    await this.fillIncompleteItems(page, orderedQs, itemMap, options, abandonedNeverReclick);
     for (const q of orderedQs) {
       if (results.length >= options.maxQuestions) break;
       const key = this.normalizeQuestion(q);
@@ -1801,9 +1890,11 @@ var PAAExtractor = class {
     this.completeness.paaWithoutSource = results.filter((r) => !r.source_title && !r.source_site && !r.source_cite).length;
     return results;
   }
-  async fillIncompleteItems(page, orderedQs, itemMap, options) {
+  async fillIncompleteItems(page, orderedQs, itemMap, options, abandonedNeverReclick) {
     const kept = new Set(orderedQs.slice(0, options.maxQuestions));
-    const missingBefore = new Set([...kept].filter((q) => !itemMap.get(q)?.answer));
+    const missingBefore = new Set(
+      [...kept].filter((q) => !itemMap.get(q)?.answer && !abandonedNeverReclick?.has(q))
+    );
     if (missingBefore.size === 0) return;
     for (let round = 0; round < 2; round++) {
       if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
@@ -2195,40 +2286,41 @@ var PAAExtractor = class {
     };
   }
   async captureAIOverviewShareUrl(page, options) {
-    for (let attempt = 0; attempt < 3; attempt++) {
-      if (options?.softDeadlineMs && Date.now() >= options.softDeadlineMs) return null;
-      const clicked = await page.evaluate(() => {
-        const heads = Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"],div')).filter((h) => (h.textContent || "").trim() === "AI Overview");
-        for (const head of heads) {
-          let sec = head;
-          for (let up = 0; up < 8 && sec; up++) {
-            const btn = sec.querySelector('[aria-label="Share"]');
-            if (btn) {
-              btn.scrollIntoView({ block: "center" });
-              btn.click();
-              return true;
-            }
-            sec = sec.parentElement;
+    if (options?.softDeadlineMs && Date.now() >= options.softDeadlineMs) return null;
+    const clicked = await page.evaluate(() => {
+      const heads = Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"],div')).filter((h) => (h.textContent || "").trim() === "AI Overview");
+      for (const head of heads) {
+        let sec = head;
+        for (let up = 0; up < 8 && sec; up++) {
+          const btn = sec.querySelector('[aria-label="Share"]');
+          if (btn) {
+            btn.scrollIntoView({ block: "center" });
+            btn.click();
+            return true;
           }
+          sec = sec.parentElement;
         }
-        return false;
-      }).catch(() => false);
-      if (!clicked) return null;
-      await page.waitForTimeout(4500);
-      const url = await page.evaluate(() => {
+      }
+      return false;
+    }).catch(() => false);
+    if (!clicked) return null;
+    let url = null;
+    try {
+      const found = await page.waitForFunction(() => {
         const w = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
         while (w.nextNode()) {
           const m = (w.currentNode.textContent || "").match(/https:\/\/share\.google\/aimode\/\S+/);
           if (m) return m[0];
         }
-        return null;
-      }).catch(() => null);
-      await page.keyboard.press("Escape").catch(() => {
-      });
-      if (url) return url;
-      await page.waitForTimeout(600);
+        return false;
+      }, void 0, { timeout: 4500, polling: 250 });
+      url = await found.jsonValue();
+    } catch {
+      url = null;
     }
-    return null;
+    await page.keyboard.press("Escape").catch(() => {
+    });
+    return url;
   }
   buildTree(flat, _seed) {
     const roots = [];