mcp-scraper 0.40.2 → 0.41.0

package/dist/{db-W3CP562I.js → db-LVVU6NOK.js}

@@ -8,6 +8,8 @@ import {
   appendWorkflowArtifacts,
   cancelJob,
   checkRateLimit,
+  checkpointJob,
+  claimInngestPaaJob,
   claimMonthlyFreeRefresh,
   claimPendingJob,
   claimPendingKpoJob,
@@ -21,6 +23,7 @@ import {
   countActiveJobsForUser,
   countActiveUsers,
   createJob,
+  createJobWithId,
   createKpoJob,
   createPasswordResetToken,
   createRunningJob,
@@ -74,7 +77,10 @@ import {
   listInboxMessages,
   listJobs,
   listKpoJobs,
+  listPendingInngestPaaJobs,
+  listRecentTerminalInngestPaaJobs,
   listRequestEvents,
+  listStaleRunningInngestPaaJobs,
   listUsers,
   listUsersDueForConnectedAccountBillingReconciliation,
   listWorkflowArtifacts,
@@ -83,6 +89,7 @@ import {
   logKpoPhaseComplete,
   logRequestEvent,
   markInboxMessageOpened,
+  markJobBillingSettled,
   markWorkflowRunRunning,
   markWorkflowScheduleRan,
   migrate,
@@ -116,7 +123,7 @@ import {
   updateKpoJobState,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-GUVKHCKE.js";
+} from "./chunk-IQTT7CAB.js";
 export {
   CORE_SCHEMA_VERSION,
   CREDIT_LOTS_EPOCH,
@@ -127,6 +134,8 @@ export {
   appendWorkflowArtifacts,
   cancelJob,
   checkRateLimit,
+  checkpointJob,
+  claimInngestPaaJob,
   claimMonthlyFreeRefresh,
   claimPendingJob,
   claimPendingKpoJob,
@@ -140,6 +149,7 @@ export {
   countActiveJobsForUser,
   countActiveUsers,
   createJob,
+  createJobWithId,
   createKpoJob,
   createPasswordResetToken,
   createRunningJob,
@@ -193,7 +203,10 @@ export {
   listInboxMessages,
   listJobs,
   listKpoJobs,
+  listPendingInngestPaaJobs,
+  listRecentTerminalInngestPaaJobs,
   listRequestEvents,
+  listStaleRunningInngestPaaJobs,
   listUsers,
   listUsersDueForConnectedAccountBillingReconciliation,
   listWorkflowArtifacts,
@@ -202,6 +215,7 @@ export {
   logKpoPhaseComplete,
   logRequestEvent,
   markInboxMessageOpened,
+  markJobBillingSettled,
   markWorkflowRunRunning,
   markWorkflowScheduleRan,
   migrate,
@@ -236,4 +250,4 @@ export {
   upsertPageSnapshot,
   verifyPassword
 };
-//# sourceMappingURL=db-W3CP562I.js.map
+//# sourceMappingURL=db-LVVU6NOK.js.map

package/dist/{extract-bundle-K4PG3RZJ.js → extract-bundle-SKXEDG4Z.js}

@@ -10,12 +10,12 @@ import {
 } from "./chunk-345BQXZH.js";
 import {
   extractJobLimitInfo
-} from "./chunk-X2LKCX6H.js";
+} from "./chunk-6J57U6HA.js";
 import "./chunk-SBLGBZZB.js";
-import "./chunk-44HZLHDV.js";
+import "./chunk-5UN33CGU.js";
 import {
   getDb
-} from "./chunk-GUVKHCKE.js";
+} from "./chunk-IQTT7CAB.js";
 
 // src/api/extract-bundle.ts
 import { createWriteStream, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
@@ -565,4 +565,4 @@ export {
   SITE_EXTRACT_PAGE_CHUNK,
   assembleExtractArtifacts
 };
-//# sourceMappingURL=extract-bundle-K4PG3RZJ.js.map
+//# sourceMappingURL=extract-bundle-SKXEDG4Z.js.map

package/dist/index.cjs

@@ -181,6 +181,9 @@ var PAASelectors = {
   itemDataInitQ: "data-initq",
   itemQuestionEl: ".JlqpRe",
   answerContainer: ".bCOlv, .hgKElc, .wDYxhc, .LGOjhe, .fo7IQd, .fmW3u",
+  aiAnswerBody: ".pWvJNd",
+  aiCitation: "a.PMDqCb[href]",
+  aiSourceCard: "a.vIWmYe[href], a.Zbfntb[href]",
   sourceTitle: "h3",
   sourceSite: ".VuuXrf",
   sourceCite: "cite",
@@ -1773,14 +1776,28 @@ var PAAExtractor = class {
         }
         return parts.join(" ").replace(/\s+/g, " ").trim();
       }
+      function cleanSourceLabel(value) {
+        return (value ?? "").replace(/\s*\(\+\d+\)\s*-\s*View related links.*$/i, "").replace(/\s*-\s*Opens in new tab.*$/i, "").trim();
+      }
       return Array.from(document.querySelectorAll(selectors.item)).map((pair) => {
-        let sourceTitle = pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0;
-        let sourceSite = pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0;
-        let sourceCite = pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0;
-        if (!sourceCite) {
+        const clickTarget = pair.querySelector(selectors.clickTarget);
+        const expanded = pair.classList.contains(selectors.expandedClass) || clickTarget?.getAttribute("aria-expanded") === "true";
+        const answerContainer = expanded ? pair.querySelector(selectors.answerContainer) : null;
+        const aiAnswerBody = answerContainer?.querySelector(selectors.aiAnswerBody);
+        const answerElement = aiAnswerBody ?? answerContainer;
+        let sourceTitle = expanded ? pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0 : void 0;
+        let sourceSite = expanded ? pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0 : void 0;
+        let sourceCite = expanded ? pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0 : void 0;
+        if (expanded && !sourceCite) {
           const seenHrefs = /* @__PURE__ */ new Set();
           const anchors = [];
-          for (const a of Array.from(pair.querySelectorAll("a[href]"))) {
+          const anchorRoot = answerElement ?? pair;
+          const prioritized = [
+            ...Array.from(anchorRoot.querySelectorAll(selectors.aiCitation)),
+            ...Array.from(anchorRoot.querySelectorAll(selectors.aiSourceCard)),
+            ...Array.from(anchorRoot.querySelectorAll("a[href]"))
+          ];
+          for (const a of prioritized) {
             const href = a.href;
             if (!/^https?:\/\//.test(href) || seenHrefs.has(href)) continue;
             let host = "";
@@ -1791,18 +1808,23 @@ var PAAExtractor = class {
             }
             if (/(^|\.)google(usercontent)?\.[a-z.]+$/.test(host)) continue;
             seenHrefs.add(href);
-            anchors.push({ text: (a.textContent || "").trim(), href, host });
+            anchors.push({
+              text: (a.textContent || "").trim(),
+              label: cleanSourceLabel(a.getAttribute("aria-label")),
+              href,
+              host
+            });
           }
-          const primary = anchors.find((x) => x.text.length > 2) ?? anchors[0];
+          const primary = anchors.find((x) => x.label.length > 2 || x.text.length > 2) ?? anchors[0];
           if (primary) {
-            sourceTitle = sourceTitle || primary.text || void 0;
+            sourceTitle = sourceTitle || primary.label || primary.text || void 0;
             sourceSite = sourceSite || primary.host.replace(/^www\./, "");
             sourceCite = primary.href;
           }
         }
         return {
           question: pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "",
-          answer: cleanText(pair.querySelector(selectors.answerContainer)) || void 0,
+          answer: cleanText(answerElement) || void 0,
           sourceTitle,
           sourceSite,
           sourceCite