mcp-scraper 0.79.2 → 0.79.4

package/dist/{chunk-WEEAXPWK.js → chunk-KNGTQD6U.js}

@@ -8,7 +8,7 @@ import {
   currentCostContext,
   recordKernelSession,
   runWithCostContext
-} from "./chunk-NGLZ6UJR.js";
+} from "./chunk-5UAFBKJR.js";
 import {
   DEFAULT_PROXY_MODE,
   HarvestOptionsSchema,
@@ -24,7 +24,7 @@ import {
   RECAPTCHA_INSTRUCTIONS,
   RequestAbortedError,
   sanitizeVendorName
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 
 // src/driver/BrowserDriver.ts
 import { chromium } from "playwright-extra";
@@ -2263,17 +2263,22 @@ async function resolveGoogleOutboundLinks(page, values, options = {}) {
 // src/extractor/PAAExtractor.ts
 var PAA_ANSWER_PLACEHOLDER_PATTERN = String.raw`^(?:an error has occurred(?:[.!…]\s*)?(?:please try again later[.!…]?)?|loading|please wait|try again|something went wrong)[.!…\s]*$`;
 var PAA_CLICK_EARLIEST_CONFIRMATION_MS = 700;
+var PAA_CLICK_MID_CONFIRMATION_MS = 1e3;
 var PAA_CLICK_ATTEMPT_BUDGET_MS = 1400;
+var PAA_CLICK_DISPATCH_TIMEOUT_MS = 900;
 var PAA_FRONTIER_BATCH_SIZE = 3;
+var MAX_RETAINED_INTERACTION_OBSERVATIONS = 250;
 var PAAExtractor = class {
-  constructor(driver, reporter, onPaaProgress) {
+  constructor(driver, reporter, onPaaProgress, onProviderSession) {
     this.driver = driver;
     this.reporter = reporter;
     this.onPaaProgress = onPaaProgress;
+    this.onProviderSession = onProviderSession;
   }
   driver;
   reporter;
   onPaaProgress;
+  onProviderSession;
   completeness = {
     paaWithoutAnswer: 0,
     paaWithoutSource: 0,
@@ -2288,6 +2293,7 @@ var PAAExtractor = class {
   clickedQuestions = /* @__PURE__ */ new Set();
   paaLifecycle = this.emptyPaaLifecycle(0);
   checkpointMaterial = null;
+  interactionObservations = [];
   emptyPaaLifecycle(requestedQuestions) {
     return {
       requestedQuestions,
@@ -2300,7 +2306,12 @@ var PAAExtractor = class {
       knownQuestions: 0,
       processedQuestions: 0,
       failedInteractions: 0,
-      noGrowthConfirmations: 0
+      noGrowthConfirmations: 0,
+      visibleControls: 0,
+      eligibleControls: 0,
+      dispatchedInteractions: 0,
+      confirmedInteractions: 0,
+      unconfirmedInteractions: 0
     };
   }
   emptyLinkDiagnostics() {
@@ -2562,7 +2573,11 @@ var PAAExtractor = class {
       phase,
       records: this.collectedRows.map((row) => ({ ...row })),
       lifecycle: { ...this.paaLifecycle },
-      ...this.checkpointMaterial ? { material: structuredClone(this.checkpointMaterial) } : {}
+      ...this.checkpointMaterial ? { material: structuredClone(this.checkpointMaterial) } : {},
+      interactions: this.interactionObservations.map((observation) => ({
+        ...observation,
+        confirmationSignals: [...observation.confirmationSignals]
+      }))
     };
     try {
       await this.onPaaProgress(snapshot);
@@ -2643,6 +2658,7 @@ var PAAExtractor = class {
     const seenQs = /* @__PURE__ */ new Set();
     const orderedQs = [];
     this.clickedQuestions = /* @__PURE__ */ new Set();
+    this.interactionObservations = [];
     const abandonedNeverReclick = /* @__PURE__ */ new Set();
     const clickAttempts = /* @__PURE__ */ new Map();
     const processedQuestions = /* @__PURE__ */ new Set();
@@ -2650,6 +2666,7 @@ var PAAExtractor = class {
     const capturedItems = /* @__PURE__ */ new Map();
     const MAX_CLICK_ATTEMPTS = 2;
     const EXHAUSTION_CONFIRMATIONS = 3;
+    let batchNumber = 0;
     const itemArgs = {
       sel: PAASelectors.item,
       dataQ: PAASelectors.itemDataQ,
@@ -2659,12 +2676,30 @@ var PAAExtractor = class {
       expandedClass: PAASelectors.expandedClass,
       clickTarget: PAASelectors.clickTarget,
       answerPlaceholderPattern: PAA_ANSWER_PLACEHOLDER_PATTERN,
-      questionsOnly: options.questionsOnly
+      questionsOnly: options.questionsOnly,
+      sourceSel: "a[href]"
     };
     const readItemStates = () => page.evaluate(
       ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget, answerSel, answerPlaceholderPattern, questionsOnly }) => {
+        const cleanVisibleMaterial = (element) => {
+          if (!element) return "";
+          const parts = [];
+          for (const node of element.childNodes) {
+            if (node.nodeType === Node.TEXT_NODE) {
+              const text2 = node.textContent?.trim();
+              if (text2) parts.push(text2);
+              continue;
+            }
+            if (node.nodeType !== Node.ELEMENT_NODE) continue;
+            const child = node;
+            if (child.tagName === "STYLE" || child.tagName === "SCRIPT" || child.getAttribute("aria-hidden") === "true") continue;
+            const text = cleanVisibleMaterial(child);
+            if (text) parts.push(text);
+          }
+          return parts.join(" ").replace(/\s+/g, " ").trim();
+        };
         return Array.from(document.querySelectorAll(sel)).map((el, index) => {
-          const answerText = (el.querySelector(answerSel)?.textContent ?? "").replace(/\s+/g, " ").trim();
+          const answerText = cleanVisibleMaterial(el.querySelector(answerSel)).replace(/An AI Overview is not available for this search/gi, " ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi, " ").replace(/\bAI Overview\b/gi, " ").replace(/\s+/g, " ").trim();
           const expanded = el.classList.contains(expandedClass) || el.querySelector(clickTarget)?.getAttribute("aria-expanded") === "true";
           const answerReady = answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
           return {
@@ -2677,34 +2712,133 @@ var PAAExtractor = class {
       },
       itemArgs
     );
-    const expandOneItemSerially = async (q, beforeCount, itemIndex, wasExpanded) => {
+    const expandOneItemSerially = async (q, beforeCount, itemIndex, wasExpanded, controlAttempt, batchNumber2) => {
       const clickStartedMs = Date.now();
+      const baseObservation = {
+        id: `${batchNumber2}:${questionIdFor(q)}:${controlAttempt}`,
+        questionId: questionIdFor(q),
+        question: q,
+        batchNumber: batchNumber2,
+        controlAttempt,
+        observedAt: new Date(clickStartedMs).toISOString(),
+        beforeQuestionCount: beforeCount,
+        expandedBefore: wasExpanded
+      };
       const clickTarget = itemIndex >= 0 ? page.locator(itemArgs.sel).nth(itemIndex).locator(itemArgs.clickTarget) : null;
+      let dispatchErrorCode = null;
       const clicked = clickTarget ? await (async () => {
         if (wasExpanded) {
-          await clickTarget.click({ timeout: 2e3, force: true });
-          await page.waitForTimeout(250);
+          await clickTarget.click({ timeout: PAA_CLICK_DISPATCH_TIMEOUT_MS, force: true });
+          await page.waitForTimeout(120);
         }
-        await clickTarget.click({ timeout: 2e3, force: true });
+        await clickTarget.click({ timeout: PAA_CLICK_DISPATCH_TIMEOUT_MS, force: true });
         return true;
-      })().catch(() => false) : false;
-      if (!clicked) return { dispatched: false, confirmed: false };
-      const remainingConfirmationMs = Math.max(100, PAA_CLICK_ATTEMPT_BUDGET_MS - (Date.now() - clickStartedMs));
-      const confirmed = await page.waitForFunction(
-        ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, answerPlaceholderPattern, questionsOnly, target, min, earliestConfirmationAt }) => {
-          if (Date.now() < earliestConfirmationAt) return false;
-          const items = Array.from(document.querySelectorAll(sel));
-          const item = items.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
-          if (!item) return false;
-          const expanded = item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true";
-          if (questionsOnly) return expanded || items.length > min;
-          const answerText = (item.querySelector(answerSel)?.textContent ?? "").replace(/\s+/g, " ").trim();
-          return answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
-        },
-        { ...itemArgs, target: q, min: beforeCount, earliestConfirmationAt: clickStartedMs + PAA_CLICK_EARLIEST_CONFIRMATION_MS },
-        { timeout: remainingConfirmationMs, polling: 100 }
-      ).catch(() => null);
-      return { dispatched: true, confirmed: Boolean(confirmed) };
+      })().catch((error) => {
+        dispatchErrorCode = error instanceof Error && (error.name === "TimeoutError" || /timeout/i.test(error.message)) ? "click_ack_timeout" : "click_failed";
+        return false;
+      }) : false;
+      const dispatchDurationMs = Date.now() - clickStartedMs;
+      if (!clicked && dispatchErrorCode !== "click_ack_timeout") {
+        const observation2 = {
+          ...baseObservation,
+          dispatched: false,
+          outcome: "dispatch_failed",
+          confirmationSignals: [],
+          confirmationCheckpointMs: null,
+          confirmedAtMs: null,
+          dispatchDurationMs,
+          totalDurationMs: dispatchDurationMs,
+          afterQuestionCount: beforeCount,
+          newQuestionCount: 0,
+          expandedAfter: wasExpanded,
+          answerObserved: false,
+          sourceCount: 0,
+          errorCode: itemIndex < 0 ? "control_missing" : dispatchErrorCode ?? "click_failed"
+        };
+        return { dispatched: false, confirmed: false, observation: observation2 };
+      }
+      let latest = {
+        exists: true,
+        expanded: wasExpanded,
+        answerReady: false,
+        sourceCount: 0,
+        questionCount: beforeCount
+      };
+      let signals = [];
+      let confirmationCheckpointMs = null;
+      let confirmedAtMs = null;
+      let confirmed = false;
+      for (const checkpointMs of [PAA_CLICK_EARLIEST_CONFIRMATION_MS, PAA_CLICK_MID_CONFIRMATION_MS, PAA_CLICK_ATTEMPT_BUDGET_MS]) {
+        const remainingMs = clickStartedMs + checkpointMs - Date.now();
+        if (remainingMs > 0) await page.waitForTimeout(remainingMs);
+        latest = await page.evaluate(
+          ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, answerPlaceholderPattern, sourceSel, target }) => {
+            const cleanVisibleMaterial = (element) => {
+              if (!element) return "";
+              const parts = [];
+              for (const node of element.childNodes) {
+                if (node.nodeType === Node.TEXT_NODE) {
+                  const text2 = node.textContent?.trim();
+                  if (text2) parts.push(text2);
+                  continue;
+                }
+                if (node.nodeType !== Node.ELEMENT_NODE) continue;
+                const child = node;
+                if (child.tagName === "STYLE" || child.tagName === "SCRIPT" || child.getAttribute("aria-hidden") === "true") continue;
+                const text = cleanVisibleMaterial(child);
+                if (text) parts.push(text);
+              }
+              return parts.join(" ").replace(/\s+/g, " ").trim();
+            };
+            const items = Array.from(document.querySelectorAll(sel));
+            const item = items.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
+            if (!item) return { exists: false, expanded: false, answerReady: false, sourceCount: 0, questionCount: items.length };
+            const expanded = item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true";
+            const answerElement = item.querySelector(answerSel);
+            const answerText = cleanVisibleMaterial(answerElement);
+            const confirmationText = answerText.replace(/An AI Overview is not available for this search/gi, " ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi, " ").replace(/\bAI Overview\b/gi, " ").replace(/\s+/g, " ").trim();
+            const answerReady = confirmationText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(confirmationText);
+            return {
+              exists: true,
+              expanded,
+              answerReady,
+              sourceCount: answerReady ? answerElement?.querySelectorAll(sourceSel).length ?? 0 : 0,
+              questionCount: items.length
+            };
+          },
+          { ...itemArgs, target: q }
+        ).catch(() => latest);
+        signals = [
+          ...latest.expanded ? ["expanded"] : [],
+          ...latest.answerReady ? ["answer"] : [],
+          ...latest.sourceCount > 0 ? ["source"] : [],
+          ...latest.questionCount > beforeCount ? ["question_growth"] : []
+        ];
+        if (latest.answerReady || latest.sourceCount > 0 || latest.questionCount > beforeCount) {
+          confirmed = true;
+          confirmationCheckpointMs = checkpointMs;
+          confirmedAtMs = Date.now() - clickStartedMs;
+          break;
+        }
+      }
+      const totalDurationMs = Date.now() - clickStartedMs;
+      const observation = {
+        ...baseObservation,
+        dispatched: clicked,
+        outcome: confirmed ? "confirmed" : clicked ? "unconfirmed" : "dispatch_failed",
+        confirmationSignals: signals,
+        confirmationCheckpointMs,
+        confirmedAtMs,
+        dispatchDurationMs,
+        totalDurationMs,
+        afterQuestionCount: latest.questionCount,
+        newQuestionCount: Math.max(0, latest.questionCount - beforeCount),
+        expandedAfter: latest.expanded,
+        answerObserved: latest.answerReady,
+        sourceCount: latest.sourceCount,
+        errorCode: !clicked ? dispatchErrorCode : confirmed ? null : latest.exists ? "confirmation_timeout" : "control_missing"
+      };
+      return { dispatched: clicked, confirmed, observation };
     };
     let round = 0;
     expansion: while (true) {
@@ -2735,7 +2869,8 @@ var PAAExtractor = class {
         targetStatus: orderedQs.length >= options.maxQuestions ? "met" : "not_met",
         knownQuestions: orderedQs.length,
         processedQuestions: processedQuestions.size,
-        failedInteractions: failedInteractions.size
+        failedInteractions: failedInteractions.size,
+        visibleControls: states.length
       };
       if (discovered) await this.emitProgress("discovered");
       if (options.questionsOnly && seenQs.size >= options.maxQuestions) {
@@ -2783,6 +2918,7 @@ var PAAExtractor = class {
       const kept = new Set(orderedQs.slice(0, options.maxQuestions));
       const clickable = states.filter((state) => kept.has(state.q) && !state.done && (clickAttempts.get(state.q) ?? 0) < MAX_CLICK_ATTEMPTS && !abandonedNeverReclick.has(state.q));
       const targets = clickable.slice(-PAA_FRONTIER_BATCH_SIZE);
+      this.paaLifecycle = { ...this.paaLifecycle, eligibleControls: clickable.length };
       if (targets.length === 0) {
         if (seenQs.size >= options.maxQuestions) {
           this.paaLifecycle = {
@@ -2832,6 +2968,7 @@ var PAAExtractor = class {
         break;
       }
       await this.throwIfCaptcha(page, "Google PAA expansion");
+      batchNumber += 1;
       for (const target of targets) {
         this.reporter.onDepth(++round);
         if ((clickAttempts.get(target.q) ?? 0) > 0) {
@@ -2844,10 +2981,21 @@ var PAAExtractor = class {
       }
       const expansionResults = await Promise.all(targets.map(async (target) => ({
         target,
-        result: await expandOneItemSerially(target.q, states.length, target.index, target.expanded)
+        result: await expandOneItemSerially(
+          target.q,
+          states.length,
+          target.index,
+          target.expanded,
+          clickAttempts.get(target.q) ?? 1,
+          batchNumber
+        )
       })));
       for (const { target, result } of expansionResults) {
-        if (result.dispatched) this.clickedQuestions.add(target.q);
+        if (result.dispatched || result.confirmed) this.clickedQuestions.add(target.q);
+        this.interactionObservations.push(result.observation);
+      }
+      if (this.interactionObservations.length > MAX_RETAINED_INTERACTION_OBSERVATIONS) {
+        this.interactionObservations.splice(0, this.interactionObservations.length - MAX_RETAINED_INTERACTION_OBSERVATIONS);
       }
       if (!options.questionsOnly) {
         const targetQuestions = new Set(targets.map((target) => target.q));
@@ -2873,7 +3021,10 @@ var PAAExtractor = class {
         ...this.paaLifecycle,
         processedQuestions: processedQuestions.size,
         failedInteractions: failedInteractions.size,
-        noGrowthConfirmations: 0
+        noGrowthConfirmations: 0,
+        dispatchedInteractions: this.interactionObservations.filter((value) => value.dispatched).length,
+        confirmedInteractions: this.interactionObservations.filter((value) => value.outcome === "confirmed").length,
+        unconfirmedInteractions: this.interactionObservations.filter((value) => value.outcome !== "confirmed").length
       };
       await this.emitProgress("click_observed");
     }
@@ -3467,6 +3618,18 @@ var PAAExtractor = class {
     try {
       this.throwIfAborted(signal);
       await this.driver.launch(config);
+      const providerSession = this.driver.getProviderSessionMetadata?.();
+      if (providerSession?.providerSessionId && this.onProviderSession) {
+        try {
+          await this.onProviderSession(providerSession);
+        } catch (err) {
+          console.warn(JSON.stringify({
+            event: "paa_provider_session_sink_failed",
+            provider: providerSession.provider,
+            message: err instanceof Error ? err.message : String(err)
+          }));
+        }
+      }
       this.throwIfAborted(signal);
       const locationResolution = options.location ? resolveLocation(options.location) : null;
       const canonicalLocation = options.location ? locationResolution?.canonical ?? options.location : null;
@@ -4602,10 +4765,10 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId) {
     }));
   }
 }
-async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress) {
+async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress, onProviderSession) {
   const driver = createSerpDriver(options, { forceManaged: forceManagedSerp });
   const reporter = new ProgressReporter();
-  const extractor = new PAAExtractor(driver, reporter, onPaaProgress);
+  const extractor = new PAAExtractor(driver, reporter, onPaaProgress, onProviderSession);
   if (signal?.aborted) {
     return {
       result: null,
@@ -4750,12 +4913,20 @@ async function harvest(rawOptions) {
         max_questions: attemptOptions.maxQuestions
       }));
       const baseCtx = currentCostContext();
+      const onProviderSession = (metadata) => emitAttemptEvent(onAttemptEvent, {
+        type: "provider_session_started",
+        attemptNumber,
+        maxAttempts,
+        provider: metadata.provider,
+        providerSessionId: metadata.providerSessionId,
+        observedAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
       const allowHeadlessTest = baseCtx?.harvestBrowserMode === "headless-test" && process.env[HEADLESS_HARVEST_TEST_ENV] === "true";
       const attempt = baseCtx ? await runWithCostContext({
         ...baseCtx,
         forceHeadless: allowHeadlessTest,
         forceHeadful: !allowHeadlessTest
-      }, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress);
+      }, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress, onProviderSession)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress, onProviderSession);
       if (attempt.error) {
         const err = attempt.error;
         const outcome2 = classifyAttemptError(err);

package/dist/{chunk-2RTELL6M.js → chunk-N63ZFFU2.js}

@@ -1,5 +1,5 @@
 // src/version.ts
-var PACKAGE_VERSION = "0.79.2";
+var PACKAGE_VERSION = "0.79.4";
 
 export {
   PACKAGE_VERSION

package/dist/{chunk-47ZCUBM5.js → chunk-NVUKO5NN.js}

@@ -6,11 +6,12 @@ import {
   LocationMismatchError,
   PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
   RequestAbortedError,
+  attachHarvestAttemptProviderSession,
   buildPublicErrorEnvelope,
   finishHarvestAttempt,
   publicErrorMessage,
   startHarvestAttempt
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 
 // src/api/harvest-problems.ts
 function errorMessage(err) {
@@ -186,6 +187,16 @@ function createHarvestAttemptRecorder(jobId, userId) {
       });
       return;
     }
+    if (event.type === "provider_session_started") {
+      await attachHarvestAttemptProviderSession({
+        jobId,
+        attemptNumber: event.attemptNumber,
+        browserProvider: event.provider,
+        providerSessionId: event.providerSessionId,
+        observedAt: event.observedAt
+      });
+      return;
+    }
     await finishHarvestAttempt({
       jobId,
       attemptNumber: event.attemptNumber,

package/dist/{chunk-XUM2LEGU.js → chunk-PQVLX5WT.js}

@@ -12,13 +12,13 @@ import {
   registerScheduledResultsMcpTools,
   registerSerpIntelligenceCaptureTools,
   resolveDeploymentProfile
-} from "./chunk-JECPQBLZ.js";
+} from "./chunk-RUBI3OXR.js";
 import {
   renderInstallTerminal
-} from "./chunk-6FL7ZZCF.js";
+} from "./chunk-ETQ7J2EB.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-2RTELL6M.js";
+} from "./chunk-N63ZFFU2.js";
 
 // src/mcp/stdio-runtime.ts
 import { readFileSync } from "fs";