mcp-scraper 0.69.0 → 0.72.0

package/dist/bin/paa-harvest.cjs

@@ -430,6 +430,7 @@ var NANGO_USD_PER_CONNECTION_MONTH = envRate("NANGO_USD_PER_CONNECTION_MONTH", 1
 var NANGO_USD_PER_FUNCTION_RUN = envRate("NANGO_USD_PER_FUNCTION_RUN", 1e-4);
 var NANGO_USD_PER_PROXY_REQUEST = envRate("NANGO_USD_PER_PROXY_REQUEST", 1e-4);
 var NANGO_USD_PER_COMPUTE_SEC = envRate("NANGO_USD_PER_COMPUTE_SEC", 2e-4);
+var BRIGHTDATA_BROWSER_USD_PER_GB = envRate("BRIGHTDATA_BROWSER_USD_PER_GB", 0);
 function kernelCostUsd(ms, headful) {
   const sec = Math.max(0, ms) / 1e3;
   return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
@@ -452,7 +453,8 @@ async function runCostTelemetryMigration() {
     SELECT
       (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
       AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
-      AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name = 'method') = 1
+      AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key')) = 2
+      AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
       AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
       AS ready
   `);
@@ -508,6 +510,7 @@ async function runCostTelemetryMigration() {
       unit_type    TEXT,
       est_cost_usd REAL NOT NULL DEFAULT 0,
       error        TEXT,
+      source_key   TEXT,
       created_at   TEXT NOT NULL DEFAULT (datetime('now'))
     )
   `);
@@ -518,6 +521,11 @@ async function runCostTelemetryMigration() {
     await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN method TEXT`);
   } catch {
   }
+  try {
+    await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN source_key TEXT`);
+  } catch {
+  }
+  await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS vendor_usage_log_vendor_source_key ON vendor_usage_log(vendor, source_key) WHERE source_key IS NOT NULL`);
   await db.execute(`
     CREATE TABLE IF NOT EXISTS cost_probe_runs (
       id                            TEXT PRIMARY KEY,
@@ -1122,6 +1130,13 @@ var BrowserDriver = class {
   getKernelSessionId() {
     return this.kernelSessionId;
   }
+  getProviderSessionMetadata() {
+    return {
+      provider: this.kernelSessionId ? "kernel" : "local",
+      providerSessionId: this.kernelSessionId,
+      disconnectObservation: null
+    };
+  }
   getDebugSnapshot() {
     return this.debugSnapshot;
   }
@@ -1136,6 +1151,7 @@ var BrowserDriver = class {
       const proxySource = this.kernelProxySource;
       const proxyType = this.kernelProxyType;
       const headlessSent = this.kernelHeadlessSent;
+      const provider = this.getProviderSessionMetadata();
       this.browser = null;
       this.context = null;
       this.page = null;
@@ -1170,7 +1186,11 @@ var BrowserDriver = class {
           kernelDeleteSucceeded: deleteResult.status === "fulfilled",
           kernelDeleteError: deleteResult.status === "rejected" ? deleteResult.reason instanceof Error ? deleteResult.reason.message : String(deleteResult.reason) : null,
           browserCloseSucceeded: closeResult.status === "fulfilled",
-          browserCloseError: closeResult.status === "rejected" ? closeResult.reason instanceof Error ? closeResult.reason.message : String(closeResult.reason) : null
+          browserCloseError: closeResult.status === "rejected" ? closeResult.reason instanceof Error ? closeResult.reason.message : String(closeResult.reason) : null,
+          provider: provider.provider,
+          providerSessionId: provider.providerSessionId,
+          providerDisconnectObserved: false,
+          providerDisconnectMessage: null
         };
         if (deleteResult.status === "rejected") {
           console.warn(JSON.stringify({
@@ -1202,7 +1222,11 @@ var BrowserDriver = class {
         kernelDeleteSucceeded: null,
         kernelDeleteError: null,
         browserCloseSucceeded: true,
-        browserCloseError: null
+        browserCloseError: null,
+        provider: provider.provider,
+        providerSessionId: provider.providerSessionId,
+        providerDisconnectObserved: false,
+        providerDisconnectMessage: null
       };
     } else if (this.context) {
       const ctx = this.context;
@@ -1215,7 +1239,11 @@ var BrowserDriver = class {
         kernelDeleteSucceeded: null,
         kernelDeleteError: null,
         browserCloseSucceeded: true,
-        browserCloseError: null
+        browserCloseError: null,
+        provider: "local",
+        providerSessionId: null,
+        providerDisconnectObserved: false,
+        providerDisconnectMessage: null
       };
     }
     return {
@@ -1224,13 +1252,64 @@ var BrowserDriver = class {
       kernelDeleteSucceeded: null,
       kernelDeleteError: null,
       browserCloseSucceeded: null,
-      browserCloseError: null
+      browserCloseError: null,
+      provider: "local",
+      providerSessionId: null,
+      providerDisconnectObserved: false,
+      providerDisconnectMessage: null
     };
   }
 };
 
 // src/driver/BrightDataSerpDriver.ts
 var import_playwright2 = require("playwright");
+
+// src/driver/browser-provider-telemetry.ts
+var MAX_PROVIDER_SESSION_ID_LENGTH = 512;
+var MAX_PROVIDER_MESSAGE_LENGTH = 320;
+function boundedText(value, maxLength) {
+  if (typeof value !== "string") return null;
+  const normalized = value.replace(/[\u0000-\u001f\u007f]+/g, " ").replace(/\s+/g, " ").trim();
+  if (!normalized) return null;
+  return normalized.slice(0, maxLength);
+}
+function boundedProviderMessage(value) {
+  const message = boundedText(value, MAX_PROVIDER_MESSAGE_LENGTH);
+  if (!message) return null;
+  return message.replace(/\b(?:wss?|https?):\/\/\S+/gi, "[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization):\s*\S+/gi, "$1: [redacted]").replace(/\bBearer\s+\S+/gi, "Bearer [redacted]").slice(0, MAX_PROVIDER_MESSAGE_LENGTH);
+}
+function sanitizeProviderLocalMessage(value) {
+  return boundedProviderMessage(value);
+}
+function validateProviderSessionId(value) {
+  if (typeof value !== "string" || value.length === 0 || value.length > MAX_PROVIDER_SESSION_ID_LENGTH || value.trim() !== value || /[\s\u0000-\u001f\u007f]/.test(value) || /:\/\//.test(value)) {
+    throw new Error("Bright Data did not return a valid browser session ID");
+  }
+  return value;
+}
+function makeProviderInterruptionObservation(source, message, observedAt = (/* @__PURE__ */ new Date()).toISOString()) {
+  const safeMessage = boundedProviderMessage(message);
+  return {
+    source,
+    message: safeMessage ?? source.replaceAll("_", " "),
+    observedAt
+  };
+}
+function projectProviderSessionPublic(metadata) {
+  return {
+    provider: metadata.provider,
+    providerDisconnectObserved: metadata.disconnectObservation !== null,
+    providerDisconnectMessage: metadata.disconnectObservation?.message ?? null
+  };
+}
+function projectProviderSessionPrivate(metadata) {
+  return {
+    ...projectProviderSessionPublic(metadata),
+    providerSessionId: metadata.providerSessionId
+  };
+}
+
+// src/driver/BrightDataSerpDriver.ts
 var BLOCKED_TYPES = /* @__PURE__ */ new Set(["image", "media", "font", "stylesheet", "ping"]);
 var BLOCKED_HOST_PARTS = [
   "doubleclick.net",
@@ -1280,8 +1359,19 @@ var BrightDataSerpDriver = class {
   browser = null;
   context = null;
   page = null;
+  cdpSession = null;
+  providerSessionId = null;
+  disconnectObservation = null;
+  closing = false;
+  observeDisconnect(source, message) {
+    if (this.closing || this.disconnectObservation) return;
+    this.disconnectObservation = makeProviderInterruptionObservation(source, message);
+  }
   async launch(_config) {
     if (this.page) return;
+    this.closing = false;
+    this.providerSessionId = null;
+    this.disconnectObservation = null;
     const endpoint = brightDataBrowserEndpoint();
     if (!endpoint) throw new Error("SERP browser service is not configured");
     this.browser = await import_playwright2.chromium.connectOverCDP(endpoint);
@@ -1297,6 +1387,14 @@ var BrightDataSerpDriver = class {
       }
     });
     this.page = this.context.pages()[0] ?? await this.context.newPage();
+    this.browser.on("disconnected", () => this.observeDisconnect("browser_disconnected"));
+    this.context.on("close", () => this.observeDisconnect("context_closed"));
+    this.page.on("close", () => this.observeDisconnect("page_closed"));
+    this.page.on("crash", () => this.observeDisconnect("page_crashed"));
+    this.cdpSession = await this.context.newCDPSession(this.page);
+    const customCdp = this.cdpSession;
+    const sessionResponse = await customCdp.send("Browser.getSessionId");
+    this.providerSessionId = validateProviderSessionId(sessionResponse?.sessionId);
     await this.page.route("**/*", async (route) => {
       const request = route.request();
       const resourceType = request.resourceType();
@@ -1352,23 +1450,38 @@ var BrightDataSerpDriver = class {
   getKernelSessionId() {
     return null;
   }
+  getProviderSessionMetadata() {
+    return {
+      provider: "bright_data_browser_api",
+      providerSessionId: this.providerSessionId,
+      disconnectObservation: this.disconnectObservation
+    };
+  }
   getDebugSnapshot() {
     return { kernel: null, context: null, networkLocation: null, serpNavigation: null };
   }
   async close() {
+    this.closing = true;
     let error = null;
     try {
       await this.browser?.close();
     } catch (caught) {
-      error = caught instanceof Error ? caught.message : String(caught);
+      error = sanitizeProviderLocalMessage(caught instanceof Error ? caught.message : String(caught)) ?? "Bright Data browser close failed";
+      this.disconnectObservation ??= makeProviderInterruptionObservation("browser_close_error", error);
     }
+    const provider = projectProviderSessionPrivate(this.getProviderSessionMetadata());
+    this.browser = null;
+    this.context = null;
+    this.page = null;
+    this.cdpSession = null;
     return {
       kernelSessionId: null,
       kernelDeleteStarted: false,
       kernelDeleteSucceeded: null,
       kernelDeleteError: null,
       browserCloseSucceeded: error === null,
-      browserCloseError: error
+      browserCloseError: error,
+      ...provider
     };
   }
 };
@@ -2264,12 +2377,14 @@ async function extractAISurfacesFromDocument(config) {
 
 // src/extractor/PAAExtractor.ts
 var PAAExtractor = class {
-  constructor(driver, reporter) {
+  constructor(driver, reporter, onPaaProgress) {
     this.driver = driver;
     this.reporter = reporter;
+    this.onPaaProgress = onPaaProgress;
   }
   driver;
   reporter;
+  onPaaProgress;
   completeness = {
     paaWithoutAnswer: 0,
     paaWithoutSource: 0,
@@ -2279,6 +2394,8 @@ var PAAExtractor = class {
   aioShareUrlEarly = null;
   collectedRows = [];
   partialContext = null;
+  progressSequence = 0;
+  clickedQuestions = /* @__PURE__ */ new Set();
   normalizeQuestion(q) {
     return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
   }
@@ -2315,6 +2432,34 @@ var PAAExtractor = class {
       function cleanSourceLabel(value) {
         return (value ?? "").replace(/\s*\(\+\d+\)\s*-\s*View related links.*$/i, "").replace(/\s*-\s*Opens in new tab.*$/i, "").trim();
       }
+      function isGoogleHost(host) {
+        const normalized = host.toLowerCase().replace(/\.$/, "");
+        return normalized === "google.com" || normalized.endsWith(".google.com") || /^google\.[a-z.]+$/.test(normalized) || /\.google\.[a-z.]+$/.test(normalized) || normalized === "googleusercontent.com" || normalized.endsWith(".googleusercontent.com");
+      }
+      function safeCitationTarget(rawHref) {
+        let parsed;
+        try {
+          parsed = new URL(rawHref);
+        } catch {
+          return null;
+        }
+        if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
+        if (parsed.username || parsed.password) return null;
+        if (isGoogleHost(parsed.hostname)) {
+          if (parsed.pathname !== "/url") return null;
+          const redirected = parsed.searchParams.get("q") ?? parsed.searchParams.get("url");
+          if (!redirected) return null;
+          try {
+            parsed = new URL(redirected);
+          } catch {
+            return null;
+          }
+          if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
+          if (parsed.username || parsed.password) return null;
+        }
+        if (isGoogleHost(parsed.hostname)) return null;
+        return { href: parsed.href, host: parsed.hostname };
+      }
       return Array.from(document.querySelectorAll(selectors.item)).filter((pair) => !onlyQuestion2 || (pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "") === onlyQuestion2).map((pair) => {
         const clickTarget = pair.querySelector(selectors.clickTarget);
         const expanded = pair.classList.contains(selectors.expandedClass) || clickTarget?.getAttribute("aria-expanded") === "true";
@@ -2336,21 +2481,14 @@ var PAAExtractor = class {
             ...Array.from(anchorRoot.querySelectorAll("a[href]"))
           ];
           for (const a of prioritized) {
-            const href = a.href;
-            if (!/^https?:\/\//.test(href) || seenHrefs.has(href)) continue;
-            let host = "";
-            try {
-              host = new URL(href).hostname;
-            } catch {
-              continue;
-            }
-            if (/(^|\.)google(usercontent)?\.[a-z.]+$/.test(host)) continue;
-            seenHrefs.add(href);
+            const target = safeCitationTarget(a.href);
+            if (!target || seenHrefs.has(target.href)) continue;
+            seenHrefs.add(target.href);
             anchors.push({
               text: (a.textContent || "").trim(),
               label: cleanSourceLabel(a.getAttribute("aria-label")),
-              href,
-              host
+              href: target.href,
+              host: target.host
             });
           }
           const primary = anchors.find((x) => x.label.length > 2 || x.text.length > 2) ?? anchors[0];
@@ -2407,10 +2545,53 @@ var PAAExtractor = class {
       extracted_at: (/* @__PURE__ */ new Date()).toISOString()
     };
   }
+  upsertCollectedRow(row) {
+    const key = this.normalizeQuestion(row.question);
+    const index = this.collectedRows.findIndex((existing2) => this.normalizeQuestion(existing2.question) === key);
+    if (index < 0) {
+      this.collectedRows.push({ ...row });
+      return true;
+    }
+    const existing = this.collectedRows[index];
+    const merged = {
+      ...existing,
+      ...row,
+      answer: row.answer || existing.answer,
+      source_title: row.source_title || existing.source_title,
+      source_site: row.source_site || existing.source_site,
+      source_cite: row.source_cite || existing.source_cite,
+      extracted_at: row.extracted_at || existing.extracted_at
+    };
+    const changed = merged.answer !== existing.answer || merged.source_title !== existing.source_title || merged.source_site !== existing.source_site || merged.source_cite !== existing.source_cite;
+    if (changed) this.collectedRows[index] = merged;
+    return changed;
+  }
+  async emitProgress(phase) {
+    if (!this.onPaaProgress || this.collectedRows.length === 0) return;
+    const snapshot = {
+      sequence: ++this.progressSequence,
+      observedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      clickedQuestions: [...this.clickedQuestions],
+      phase,
+      records: this.collectedRows.map((row) => ({ ...row }))
+    };
+    try {
+      await this.onPaaProgress(snapshot);
+    } catch (err) {
+      console.warn(JSON.stringify({
+        event: "paa_progress_sink_failed",
+        sequence: snapshot.sequence,
+        phase: snapshot.phase,
+        message: err instanceof Error ? err.message : String(err)
+      }));
+    }
+  }
   getPartialResult() {
     const ctx = this.partialContext;
     if (!ctx || this.collectedRows.length === 0) return null;
-    const flat = [...this.collectedRows];
+    const flat = this.collectedRows.map((row) => ({ ...row }));
+    const paaWithoutAnswer = ctx.questionsOnly ? 0 : flat.filter((row) => !row.answer?.trim()).length;
+    const paaWithoutSource = ctx.questionsOnly ? 0 : flat.filter((row) => !row.source_cite?.trim()).length;
     const stats = {
       seed: ctx.seed,
       totalQuestions: flat.length,
@@ -2425,7 +2606,17 @@ var PAAExtractor = class {
       diagnostics: {
         completionStatus: "paa_found",
         problem: null,
-        completeness: { ...this.completeness },
+        resultQuality: "partial",
+        degradedResult: false,
+        retryRecommended: true,
+        completeness: {
+          ...this.completeness,
+          paaWithoutAnswer,
+          paaWithoutSource,
+          paaRequested: ctx.maxQuestions,
+          paaReturned: flat.length,
+          paaUnique: flat.length
+        },
         warnings: [{
           code: "paa_partial",
           surface: "paa",
@@ -2434,6 +2625,9 @@ var PAAExtractor = class {
         }]
       },
       totalQuestions: flat.length,
+      surface: "web",
+      aiOverview: { detected: false, text: null, citations: [] },
+      aiMode: { detected: false, text: null, citations: [] },
       whatPeopleSaying: [],
       tree: this.buildTree(flat, ctx.seed),
       flat,
@@ -2441,7 +2635,7 @@ var PAAExtractor = class {
       forums: [],
       organicResults: [],
       localPack: [],
-      entityIds: {},
+      entityIds: { entities: [], kgIds: [], cids: [], gcids: [] },
       stats
     };
   }
@@ -2451,7 +2645,8 @@ var PAAExtractor = class {
     const seenQs = /* @__PURE__ */ new Set();
     const orderedQs = [];
     this.collectedRows = [];
-    const results = this.collectedRows;
+    this.progressSequence = 0;
+    this.clickedQuestions = /* @__PURE__ */ new Set();
     const abandonedNeverReclick = /* @__PURE__ */ new Set();
     const clickedOnceEver = /* @__PURE__ */ new Set();
     const capturedItems = /* @__PURE__ */ new Map();
@@ -2499,12 +2694,17 @@ var PAAExtractor = class {
       if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
       this.throwIfAborted(signal);
       const states = await readItemStates();
+      let discovered = false;
       for (const s of states) {
         if (!seenQs.has(s.q)) {
           seenQs.add(s.q);
           orderedQs.push(s.q);
+          if (orderedQs.length <= options.maxQuestions) {
+            discovered = this.upsertCollectedRow(this.toFlatRow({ question: s.q }, 1, null, options.query)) || discovered;
+          }
         }
       }
+      if (discovered) await this.emitProgress("discovered");
       if (options.questionsOnly && seenQs.size >= options.maxQuestions) break;
       const kept = new Set(orderedQs.slice(0, options.maxQuestions));
       const clickable = states.filter((s) => kept.has(s.q) && !clickedOnceEver.has(s.q) && !abandonedNeverReclick.has(s.q));
@@ -2524,6 +2724,7 @@ var PAAExtractor = class {
       this.reporter.onDepth(++round);
       await this.throwIfCaptcha(page, "Google PAA expansion");
       clickedOnceEver.add(target.q);
+      this.clickedQuestions.add(target.q);
       const expansionStatus = await expandOneItemSerially(target.q);
       if (!options.questionsOnly) {
         let clickedItem = (await this.extractVisibleItems(page, target.q))[0];
@@ -2531,7 +2732,12 @@ var PAAExtractor = class {
           await page.waitForTimeout(1300);
           clickedItem = (await this.extractVisibleItems(page, target.q))[0];
         }
-        if (clickedItem?.answer) capturedItems.set(target.q, clickedItem);
+        if (clickedItem) {
+          capturedItems.set(target.q, clickedItem);
+          if (this.upsertCollectedRow(this.toFlatRow(clickedItem, 1, null, options.query))) {
+            await this.emitProgress("answer_captured");
+          }
+        }
       }
       if (expansionStatus === "failed" && !capturedItems.has(target.q)) {
         abandonedNeverReclick.add(target.q);
@@ -2547,14 +2753,19 @@ var PAAExtractor = class {
           if (!seenQs.has(state.q)) {
             seenQs.add(state.q);
             orderedQs.push(state.q);
+            if (orderedQs.length <= options.maxQuestions) {
+              this.upsertCollectedRow(this.toFlatRow({ question: state.q }, 1, null, options.query));
+            }
           }
         }
+        await this.emitProgress("discovered");
       }
     }
     const itemMap = options.questionsOnly ? /* @__PURE__ */ new Map() : new Map(capturedItems);
     if (!options.questionsOnly) {
       await this.fillIncompleteItems(page, orderedQs, itemMap, options, abandonedNeverReclick);
     }
+    const results = [];
     for (const q of orderedQs) {
       if (results.length >= options.maxQuestions) break;
       const key = this.normalizeQuestion(q);
@@ -2562,14 +2773,22 @@ var PAAExtractor = class {
       seenKeys.add(key);
       const item = itemMap.get(q);
       if (item) {
-        results.push(this.toFlatRow(item, 1, null, options.query));
+        const row = this.toFlatRow(item, 1, null, options.query);
+        this.upsertCollectedRow(row);
+        results.push(row);
         this.reporter.onQuestion({ question: item.question, answer: item.answer ?? null, sourceTitle: item.sourceTitle ?? null, sourceSite: item.sourceSite ?? null, sourceCite: item.sourceCite ?? null, depth: 1, parentQuestion: null, children: [] });
       } else {
-        results.push(this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, 1, null, options.query));
+        const row = this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, 1, null, options.query);
+        this.upsertCollectedRow(row);
+        results.push(row);
       }
     }
     this.completeness.paaWithoutAnswer = options.questionsOnly ? 0 : results.filter((r) => !r.answer).length;
-    this.completeness.paaWithoutSource = options.questionsOnly ? 0 : results.filter((r) => !r.source_title && !r.source_site && !r.source_cite).length;
+    this.completeness.paaWithoutSource = options.questionsOnly ? 0 : results.filter((r) => !r.source_cite).length;
+    this.completeness.paaRequested = options.maxQuestions;
+    this.completeness.paaReturned = results.length;
+    this.completeness.paaUnique = seenKeys.size;
+    await this.emitProgress("fill_incomplete");
     return results;
   }
   async fillIncompleteItems(page, orderedQs, itemMap, options, abandonedNeverReclick) {
@@ -2594,6 +2813,10 @@ var PAAExtractor = class {
         sourceSite: item.sourceSite ?? existing?.sourceSite,
         sourceCite: item.sourceCite ?? existing?.sourceCite
       });
+      const upgraded = itemMap.get(q);
+      if (upgraded && this.upsertCollectedRow(this.toFlatRow(upgraded, 1, null, options.query))) {
+        await this.emitProgress("fill_incomplete");
+      }
     }
     this.completeness.paaAnswersRecovered = [...missingAnswersBefore].filter((q) => itemMap.get(q)?.answer).length;
   }
@@ -3058,7 +3281,15 @@ var PAAExtractor = class {
     this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
     this.aioShareUrlEarly = null;
     this.collectedRows = [];
-    this.partialContext = { seed: options.query, location: options.location ?? null, startMs };
+    this.progressSequence = 0;
+    this.clickedQuestions = /* @__PURE__ */ new Set();
+    this.partialContext = {
+      seed: options.query,
+      location: options.location ?? null,
+      startMs,
+      maxQuestions: options.maxQuestions,
+      questionsOnly: options.questionsOnly
+    };
     const isMobile = options.device === "mobile";
     const config = {
       headless: options.headless,
@@ -3130,6 +3361,7 @@ var PAAExtractor = class {
           extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
           diagnostics: {
             completionStatus: hasPaa ? "paa_found" : "no_paa",
+            ...!hasPaa ? { noPaaObserved: true } : {},
             problem: null,
             completeness: { ...this.completeness },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, void 0, locationResolution) } : {}
@@ -3249,6 +3481,7 @@ var PAAExtractor = class {
           extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
           diagnostics: {
             completionStatus: "no_paa",
+            noPaaObserved: true,
             problem: null,
             completeness: { ...this.completeness },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence2, locationResolution) } : {}
@@ -3340,6 +3573,8 @@ var PAAExtractor = class {
       };
     } catch (err) {
       errorCount++;
+      await this.emitProgress("attempt_ending").catch(() => {
+      });
       this.reporter.onError(err instanceof Error ? err : new Error(String(err)));
       throw err;
     }
@@ -4019,6 +4254,11 @@ function getAttemptLogSink(rawOptions) {
   const sink = rawOptions.onAttemptEvent;
   return typeof sink === "function" ? sink : void 0;
 }
+function getPaaProgressSink(rawOptions) {
+  if (!rawOptions || typeof rawOptions !== "object") return void 0;
+  const sink = rawOptions.onPaaProgress;
+  return typeof sink === "function" ? sink : void 0;
+}
 async function emitAttemptEvent(sink, event) {
   if (!sink) return;
   try {
@@ -4051,17 +4291,32 @@ function classifyAttemptResult(result) {
 function resultHasUsableContent(result) {
   return result.flat.length > 0 || result.organicResults.length > 0 || result.localPack.length > 0 || result.videos.length > 0 || result.forums.length > 0 || result.whatPeopleSaying.length > 0 || result.aiOverview.detected || result.aiMode.detected;
 }
-function withResultQuality(result) {
+function withResultQuality(result, requestedPaaCount, questionsOnly) {
   const hasUsableContent = resultHasUsableContent(result);
   const hasWarnings = (result.diagnostics.warnings?.length ?? 0) > 0;
-  const resultQuality = hasUsableContent ? hasWarnings ? "partial" : "complete" : "degraded";
+  const uniquePaaCount = new Set(result.flat.map((row) => row.question.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim())).size;
+  const paaWithoutAnswer = questionsOnly ? 0 : result.flat.filter((row) => !row.answer?.trim()).length;
+  const paaWithoutSource = questionsOnly ? 0 : result.flat.filter((row) => !row.source_cite?.trim()).length;
+  const noPaa = result.diagnostics.completionStatus === "no_paa" && (result.diagnostics.noPaaObserved === true || hasUsableContent);
+  const hasPaa = result.diagnostics.completionStatus === "paa_found";
+  const paaIncomplete = hasPaa && (uniquePaaCount < requestedPaaCount || paaWithoutAnswer > 0 || paaWithoutSource > 0 || Boolean(result.diagnostics.interruption));
+  const resultQuality = noPaa ? "complete" : hasPaa && uniquePaaCount > 0 ? paaIncomplete || hasWarnings ? "partial" : "complete" : hasUsableContent ? hasWarnings ? "partial" : "complete" : "degraded";
   const degradedResult = resultQuality === "degraded";
   result.diagnostics = {
     ...result.diagnostics,
     resultQuality,
     degradedResult,
     degradationReasons: degradedResult ? ["empty_primary_serp"] : [],
-    retryRecommended: degradedResult
+    retryRecommended: degradedResult || resultQuality === "partial" && (paaIncomplete || result.diagnostics.warnings?.some((warning) => warning.retryable) === true),
+    completeness: {
+      paaWithoutAnswer,
+      paaWithoutSource,
+      paaAnswersRecovered: result.diagnostics.completeness?.paaAnswersRecovered ?? 0,
+      aioShareCaptured: result.diagnostics.completeness?.aioShareCaptured ?? null,
+      paaRequested: requestedPaaCount,
+      paaReturned: result.flat.length,
+      paaUnique: uniquePaaCount
+    }
   };
   return result;
 }
@@ -4121,10 +4376,10 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId) {
     }));
   }
 }
-async function extractOnce(options, signal, forceManagedSerp = false) {
+async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress) {
   const driver = createSerpDriver(options, { forceManaged: forceManagedSerp });
   const reporter = new ProgressReporter();
-  const extractor = new PAAExtractor(driver, reporter);
+  const extractor = new PAAExtractor(driver, reporter, onPaaProgress);
   if (signal?.aborted) {
     return {
       result: null,
@@ -4176,17 +4431,25 @@ async function extractOnce(options, signal, forceManagedSerp = false) {
   }
   if (error) {
     const outcome = classifyAttemptError(error);
-    const budgetExhausted = outcome === "timeout" || outcome === "request_aborted";
+    const interruptionCode = outcome === "timeout" || outcome === "request_aborted" || outcome === "browser_session_interrupted" ? outcome : null;
     let salvaged = null;
-    if (budgetExhausted) {
+    if (interruptionCode) {
       try {
         salvaged = typeof extractor.getPartialResult === "function" ? extractor.getPartialResult() : null;
       } catch {
         salvaged = null;
       }
     }
-    if (salvaged) {
-      return { result: salvaged, error: null, cleanup, debug, salvagedFrom: error };
+    if (salvaged && interruptionCode) {
+      salvaged.diagnostics = {
+        ...salvaged.diagnostics,
+        interruption: {
+          code: interruptionCode,
+          reasonSource: "client_runtime",
+          message: errorMessage(error)
+        }
+      };
+      return { result: salvaged, error, cleanup, debug, salvagedFrom: error };
     }
     return { result: null, error, cleanup, debug };
   }
@@ -4196,6 +4459,7 @@ async function harvest(rawOptions) {
   const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};
   const signal = getAbortSignal(rawOptions);
   const onAttemptEvent = getAttemptLogSink(rawOptions);
+  const onPaaProgress = getPaaProgressSink(rawOptions);
   const forceManagedSerp = raw.forceManagedSerp === true;
   const requestedProxyMode = raw.proxyMode;
   const proxyMode = requestedProxyMode === "location" || requestedProxyMode === "none" || requestedProxyMode === "configured" ? requestedProxyMode : DEFAULT_PROXY_MODE;
@@ -4265,7 +4529,7 @@ async function harvest(rawOptions) {
         ...baseCtx,
         forceHeadless: allowHeadlessTest,
         forceHeadful: !allowHeadlessTest
-      }, () => extractOnce(attemptOptions, signal, forceManagedSerp)) : await extractOnce(attemptOptions, signal, forceManagedSerp);
+      }, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress);
       if (attempt.error) {
         const err = attempt.error;
         const outcome2 = classifyAttemptError(err);
@@ -4298,7 +4562,7 @@ async function harvest(rawOptions) {
           maxAttempts,
           outcome: outcome2,
           kernelSessionId: attempt.cleanup.kernelSessionId,
-          questionCount: 0,
+          questionCount: attempt.result?.totalQuestions ?? 0,
           durationMs: Date.now() - startedAtMs,
           error: errorMessage(err),
           willRetry: willRetry2,
@@ -4309,6 +4573,9 @@ async function harvest(rawOptions) {
         await cleanupDisposableProxy(kernelApiKey, resolution2.disposableProxyId);
         lastError = err;
         if (willRetry2) continue;
+        if (attempt.result) {
+          return withResultQuality(stripInternalDebug(attempt.result, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
+        }
         break;
       }
       const result = attempt.result;
@@ -4342,7 +4609,7 @@ async function harvest(rawOptions) {
         if (willRetry2) continue;
         break;
       }
-      const finalResult = withResultQuality(stripInternalDebug(result, requestedDebug));
+      const finalResult = withResultQuality(stripInternalDebug(result, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
       const outcome = classifyAttemptResult(finalResult);
       const willRetry = outcome === "degraded_result" && i < maxAttempts - 1;
       const resultError = outcome === "degraded_result" ? degradedResultMessage(finalResult) : null;