mcp-scraper 0.69.0 → 0.72.0

package/dist/index.cjs

@@ -443,6 +443,7 @@ var NANGO_USD_PER_CONNECTION_MONTH = envRate("NANGO_USD_PER_CONNECTION_MONTH", 1
 var NANGO_USD_PER_FUNCTION_RUN = envRate("NANGO_USD_PER_FUNCTION_RUN", 1e-4);
 var NANGO_USD_PER_PROXY_REQUEST = envRate("NANGO_USD_PER_PROXY_REQUEST", 1e-4);
 var NANGO_USD_PER_COMPUTE_SEC = envRate("NANGO_USD_PER_COMPUTE_SEC", 2e-4);
+var BRIGHTDATA_BROWSER_USD_PER_GB = envRate("BRIGHTDATA_BROWSER_USD_PER_GB", 0);
 function kernelCostUsd(ms, headful) {
   const sec = Math.max(0, ms) / 1e3;
   return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
@@ -460,6 +461,9 @@ function vendorCostUsd(vendor, units) {
   if (vendor === "nango_function_run") return Math.max(0, units) * NANGO_USD_PER_FUNCTION_RUN;
   if (vendor === "nango_proxy_request") return Math.max(0, units) * NANGO_USD_PER_PROXY_REQUEST;
   if (vendor === "nango_compute") return Math.max(0, units) * NANGO_USD_PER_COMPUTE_SEC;
+  if (vendor === "brightdata_browser_api") {
+    return Math.max(0, units) / 1e9 * BRIGHTDATA_BROWSER_USD_PER_GB;
+  }
   return 0;
 }
 
@@ -480,7 +484,8 @@ async function runCostTelemetryMigration() {
     SELECT
       (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
       AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
-      AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name = 'method') = 1
+      AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key')) = 2
+      AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
       AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
       AS ready
   `);
@@ -536,6 +541,7 @@ async function runCostTelemetryMigration() {
       unit_type    TEXT,
       est_cost_usd REAL NOT NULL DEFAULT 0,
       error        TEXT,
+      source_key   TEXT,
       created_at   TEXT NOT NULL DEFAULT (datetime('now'))
     )
   `);
@@ -546,6 +552,11 @@ async function runCostTelemetryMigration() {
     await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN method TEXT`);
   } catch {
   }
+  try {
+    await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN source_key TEXT`);
+  } catch {
+  }
+  await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS vendor_usage_log_vendor_source_key ON vendor_usage_log(vendor, source_key) WHERE source_key IS NOT NULL`);
   await db.execute(`
     CREATE TABLE IF NOT EXISTS cost_probe_runs (
       id                            TEXT PRIMARY KEY,
@@ -625,26 +636,29 @@ async function recordVendorUsage(r) {
     await migrateCostTelemetry();
     const ctx = currentCostContext();
     const db = getDb();
-    await db.execute({
-      sql: `INSERT INTO vendor_usage_log
-        (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method)
-        VALUES (?,?,?,?,?,?,?,?,?,?,?)`,
+    const result = await db.execute({
+      sql: `INSERT OR IGNORE INTO vendor_usage_log
+        (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key)
+        VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`,
       args: [
         (0, import_node_crypto2.randomUUID)(),
-        ctx?.op ?? null,
-        ctx?.probeRunId ?? null,
-        ctx?.userId ?? null,
+        r.op ?? ctx?.op ?? null,
+        r.probeRunId ?? ctx?.probeRunId ?? null,
+        r.userId ?? ctx?.userId ?? null,
         r.vendor,
         r.model ?? null,
         r.units,
         r.unitType,
         vendorCostUsd(r.vendor, r.units),
         r.error ?? null,
-        ctx?.subOp ?? null
+        r.method ?? ctx?.subOp ?? null,
+        r.sourceKey ?? null
       ]
     });
+    return result.rowsAffected === 1;
   } catch (err) {
     console.warn("[cost-telemetry] recordVendorUsage failed:", err instanceof Error ? err.message : String(err));
+    return false;
   }
 }
 function boolToInt(v) {
@@ -1177,6 +1191,13 @@ var BrowserDriver = class {
   getKernelSessionId() {
     return this.kernelSessionId;
   }
+  getProviderSessionMetadata() {
+    return {
+      provider: this.kernelSessionId ? "kernel" : "local",
+      providerSessionId: this.kernelSessionId,
+      disconnectObservation: null
+    };
+  }
   getDebugSnapshot() {
     return this.debugSnapshot;
   }
@@ -1191,6 +1212,7 @@ var BrowserDriver = class {
       const proxySource = this.kernelProxySource;
       const proxyType = this.kernelProxyType;
       const headlessSent = this.kernelHeadlessSent;
+      const provider = this.getProviderSessionMetadata();
       this.browser = null;
       this.context = null;
       this.page = null;
@@ -1225,7 +1247,11 @@ var BrowserDriver = class {
           kernelDeleteSucceeded: deleteResult.status === "fulfilled",
           kernelDeleteError: deleteResult.status === "rejected" ? deleteResult.reason instanceof Error ? deleteResult.reason.message : String(deleteResult.reason) : null,
           browserCloseSucceeded: closeResult.status === "fulfilled",
-          browserCloseError: closeResult.status === "rejected" ? closeResult.reason instanceof Error ? closeResult.reason.message : String(closeResult.reason) : null
+          browserCloseError: closeResult.status === "rejected" ? closeResult.reason instanceof Error ? closeResult.reason.message : String(closeResult.reason) : null,
+          provider: provider.provider,
+          providerSessionId: provider.providerSessionId,
+          providerDisconnectObserved: false,
+          providerDisconnectMessage: null
         };
         if (deleteResult.status === "rejected") {
           console.warn(JSON.stringify({
@@ -1257,7 +1283,11 @@ var BrowserDriver = class {
         kernelDeleteSucceeded: null,
         kernelDeleteError: null,
         browserCloseSucceeded: true,
-        browserCloseError: null
+        browserCloseError: null,
+        provider: provider.provider,
+        providerSessionId: provider.providerSessionId,
+        providerDisconnectObserved: false,
+        providerDisconnectMessage: null
       };
     } else if (this.context) {
       const ctx = this.context;
@@ -1270,7 +1300,11 @@ var BrowserDriver = class {
         kernelDeleteSucceeded: null,
         kernelDeleteError: null,
         browserCloseSucceeded: true,
-        browserCloseError: null
+        browserCloseError: null,
+        provider: "local",
+        providerSessionId: null,
+        providerDisconnectObserved: false,
+        providerDisconnectMessage: null
       };
     }
     return {
@@ -1279,13 +1313,64 @@ var BrowserDriver = class {
       kernelDeleteSucceeded: null,
       kernelDeleteError: null,
       browserCloseSucceeded: null,
-      browserCloseError: null
+      browserCloseError: null,
+      provider: "local",
+      providerSessionId: null,
+      providerDisconnectObserved: false,
+      providerDisconnectMessage: null
     };
   }
 };
 
 // src/driver/BrightDataSerpDriver.ts
 var import_playwright2 = require("playwright");
+
+// src/driver/browser-provider-telemetry.ts
+var MAX_PROVIDER_SESSION_ID_LENGTH = 512;
+var MAX_PROVIDER_MESSAGE_LENGTH = 320;
+function boundedText(value, maxLength) {
+  if (typeof value !== "string") return null;
+  const normalized = value.replace(/[\u0000-\u001f\u007f]+/g, " ").replace(/\s+/g, " ").trim();
+  if (!normalized) return null;
+  return normalized.slice(0, maxLength);
+}
+function boundedProviderMessage(value) {
+  const message = boundedText(value, MAX_PROVIDER_MESSAGE_LENGTH);
+  if (!message) return null;
+  return message.replace(/\b(?:wss?|https?):\/\/\S+/gi, "[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization):\s*\S+/gi, "$1: [redacted]").replace(/\bBearer\s+\S+/gi, "Bearer [redacted]").slice(0, MAX_PROVIDER_MESSAGE_LENGTH);
+}
+function sanitizeProviderLocalMessage(value) {
+  return boundedProviderMessage(value);
+}
+function validateProviderSessionId(value) {
+  if (typeof value !== "string" || value.length === 0 || value.length > MAX_PROVIDER_SESSION_ID_LENGTH || value.trim() !== value || /[\s\u0000-\u001f\u007f]/.test(value) || /:\/\//.test(value)) {
+    throw new Error("Bright Data did not return a valid browser session ID");
+  }
+  return value;
+}
+function makeProviderInterruptionObservation(source, message, observedAt = (/* @__PURE__ */ new Date()).toISOString()) {
+  const safeMessage = boundedProviderMessage(message);
+  return {
+    source,
+    message: safeMessage ?? source.replaceAll("_", " "),
+    observedAt
+  };
+}
+function projectProviderSessionPublic(metadata) {
+  return {
+    provider: metadata.provider,
+    providerDisconnectObserved: metadata.disconnectObservation !== null,
+    providerDisconnectMessage: metadata.disconnectObservation?.message ?? null
+  };
+}
+function projectProviderSessionPrivate(metadata) {
+  return {
+    ...projectProviderSessionPublic(metadata),
+    providerSessionId: metadata.providerSessionId
+  };
+}
+
+// src/driver/BrightDataSerpDriver.ts
 var BLOCKED_TYPES = /* @__PURE__ */ new Set(["image", "media", "font", "stylesheet", "ping"]);
 var BLOCKED_HOST_PARTS = [
   "doubleclick.net",
@@ -1335,8 +1420,19 @@ var BrightDataSerpDriver = class {
   browser = null;
   context = null;
   page = null;
+  cdpSession = null;
+  providerSessionId = null;
+  disconnectObservation = null;
+  closing = false;
+  observeDisconnect(source, message) {
+    if (this.closing || this.disconnectObservation) return;
+    this.disconnectObservation = makeProviderInterruptionObservation(source, message);
+  }
   async launch(_config) {
     if (this.page) return;
+    this.closing = false;
+    this.providerSessionId = null;
+    this.disconnectObservation = null;
     const endpoint = brightDataBrowserEndpoint();
     if (!endpoint) throw new Error("SERP browser service is not configured");
     this.browser = await import_playwright2.chromium.connectOverCDP(endpoint);
@@ -1352,6 +1448,14 @@ var BrightDataSerpDriver = class {
       }
     });
     this.page = this.context.pages()[0] ?? await this.context.newPage();
+    this.browser.on("disconnected", () => this.observeDisconnect("browser_disconnected"));
+    this.context.on("close", () => this.observeDisconnect("context_closed"));
+    this.page.on("close", () => this.observeDisconnect("page_closed"));
+    this.page.on("crash", () => this.observeDisconnect("page_crashed"));
+    this.cdpSession = await this.context.newCDPSession(this.page);
+    const customCdp = this.cdpSession;
+    const sessionResponse = await customCdp.send("Browser.getSessionId");
+    this.providerSessionId = validateProviderSessionId(sessionResponse?.sessionId);
     await this.page.route("**/*", async (route) => {
       const request = route.request();
       const resourceType = request.resourceType();
@@ -1407,23 +1511,38 @@ var BrightDataSerpDriver = class {
   getKernelSessionId() {
     return null;
   }
+  getProviderSessionMetadata() {
+    return {
+      provider: "bright_data_browser_api",
+      providerSessionId: this.providerSessionId,
+      disconnectObservation: this.disconnectObservation
+    };
+  }
   getDebugSnapshot() {
     return { kernel: null, context: null, networkLocation: null, serpNavigation: null };
   }
   async close() {
+    this.closing = true;
     let error = null;
     try {
       await this.browser?.close();
     } catch (caught) {
-      error = caught instanceof Error ? caught.message : String(caught);
+      error = sanitizeProviderLocalMessage(caught instanceof Error ? caught.message : String(caught)) ?? "Bright Data browser close failed";
+      this.disconnectObservation ??= makeProviderInterruptionObservation("browser_close_error", error);
     }
+    const provider = projectProviderSessionPrivate(this.getProviderSessionMetadata());
+    this.browser = null;
+    this.context = null;
+    this.page = null;
+    this.cdpSession = null;
     return {
       kernelSessionId: null,
       kernelDeleteStarted: false,
       kernelDeleteSucceeded: null,
       kernelDeleteError: null,
       browserCloseSucceeded: error === null,
-      browserCloseError: error
+      browserCloseError: error,
+      ...provider
     };
   }
 };
@@ -2319,12 +2438,14 @@ async function extractAISurfacesFromDocument(config) {
 
 // src/extractor/PAAExtractor.ts
 var PAAExtractor = class {
-  constructor(driver, reporter) {
+  constructor(driver, reporter, onPaaProgress) {
     this.driver = driver;
     this.reporter = reporter;
+    this.onPaaProgress = onPaaProgress;
   }
   driver;
   reporter;
+  onPaaProgress;
   completeness = {
     paaWithoutAnswer: 0,
     paaWithoutSource: 0,
@@ -2334,6 +2455,8 @@ var PAAExtractor = class {
   aioShareUrlEarly = null;
   collectedRows = [];
   partialContext = null;
+  progressSequence = 0;
+  clickedQuestions = /* @__PURE__ */ new Set();
   normalizeQuestion(q) {
     return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
   }
@@ -2370,6 +2493,34 @@ var PAAExtractor = class {
       function cleanSourceLabel(value) {
         return (value ?? "").replace(/\s*\(\+\d+\)\s*-\s*View related links.*$/i, "").replace(/\s*-\s*Opens in new tab.*$/i, "").trim();
       }
+      function isGoogleHost(host) {
+        const normalized = host.toLowerCase().replace(/\.$/, "");
+        return normalized === "google.com" || normalized.endsWith(".google.com") || /^google\.[a-z.]+$/.test(normalized) || /\.google\.[a-z.]+$/.test(normalized) || normalized === "googleusercontent.com" || normalized.endsWith(".googleusercontent.com");
+      }
+      function safeCitationTarget(rawHref) {
+        let parsed;
+        try {
+          parsed = new URL(rawHref);
+        } catch {
+          return null;
+        }
+        if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
+        if (parsed.username || parsed.password) return null;
+        if (isGoogleHost(parsed.hostname)) {
+          if (parsed.pathname !== "/url") return null;
+          const redirected = parsed.searchParams.get("q") ?? parsed.searchParams.get("url");
+          if (!redirected) return null;
+          try {
+            parsed = new URL(redirected);
+          } catch {
+            return null;
+          }
+          if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
+          if (parsed.username || parsed.password) return null;
+        }
+        if (isGoogleHost(parsed.hostname)) return null;
+        return { href: parsed.href, host: parsed.hostname };
+      }
       return Array.from(document.querySelectorAll(selectors.item)).filter((pair) => !onlyQuestion2 || (pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "") === onlyQuestion2).map((pair) => {
         const clickTarget = pair.querySelector(selectors.clickTarget);
         const expanded = pair.classList.contains(selectors.expandedClass) || clickTarget?.getAttribute("aria-expanded") === "true";
@@ -2391,21 +2542,14 @@ var PAAExtractor = class {
             ...Array.from(anchorRoot.querySelectorAll("a[href]"))
           ];
           for (const a of prioritized) {
-            const href = a.href;
-            if (!/^https?:\/\//.test(href) || seenHrefs.has(href)) continue;
-            let host = "";
-            try {
-              host = new URL(href).hostname;
-            } catch {
-              continue;
-            }
-            if (/(^|\.)google(usercontent)?\.[a-z.]+$/.test(host)) continue;
-            seenHrefs.add(href);
+            const target = safeCitationTarget(a.href);
+            if (!target || seenHrefs.has(target.href)) continue;
+            seenHrefs.add(target.href);
             anchors.push({
               text: (a.textContent || "").trim(),
               label: cleanSourceLabel(a.getAttribute("aria-label")),
-              href,
-              host
+              href: target.href,
+              host: target.host
             });
           }
           const primary = anchors.find((x) => x.label.length > 2 || x.text.length > 2) ?? anchors[0];
@@ -2462,10 +2606,53 @@ var PAAExtractor = class {
       extracted_at: (/* @__PURE__ */ new Date()).toISOString()
     };
   }
+  upsertCollectedRow(row) {
+    const key = this.normalizeQuestion(row.question);
+    const index = this.collectedRows.findIndex((existing2) => this.normalizeQuestion(existing2.question) === key);
+    if (index < 0) {
+      this.collectedRows.push({ ...row });
+      return true;
+    }
+    const existing = this.collectedRows[index];
+    const merged = {
+      ...existing,
+      ...row,
+      answer: row.answer || existing.answer,
+      source_title: row.source_title || existing.source_title,
+      source_site: row.source_site || existing.source_site,
+      source_cite: row.source_cite || existing.source_cite,
+      extracted_at: row.extracted_at || existing.extracted_at
+    };
+    const changed = merged.answer !== existing.answer || merged.source_title !== existing.source_title || merged.source_site !== existing.source_site || merged.source_cite !== existing.source_cite;
+    if (changed) this.collectedRows[index] = merged;
+    return changed;
+  }
+  async emitProgress(phase) {
+    if (!this.onPaaProgress || this.collectedRows.length === 0) return;
+    const snapshot = {
+      sequence: ++this.progressSequence,
+      observedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      clickedQuestions: [...this.clickedQuestions],
+      phase,
+      records: this.collectedRows.map((row) => ({ ...row }))
+    };
+    try {
+      await this.onPaaProgress(snapshot);
+    } catch (err) {
+      console.warn(JSON.stringify({
+        event: "paa_progress_sink_failed",
+        sequence: snapshot.sequence,
+        phase: snapshot.phase,
+        message: err instanceof Error ? err.message : String(err)
+      }));
+    }
+  }
   getPartialResult() {
     const ctx = this.partialContext;
     if (!ctx || this.collectedRows.length === 0) return null;
-    const flat = [...this.collectedRows];
+    const flat = this.collectedRows.map((row) => ({ ...row }));
+    const paaWithoutAnswer = ctx.questionsOnly ? 0 : flat.filter((row) => !row.answer?.trim()).length;
+    const paaWithoutSource = ctx.questionsOnly ? 0 : flat.filter((row) => !row.source_cite?.trim()).length;
     const stats = {
       seed: ctx.seed,
       totalQuestions: flat.length,
@@ -2480,7 +2667,17 @@ var PAAExtractor = class {
       diagnostics: {
         completionStatus: "paa_found",
         problem: null,
-        completeness: { ...this.completeness },
+        resultQuality: "partial",
+        degradedResult: false,
+        retryRecommended: true,
+        completeness: {
+          ...this.completeness,
+          paaWithoutAnswer,
+          paaWithoutSource,
+          paaRequested: ctx.maxQuestions,
+          paaReturned: flat.length,
+          paaUnique: flat.length
+        },
         warnings: [{
           code: "paa_partial",
           surface: "paa",
@@ -2489,6 +2686,9 @@ var PAAExtractor = class {
         }]
       },
       totalQuestions: flat.length,
+      surface: "web",
+      aiOverview: { detected: false, text: null, citations: [] },
+      aiMode: { detected: false, text: null, citations: [] },
       whatPeopleSaying: [],
       tree: this.buildTree(flat, ctx.seed),
       flat,
@@ -2496,7 +2696,7 @@ var PAAExtractor = class {
       forums: [],
       organicResults: [],
       localPack: [],
-      entityIds: {},
+      entityIds: { entities: [], kgIds: [], cids: [], gcids: [] },
       stats
     };
   }
@@ -2506,7 +2706,8 @@ var PAAExtractor = class {
     const seenQs = /* @__PURE__ */ new Set();
     const orderedQs = [];
     this.collectedRows = [];
-    const results = this.collectedRows;
+    this.progressSequence = 0;
+    this.clickedQuestions = /* @__PURE__ */ new Set();
     const abandonedNeverReclick = /* @__PURE__ */ new Set();
     const clickedOnceEver = /* @__PURE__ */ new Set();
     const capturedItems = /* @__PURE__ */ new Map();
@@ -2554,12 +2755,17 @@ var PAAExtractor = class {
       if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
       this.throwIfAborted(signal);
       const states = await readItemStates();
+      let discovered = false;
       for (const s of states) {
         if (!seenQs.has(s.q)) {
           seenQs.add(s.q);
           orderedQs.push(s.q);
+          if (orderedQs.length <= options.maxQuestions) {
+            discovered = this.upsertCollectedRow(this.toFlatRow({ question: s.q }, 1, null, options.query)) || discovered;
+          }
         }
       }
+      if (discovered) await this.emitProgress("discovered");
       if (options.questionsOnly && seenQs.size >= options.maxQuestions) break;
       const kept = new Set(orderedQs.slice(0, options.maxQuestions));
       const clickable = states.filter((s) => kept.has(s.q) && !clickedOnceEver.has(s.q) && !abandonedNeverReclick.has(s.q));
@@ -2579,6 +2785,7 @@ var PAAExtractor = class {
       this.reporter.onDepth(++round);
       await this.throwIfCaptcha(page, "Google PAA expansion");
       clickedOnceEver.add(target.q);
+      this.clickedQuestions.add(target.q);
       const expansionStatus = await expandOneItemSerially(target.q);
       if (!options.questionsOnly) {
         let clickedItem = (await this.extractVisibleItems(page, target.q))[0];
@@ -2586,7 +2793,12 @@ var PAAExtractor = class {
           await page.waitForTimeout(1300);
           clickedItem = (await this.extractVisibleItems(page, target.q))[0];
         }
-        if (clickedItem?.answer) capturedItems.set(target.q, clickedItem);
+        if (clickedItem) {
+          capturedItems.set(target.q, clickedItem);
+          if (this.upsertCollectedRow(this.toFlatRow(clickedItem, 1, null, options.query))) {
+            await this.emitProgress("answer_captured");
+          }
+        }
       }
       if (expansionStatus === "failed" && !capturedItems.has(target.q)) {
         abandonedNeverReclick.add(target.q);
@@ -2602,14 +2814,19 @@ var PAAExtractor = class {
           if (!seenQs.has(state.q)) {
             seenQs.add(state.q);
             orderedQs.push(state.q);
+            if (orderedQs.length <= options.maxQuestions) {
+              this.upsertCollectedRow(this.toFlatRow({ question: state.q }, 1, null, options.query));
+            }
           }
         }
+        await this.emitProgress("discovered");
       }
     }
     const itemMap = options.questionsOnly ? /* @__PURE__ */ new Map() : new Map(capturedItems);
     if (!options.questionsOnly) {
       await this.fillIncompleteItems(page, orderedQs, itemMap, options, abandonedNeverReclick);
     }
+    const results = [];
     for (const q of orderedQs) {
       if (results.length >= options.maxQuestions) break;
       const key = this.normalizeQuestion(q);
@@ -2617,14 +2834,22 @@ var PAAExtractor = class {
       seenKeys.add(key);
       const item = itemMap.get(q);
       if (item) {
-        results.push(this.toFlatRow(item, 1, null, options.query));
+        const row = this.toFlatRow(item, 1, null, options.query);
+        this.upsertCollectedRow(row);
+        results.push(row);
         this.reporter.onQuestion({ question: item.question, answer: item.answer ?? null, sourceTitle: item.sourceTitle ?? null, sourceSite: item.sourceSite ?? null, sourceCite: item.sourceCite ?? null, depth: 1, parentQuestion: null, children: [] });
       } else {
-        results.push(this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, 1, null, options.query));
+        const row = this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, 1, null, options.query);
+        this.upsertCollectedRow(row);
+        results.push(row);
       }
     }
     this.completeness.paaWithoutAnswer = options.questionsOnly ? 0 : results.filter((r) => !r.answer).length;
-    this.completeness.paaWithoutSource = options.questionsOnly ? 0 : results.filter((r) => !r.source_title && !r.source_site && !r.source_cite).length;
+    this.completeness.paaWithoutSource = options.questionsOnly ? 0 : results.filter((r) => !r.source_cite).length;
+    this.completeness.paaRequested = options.maxQuestions;
+    this.completeness.paaReturned = results.length;
+    this.completeness.paaUnique = seenKeys.size;
+    await this.emitProgress("fill_incomplete");
     return results;
   }
   async fillIncompleteItems(page, orderedQs, itemMap, options, abandonedNeverReclick) {
@@ -2649,6 +2874,10 @@ var PAAExtractor = class {
         sourceSite: item.sourceSite ?? existing?.sourceSite,
         sourceCite: item.sourceCite ?? existing?.sourceCite
       });
+      const upgraded = itemMap.get(q);
+      if (upgraded && this.upsertCollectedRow(this.toFlatRow(upgraded, 1, null, options.query))) {
+        await this.emitProgress("fill_incomplete");
+      }
     }
     this.completeness.paaAnswersRecovered = [...missingAnswersBefore].filter((q) => itemMap.get(q)?.answer).length;
   }
@@ -3113,7 +3342,15 @@ var PAAExtractor = class {
     this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
     this.aioShareUrlEarly = null;
     this.collectedRows = [];
-    this.partialContext = { seed: options.query, location: options.location ?? null, startMs };
+    this.progressSequence = 0;
+    this.clickedQuestions = /* @__PURE__ */ new Set();
+    this.partialContext = {
+      seed: options.query,
+      location: options.location ?? null,
+      startMs,
+      maxQuestions: options.maxQuestions,
+      questionsOnly: options.questionsOnly
+    };
     const isMobile = options.device === "mobile";
     const config = {
       headless: options.headless,
@@ -3185,6 +3422,7 @@ var PAAExtractor = class {
           extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
           diagnostics: {
             completionStatus: hasPaa ? "paa_found" : "no_paa",
+            ...!hasPaa ? { noPaaObserved: true } : {},
             problem: null,
             completeness: { ...this.completeness },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, void 0, locationResolution) } : {}
@@ -3304,6 +3542,7 @@ var PAAExtractor = class {
           extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
           diagnostics: {
             completionStatus: "no_paa",
+            noPaaObserved: true,
             problem: null,
             completeness: { ...this.completeness },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence2, locationResolution) } : {}
@@ -3395,6 +3634,8 @@ var PAAExtractor = class {
       };
     } catch (err) {
       errorCount++;
+      await this.emitProgress("attempt_ending").catch(() => {
+      });
       this.reporter.onError(err instanceof Error ? err : new Error(String(err)));
       throw err;
     }
@@ -4074,6 +4315,11 @@ function getAttemptLogSink(rawOptions) {
   const sink = rawOptions.onAttemptEvent;
   return typeof sink === "function" ? sink : void 0;
 }
+function getPaaProgressSink(rawOptions) {
+  if (!rawOptions || typeof rawOptions !== "object") return void 0;
+  const sink = rawOptions.onPaaProgress;
+  return typeof sink === "function" ? sink : void 0;
+}
 async function emitAttemptEvent(sink, event) {
   if (!sink) return;
   try {
@@ -4106,17 +4352,32 @@ function classifyAttemptResult(result) {
 function resultHasUsableContent(result) {
   return result.flat.length > 0 || result.organicResults.length > 0 || result.localPack.length > 0 || result.videos.length > 0 || result.forums.length > 0 || result.whatPeopleSaying.length > 0 || result.aiOverview.detected || result.aiMode.detected;
 }
-function withResultQuality(result) {
+function withResultQuality(result, requestedPaaCount, questionsOnly) {
   const hasUsableContent = resultHasUsableContent(result);
   const hasWarnings = (result.diagnostics.warnings?.length ?? 0) > 0;
-  const resultQuality = hasUsableContent ? hasWarnings ? "partial" : "complete" : "degraded";
+  const uniquePaaCount = new Set(result.flat.map((row) => row.question.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim())).size;
+  const paaWithoutAnswer = questionsOnly ? 0 : result.flat.filter((row) => !row.answer?.trim()).length;
+  const paaWithoutSource = questionsOnly ? 0 : result.flat.filter((row) => !row.source_cite?.trim()).length;
+  const noPaa = result.diagnostics.completionStatus === "no_paa" && (result.diagnostics.noPaaObserved === true || hasUsableContent);
+  const hasPaa = result.diagnostics.completionStatus === "paa_found";
+  const paaIncomplete = hasPaa && (uniquePaaCount < requestedPaaCount || paaWithoutAnswer > 0 || paaWithoutSource > 0 || Boolean(result.diagnostics.interruption));
+  const resultQuality = noPaa ? "complete" : hasPaa && uniquePaaCount > 0 ? paaIncomplete || hasWarnings ? "partial" : "complete" : hasUsableContent ? hasWarnings ? "partial" : "complete" : "degraded";
   const degradedResult = resultQuality === "degraded";
   result.diagnostics = {
     ...result.diagnostics,
     resultQuality,
     degradedResult,
     degradationReasons: degradedResult ? ["empty_primary_serp"] : [],
-    retryRecommended: degradedResult
+    retryRecommended: degradedResult || resultQuality === "partial" && (paaIncomplete || result.diagnostics.warnings?.some((warning) => warning.retryable) === true),
+    completeness: {
+      paaWithoutAnswer,
+      paaWithoutSource,
+      paaAnswersRecovered: result.diagnostics.completeness?.paaAnswersRecovered ?? 0,
+      aioShareCaptured: result.diagnostics.completeness?.aioShareCaptured ?? null,
+      paaRequested: requestedPaaCount,
+      paaReturned: result.flat.length,
+      paaUnique: uniquePaaCount
+    }
   };
   return result;
 }
@@ -4176,10 +4437,10 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId) {
     }));
   }
 }
-async function extractOnce(options, signal, forceManagedSerp = false) {
+async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress) {
   const driver = createSerpDriver(options, { forceManaged: forceManagedSerp });
   const reporter = new ProgressReporter();
-  const extractor = new PAAExtractor(driver, reporter);
+  const extractor = new PAAExtractor(driver, reporter, onPaaProgress);
   if (signal?.aborted) {
     return {
       result: null,
@@ -4231,17 +4492,25 @@ async function extractOnce(options, signal, forceManagedSerp = false) {
   }
   if (error) {
     const outcome = classifyAttemptError(error);
-    const budgetExhausted = outcome === "timeout" || outcome === "request_aborted";
+    const interruptionCode = outcome === "timeout" || outcome === "request_aborted" || outcome === "browser_session_interrupted" ? outcome : null;
     let salvaged = null;
-    if (budgetExhausted) {
+    if (interruptionCode) {
       try {
         salvaged = typeof extractor.getPartialResult === "function" ? extractor.getPartialResult() : null;
       } catch {
         salvaged = null;
       }
     }
-    if (salvaged) {
-      return { result: salvaged, error: null, cleanup, debug, salvagedFrom: error };
+    if (salvaged && interruptionCode) {
+      salvaged.diagnostics = {
+        ...salvaged.diagnostics,
+        interruption: {
+          code: interruptionCode,
+          reasonSource: "client_runtime",
+          message: errorMessage(error)
+        }
+      };
+      return { result: salvaged, error, cleanup, debug, salvagedFrom: error };
     }
     return { result: null, error, cleanup, debug };
   }
@@ -4251,6 +4520,7 @@ async function harvest(rawOptions) {
   const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};
   const signal = getAbortSignal(rawOptions);
   const onAttemptEvent = getAttemptLogSink(rawOptions);
+  const onPaaProgress = getPaaProgressSink(rawOptions);
   const forceManagedSerp = raw.forceManagedSerp === true;
   const requestedProxyMode = raw.proxyMode;
   const proxyMode = requestedProxyMode === "location" || requestedProxyMode === "none" || requestedProxyMode === "configured" ? requestedProxyMode : DEFAULT_PROXY_MODE;
@@ -4320,7 +4590,7 @@ async function harvest(rawOptions) {
         ...baseCtx,
         forceHeadless: allowHeadlessTest,
         forceHeadful: !allowHeadlessTest
-      }, () => extractOnce(attemptOptions, signal, forceManagedSerp)) : await extractOnce(attemptOptions, signal, forceManagedSerp);
+      }, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress);
       if (attempt.error) {
         const err = attempt.error;
         const outcome2 = classifyAttemptError(err);
@@ -4353,7 +4623,7 @@ async function harvest(rawOptions) {
           maxAttempts,
           outcome: outcome2,
           kernelSessionId: attempt.cleanup.kernelSessionId,
-          questionCount: 0,
+          questionCount: attempt.result?.totalQuestions ?? 0,
           durationMs: Date.now() - startedAtMs,
           error: errorMessage(err),
           willRetry: willRetry2,
@@ -4364,6 +4634,9 @@ async function harvest(rawOptions) {
         await cleanupDisposableProxy(kernelApiKey, resolution2.disposableProxyId);
         lastError = err;
         if (willRetry2) continue;
+        if (attempt.result) {
+          return withResultQuality(stripInternalDebug(attempt.result, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
+        }
         break;
       }
       const result = attempt.result;
@@ -4397,7 +4670,7 @@ async function harvest(rawOptions) {
         if (willRetry2) continue;
         break;
       }
-      const finalResult = withResultQuality(stripInternalDebug(result, requestedDebug));
+      const finalResult = withResultQuality(stripInternalDebug(result, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
       const outcome = classifyAttemptResult(finalResult);
       const willRetry = outcome === "degraded_result" && i < maxAttempts - 1;
       const resultError = outcome === "degraded_result" ? degradedResultMessage(finalResult) : null;