mcp-scraper 0.3.27 → 0.3.40

package/dist/bin/mcp-stdio-server.js

@@ -3,14 +3,15 @@ import {
   HttpMcpToolExecutor,
   SERVER_INSTRUCTIONS,
   registerBrowserAgentMcpTools,
-  registerPaaExtractorMcpTools
-} from "../chunk-6KHGYJIT.js";
+  registerPaaExtractorMcpTools,
+  registerSerpIntelligenceCaptureTools
+} from "../chunk-QLQYNE7E.js";
 import {
   renderInstallTerminal
 } from "../chunk-T3VZD2I4.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-DYV72F6A.js";
+} from "../chunk-I5EFBTYE.js";
 import "../chunk-3PRO376E.js";
 import "../chunk-BP27CZ5Q.js";
 
@@ -52,7 +53,9 @@ if (!apiKey) {
 var baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || "https://mcpscraper.dev";
 var consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
 var server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS });
-registerPaaExtractorMcpTools(server, new HttpMcpToolExecutor(baseUrl, apiKey));
+var httpExecutor = new HttpMcpToolExecutor(baseUrl, apiKey);
+registerPaaExtractorMcpTools(server, httpExecutor);
+registerSerpIntelligenceCaptureTools(server, httpExecutor);
 registerBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl });
 var transport = new StdioServerTransport();
 async function main() {

package/dist/bin/mcp-stdio-server.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../bin/mcp-stdio-server.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools } from '../src/mcp/paa-mcp-server.js'\nimport { SERVER_INSTRUCTIONS } from '../src/mcp/server-instructions.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst forceStdio =\n  process.argv.includes('--stdio') ||\n  process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n  const noColor =\n    process.argv.includes('--no-color') ||\n    process.env.NO_COLOR !== undefined ||\n    process.env.FORCE_COLOR === '0' ||\n    !process.stdout.isTTY\n\n  process.stdout.write(renderInstallTerminal({\n    version: PACKAGE_VERSION,\n    color: !noColor,\n    apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n  }))\n  process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n  const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n  for (const path of paths) {\n    try {\n      const value = readFileSync(path, 'utf8').trim()\n      if (value) return value\n    } catch {\n      void 0\n    }\n  }\n  return undefined\n}\n\nconst apiKey = (\n  process.env.MCP_SCRAPER_API_KEY ??\n  process.env.MCP_SCRAPER_KEY ??\n  process.env.MCP_API_KEY ??\n  readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n  process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n  process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS })\n\nregisterPaaExtractorMcpTools(server, new HttpMcpToolExecutor(baseUrl, apiKey))\nregisterBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl })\n\nconst transport = new StdioServerTransport()\n\nasync function main() {\n  await server.connect(transport)\n}\n\nmain().catch(err => {\n  process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n  process.exit(1)\n})\n"],"mappings":";;;;;;;;;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAC1B,SAAS,4BAA4B;AAQrC,IAAM,aACJ,QAAQ,KAAK,SAAS,SAAS,KAC/B,QAAQ,IAAI,4BAA4B;AAC1C,IAAM,sBAAsB,QAAQ,QAAQ,MAAM,SAAS,QAAQ,OAAO,KAAK;AAC/E,IAAM,YAAY,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAE/E,IAAI,CAAC,eAAe,uBAAuB,YAAY;AACrD,QAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,UAAQ,OAAO,MAAM,sBAAsB;AAAA,IACzC,SAAS;AAAA,IACT,OAAO,CAAC;AAAA,IACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAAA,EACnE,CAAC,CAAC;AACF,UAAQ,KAAK,CAAC;AAChB;AAEA,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,IAAI,UAAU,EAAE,MAAM,eAAe,SAAS,gBAAgB,GAAG,EAAE,cAAc,oBAAoB,CAAC;AAErH,6BAA6B,QAAQ,IAAI,oBAAoB,SAAS,MAAM,CAAC;AAC7E,6BAA6B,QAAQ,EAAE,SAAS,QAAQ,eAAe,CAAC;AAExE,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
+{"version":3,"sources":["../../bin/mcp-stdio-server.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools, registerSerpIntelligenceCaptureTools } from '../src/mcp/paa-mcp-server.js'\nimport { SERVER_INSTRUCTIONS } from '../src/mcp/server-instructions.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst forceStdio =\n  process.argv.includes('--stdio') ||\n  process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n  const noColor =\n    process.argv.includes('--no-color') ||\n    process.env.NO_COLOR !== undefined ||\n    process.env.FORCE_COLOR === '0' ||\n    !process.stdout.isTTY\n\n  process.stdout.write(renderInstallTerminal({\n    version: PACKAGE_VERSION,\n    color: !noColor,\n    apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n  }))\n  process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n  const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n  for (const path of paths) {\n    try {\n      const value = readFileSync(path, 'utf8').trim()\n      if (value) return value\n    } catch {\n      void 0\n    }\n  }\n  return undefined\n}\n\nconst apiKey = (\n  process.env.MCP_SCRAPER_API_KEY ??\n  process.env.MCP_SCRAPER_KEY ??\n  process.env.MCP_API_KEY ??\n  readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n  process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n  process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS })\n\nconst httpExecutor = new HttpMcpToolExecutor(baseUrl, apiKey)\nregisterPaaExtractorMcpTools(server, httpExecutor)\nregisterSerpIntelligenceCaptureTools(server, httpExecutor)\nregisterBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl })\n\nconst transport = new StdioServerTransport()\n\nasync function main() {\n  await server.connect(transport)\n}\n\nmain().catch(err => {\n  process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n  process.exit(1)\n})\n"],"mappings":";;;;;;;;;;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAC1B,SAAS,4BAA4B;AAQrC,IAAM,aACJ,QAAQ,KAAK,SAAS,SAAS,KAC/B,QAAQ,IAAI,4BAA4B;AAC1C,IAAM,sBAAsB,QAAQ,QAAQ,MAAM,SAAS,QAAQ,OAAO,KAAK;AAC/E,IAAM,YAAY,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAE/E,IAAI,CAAC,eAAe,uBAAuB,YAAY;AACrD,QAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,UAAQ,OAAO,MAAM,sBAAsB;AAAA,IACzC,SAAS;AAAA,IACT,OAAO,CAAC;AAAA,IACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAAA,EACnE,CAAC,CAAC;AACF,UAAQ,KAAK,CAAC;AAChB;AAEA,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,IAAI,UAAU,EAAE,MAAM,eAAe,SAAS,gBAAgB,GAAG,EAAE,cAAc,oBAAoB,CAAC;AAErH,IAAM,eAAe,IAAI,oBAAoB,SAAS,MAAM;AAC5D,6BAA6B,QAAQ,YAAY;AACjD,qCAAqC,QAAQ,YAAY;AACzD,6BAA6B,QAAQ,EAAE,SAAS,QAAQ,eAAe,CAAC;AAExE,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}

package/dist/bin/paa-harvest.cjs

@@ -1587,6 +1587,13 @@ var PAAExtractor = class {
   }
   driver;
   reporter;
+  completeness = {
+    paaWithoutAnswer: 0,
+    paaWithoutSource: 0,
+    paaAnswersRecovered: 0,
+    aioShareCaptured: null
+  };
+  aioShareUrlEarly = null;
   normalizeQuestion(q) {
     return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
   }
@@ -1715,6 +1722,7 @@ var PAAExtractor = class {
       }
     }
     const itemMap = new Map((await this.extractVisibleItems(page)).map((i) => [i.question, i]));
+    await this.fillIncompleteItems(page, orderedQs, itemMap, options);
     for (const q of orderedQs) {
       if (results.length >= options.maxQuestions) break;
       const key = this.normalizeQuestion(q);
@@ -1728,8 +1736,52 @@ var PAAExtractor = class {
         results.push(this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, 1, null, options.query));
       }
     }
+    this.completeness.paaWithoutAnswer = results.filter((r) => !r.answer).length;
+    this.completeness.paaWithoutSource = results.filter((r) => !r.source_title && !r.source_site && !r.source_cite).length;
     return results;
   }
+  async fillIncompleteItems(page, orderedQs, itemMap, options) {
+    const kept = new Set(orderedQs.slice(0, options.maxQuestions));
+    const missingBefore = new Set([...kept].filter((q) => !itemMap.get(q)?.answer));
+    if (missingBefore.size === 0) return;
+    for (let round = 0; round < 2; round++) {
+      if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
+      const stillMissing = [...missingBefore].filter((q) => !itemMap.get(q)?.answer);
+      if (stillMissing.length === 0) break;
+      const reExpanded = await this.reExpandCollapsedItems(page).catch(() => false);
+      await page.waitForTimeout(reExpanded ? 900 : 500);
+      let improved = false;
+      for (const item of await this.extractVisibleItems(page)) {
+        if (!kept.has(item.question)) continue;
+        const existing = itemMap.get(item.question);
+        if (!item.answer || existing?.answer) continue;
+        itemMap.set(item.question, {
+          ...item,
+          sourceTitle: item.sourceTitle ?? existing?.sourceTitle,
+          sourceSite: item.sourceSite ?? existing?.sourceSite,
+          sourceCite: item.sourceCite ?? existing?.sourceCite
+        });
+        improved = true;
+      }
+      if (!improved) break;
+    }
+    this.completeness.paaAnswersRecovered = [...missingBefore].filter((q) => itemMap.get(q)?.answer).length;
+  }
+  async reExpandCollapsedItems(page) {
+    const unexpandedSel = `${PAASelectors.item}:not(.${PAASelectors.expandedClass}) ${PAASelectors.clickTarget}`;
+    const count = await page.locator(unexpandedSel).count().catch(() => 0);
+    if (count === 0) return false;
+    for (let i = 0; i < count; i++) {
+      try {
+        const btn = page.locator(unexpandedSel).first();
+        await btn.scrollIntoViewIfNeeded();
+        await btn.click({ force: true });
+        await page.waitForTimeout(300);
+      } catch {
+      }
+    }
+    return true;
+  }
   async extractVideos(page) {
     const vsels = VideoSelectors;
     return page.evaluate((sels) => {
@@ -2064,12 +2116,58 @@ var PAAExtractor = class {
     }
     return { ...entityIds, entities: records, cids: [...cidSet] };
   }
-  async extractAISurfaces(page) {
-    return page.evaluate(extractAISurfacesFromDocument, {
+  async extractAISurfaces(page, options) {
+    const surfaces = await page.evaluate(extractAISurfacesFromDocument, {
       aio: AIOverviewSelectors,
       aim: AIModeSelectors,
       expandWaitMs: 1500
     });
+    if (!surfaces.aiOverview.detected) {
+      this.completeness.aioShareCaptured = null;
+      return surfaces;
+    }
+    const shareUrl = this.aioShareUrlEarly ?? await this.captureAIOverviewShareUrl(page, options).catch(() => null);
+    this.completeness.aioShareCaptured = shareUrl !== null;
+    return {
+      ...surfaces,
+      aiOverview: { ...surfaces.aiOverview, shareUrl }
+    };
+  }
+  async captureAIOverviewShareUrl(page, options) {
+    for (let attempt = 0; attempt < 3; attempt++) {
+      if (options?.softDeadlineMs && Date.now() >= options.softDeadlineMs) return null;
+      const clicked = await page.evaluate(() => {
+        const heads = Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"],div')).filter((h) => (h.textContent || "").trim() === "AI Overview");
+        for (const head of heads) {
+          let sec = head;
+          for (let up = 0; up < 8 && sec; up++) {
+            const btn = sec.querySelector('[aria-label="Share"]');
+            if (btn) {
+              btn.scrollIntoView({ block: "center" });
+              btn.click();
+              return true;
+            }
+            sec = sec.parentElement;
+          }
+        }
+        return false;
+      }).catch(() => false);
+      if (!clicked) return null;
+      await page.waitForTimeout(4500);
+      const url = await page.evaluate(() => {
+        const w = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
+        while (w.nextNode()) {
+          const m = (w.currentNode.textContent || "").match(/https:\/\/share\.google\/aimode\/\S+/);
+          if (m) return m[0];
+        }
+        return null;
+      }).catch(() => null);
+      await page.keyboard.press("Escape").catch(() => {
+      });
+      if (url) return url;
+      await page.waitForTimeout(600);
+    }
+    return null;
   }
   buildTree(flat, _seed) {
     const roots = [];
@@ -2122,6 +2220,8 @@ var PAAExtractor = class {
   }
   async extract(options, signal) {
     const startMs = Date.now();
+    this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
+    this.aioShareUrlEarly = null;
     const isMobile = options.device === "mobile";
     const config = {
       headless: options.headless,
@@ -2160,6 +2260,7 @@ var PAAExtractor = class {
       this.throwIfAborted(signal);
       const page = this.driver.getPage();
       await this.throwIfCaptcha(page, "Google SERP");
+      this.aioShareUrlEarly = await this.captureAIOverviewShareUrl(page, options).catch(() => null);
       if (options.serpOnly) {
         const [organicResults2, localPack2, rawEntityIds2] = await Promise.all([
           this.extractOrganicResults(page),
@@ -2167,7 +2268,7 @@ var PAAExtractor = class {
           this.extractEntityIds(page)
         ]);
         const entityIds2 = this.mergeLocalPackIntoEntities(rawEntityIds2, localPack2);
-        const aiSurfaces2 = await this.extractAISurfaces(page);
+        const aiSurfaces2 = await this.extractAISurfaces(page, options);
         let locationEvidence2 = options.debug ? inferSerpLocationEvidence(canonicalLocation, organicResults2, localPack2) : void 0;
         let allOrganic2 = organicResults2;
         if ((options.pages ?? 1) >= 2) {
@@ -2196,6 +2297,7 @@ var PAAExtractor = class {
           diagnostics: {
             completionStatus: "serp_only",
             problem: null,
+            completeness: { ...this.completeness },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence2) } : {}
           },
           totalQuestions: 0,
@@ -2239,7 +2341,7 @@ var PAAExtractor = class {
             locationEvidence2 = inferSerpLocationEvidence(canonicalLocation, noPaaOrganic, localPack);
           }
         }
-        const aiSurfaces2 = await this.extractAISurfaces(page);
+        const aiSurfaces2 = await this.extractAISurfaces(page, options);
         const stats2 = {
           seed: options.query,
           totalQuestions: 0,
@@ -2255,6 +2357,7 @@ var PAAExtractor = class {
           diagnostics: {
             completionStatus: "no_paa",
             problem: null,
+            completeness: { ...this.completeness },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence2) } : {}
           },
           totalQuestions: 0,
@@ -2274,7 +2377,7 @@ var PAAExtractor = class {
       }
       const flat = await this.runBFS(page, options, signal);
       this.throwIfAborted(signal);
-      const aiSurfaces = await this.extractAISurfaces(page);
+      const aiSurfaces = await this.extractAISurfaces(page, options);
       const shortVidsParams = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", udm: ShortVideoSelectors.udm });
       if (uule) shortVidsParams.set("uule", uule);
       let shortVideos = [];
@@ -2321,6 +2424,7 @@ var PAAExtractor = class {
         diagnostics: {
           completionStatus: "paa_found",
           problem: null,
+          completeness: { ...this.completeness },
           ...diagnosticWarnings.length > 0 ? { warnings: diagnosticWarnings } : {},
           ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence) } : {}
         },