mcp-scraper 0.1.0

package/dist/bin/paa-harvest.cjs

@@ -0,0 +1,1438 @@
+#!/usr/bin/env node
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+
+// src/cli.ts
+var import_commander = require("commander");
+
+// src/schemas.ts
+var import_zod = require("zod");
+var HarvestOptionsSchema = import_zod.z.object({
+  query: import_zod.z.string().min(1),
+  location: import_zod.z.string().optional(),
+  gl: import_zod.z.string().length(2).default("us"),
+  hl: import_zod.z.string().length(2).default("en"),
+  depth: import_zod.z.number().int().min(1).max(30).default(3),
+  maxQuestions: import_zod.z.number().int().min(1).max(1e3).default(100),
+  headless: import_zod.z.boolean().default(false),
+  profileDir: import_zod.z.string().optional(),
+  proxy: import_zod.z.string().url().optional(),
+  kernelApiKey: import_zod.z.string().optional(),
+  kernelProxyId: import_zod.z.string().optional(),
+  outputDir: import_zod.z.string().default("./paa-output"),
+  format: import_zod.z.enum(["json", "csv", "both"]).default("both"),
+  serpOnly: import_zod.z.boolean().default(false),
+  pages: import_zod.z.number().int().min(1).max(2).default(1)
+});
+var MapsPlaceOptionsSchema = import_zod.z.object({
+  businessName: import_zod.z.string().min(1),
+  location: import_zod.z.string().min(1),
+  gl: import_zod.z.string().length(2).default("us"),
+  hl: import_zod.z.string().length(2).default("en"),
+  includeReviews: import_zod.z.boolean().default(false),
+  maxReviews: import_zod.z.number().int().min(1).max(500).default(50),
+  kernelApiKey: import_zod.z.string().optional(),
+  kernelProxyId: import_zod.z.string().optional(),
+  headless: import_zod.z.boolean().default(true)
+});
+var RawPAAItemSchema = import_zod.z.object({
+  question: import_zod.z.string().min(1),
+  answer: import_zod.z.string().optional(),
+  sourceTitle: import_zod.z.string().optional(),
+  sourceSite: import_zod.z.string().optional(),
+  sourceCite: import_zod.z.string().optional()
+});
+
+// src/driver/BrowserDriver.ts
+var import_playwright_extra = require("playwright-extra");
+var import_puppeteer_extra_plugin_stealth = __toESM(require("puppeteer-extra-plugin-stealth"), 1);
+var import_playwright = require("playwright");
+var import_sdk = __toESM(require("@onkernel/sdk"), 1);
+
+// src/selectors.ts
+var PAASelectors = {
+  container: ".eJH8qe.adDDi",
+  dataInitq: "[data-initq]",
+  item: ".related-question-pair",
+  itemDataQ: "data-q",
+  itemDataInitQ: "data-initq",
+  itemQuestionEl: ".JlqpRe",
+  answerContainer: ".bCOlv",
+  sourceTitle: "h3",
+  sourceSite: ".VuuXrf",
+  sourceCite: "cite",
+  clickTarget: ".JlqpRe",
+  expandedClass: "aoRk1c",
+  captchaMarker: '#captcha-form, #recaptcha, form[action*="/sorry/"], .g-recaptcha, [data-sitekey]'
+};
+var VideoSelectors = {
+  container: 'div[jscontroller="HWk0Gf"]',
+  sectionHeading: '.mgAbYb[role="heading"]',
+  item: "a.rIRoqf"
+};
+var ShortVideoSelectors = {
+  udm: "39",
+  item: "a.rIRoqf",
+  durationPattern: /^\d+:\d+$/,
+  platforms: ["YouTube", "TikTok", "Instagram", "Facebook", "X"]
+};
+var ForumSelectors = {
+  section: ".ULSxyf",
+  item: "a.KYg7td.INpicf",
+  title: ".hyYc0c",
+  source: ".K4ETW"
+};
+var WhatPeopleSayingSelectors = {
+  sectionTag: "g-section-with-header",
+  sectionHeadingText: "What people are saying",
+  card: '.dRzkFf[role="listitem"]',
+  cardLink: 'a.WlydOe[jsname="YKoRaf"]',
+  titleH1: "h1.WQWxe",
+  titleDiv: ".eAaXgc",
+  popularCommentLabel: ".qgdis",
+  source: ".sTl1Td",
+  platformBadge: ".appd0, .KrMNbf",
+  ytChannel: ".sjVJQd",
+  ytDate: ".PLq9Je",
+  authorNote: ".nDgy9d"
+};
+var AIOverviewSelectors = {
+  root: '[data-hveid="CBMQAA"]',
+  wrapper: ".Fgyi2e",
+  citations: '.Fgyi2e [data-hveid] a[jsname="pxBnId"]'
+};
+var AIModeSelectors = {
+  root: '[data-hveid="CAUQAA"]',
+  wrapper: ".Fgyi2e",
+  citations: '.Fgyi2e [data-hveid] a[jsname="pxBnId"]'
+};
+var OrganicSelectors = {
+  result: ".wHYlTd.tF2Cxc",
+  title: "h3.LC20lb",
+  siteName: ".VuuXrf",
+  cite: "cite.tjvcx",
+  snippet: ".VwiC3b",
+  redditCite: "cite.qLRx3b",
+  ratingWrap: ".Y0A0hc",
+  ratingValue: ".yi40Hd",
+  reviewCount: ".RDApEe"
+};
+var LocalPackSelectors = {
+  headingText: "Businesses",
+  card: ".w7Dbne",
+  name: ".OSrXXb",
+  ratingValue: ".yi40Hd",
+  reviewCount: ".RDApEe"
+};
+
+// src/errors.ts
+var RECAPTCHA_INSTRUCTIONS = "Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.";
+var CaptchaError = class extends Error {
+  constructor(instructions) {
+    super(`CAPTCHA detected. ${instructions}`);
+    this.instructions = instructions;
+  }
+  instructions;
+  name = "CaptchaError";
+};
+var ExtractionError = class extends Error {
+  constructor(message, cause) {
+    super(message);
+    this.cause = cause;
+  }
+  cause;
+  name = "ExtractionError";
+};
+
+// src/driver/BrowserDriver.ts
+import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
+var DESKTOP_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
+function buildYouTubeChannelVideosUrl(channelInput) {
+  const raw = channelInput.trim();
+  if (!raw) throw new Error("channelHandle is required");
+  const urlLike = /^https?:\/\//i.test(raw) || /^(www\.|m\.)?youtube\.com\//i.test(raw);
+  if (urlLike) {
+    const parsed = new URL(/^https?:\/\//i.test(raw) ? raw : `https://${raw}`);
+    const host = parsed.hostname.replace(/^www\./, "").replace(/^m\./, "").toLowerCase();
+    if (host !== "youtube.com") throw new Error("channel URL must be on youtube.com");
+    const segments = parsed.pathname.split("/").filter(Boolean);
+    const first = segments[0] ?? "";
+    const second = segments[1] ?? "";
+    if (first.startsWith("@")) return `https://www.youtube.com/${first}/videos`;
+    if (first === "channel" && second) return `https://www.youtube.com/channel/${second}/videos`;
+    if ((first === "c" || first === "user") && second) return `https://www.youtube.com/${first}/${second}/videos`;
+    throw new Error("channel URL must be a YouTube handle, /channel/UC..., /c/..., or /user/... URL");
+  }
+  const stripped = raw.replace(/^\/+/, "").replace(/\/+$/, "");
+  const withoutVideos = stripped.replace(/\/videos$/i, "");
+  if (/^UC[\w-]{20,}$/.test(withoutVideos)) {
+    return `https://www.youtube.com/channel/${withoutVideos}/videos`;
+  }
+  const handle = withoutVideos.startsWith("@") ? withoutVideos : `@${withoutVideos}`;
+  if (!/^@[\w.-]+$/.test(handle)) {
+    throw new Error("channelHandle must be an @handle, UC channel ID, or YouTube channel URL");
+  }
+  return `https://www.youtube.com/${handle}/videos`;
+}
+var BrowserDriver = class {
+  browser = null;
+  context = null;
+  page = null;
+  kernelClient = null;
+  kernelSessionId = null;
+  async launch(config) {
+    if (config.kernelApiKey) {
+      this.kernelClient = new import_sdk.default({ apiKey: config.kernelApiKey });
+      const kernelBrowser = await this.kernelClient.browsers.create({
+        stealth: true,
+        timeout_seconds: 600,
+        ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {}
+      });
+      this.kernelSessionId = kernelBrowser.session_id;
+      this.browser = await import_playwright.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
+      this.context = this.browser.contexts()[0] ?? await this.browser.newContext();
+      await this.installEsbuildHelperShims(this.context);
+      this.page = this.context.pages()[0] ?? await this.context.newPage();
+      return;
+    }
+    const launchOpts = {
+      headless: config.headless,
+      proxy: config.proxy ? { server: config.proxy } : void 0
+    };
+    const ctxOpts = {
+      viewport: config.viewport,
+      locale: config.locale,
+      userAgent: DESKTOP_USER_AGENT
+    };
+    if (config.profileDir) {
+      this.context = await import_playwright_extra.chromium.launchPersistentContext(config.profileDir, {
+        ...launchOpts,
+        ...ctxOpts
+      });
+      await this.installEsbuildHelperShims(this.context);
+      this.page = await this.context.newPage();
+    } else {
+      this.browser = await import_playwright_extra.chromium.launch(launchOpts);
+      this.context = await this.browser.newContext(ctxOpts);
+      await this.installEsbuildHelperShims(this.context);
+      this.page = await this.context.newPage();
+    }
+  }
+  async installEsbuildHelperShims(context) {
+    await context.addInitScript(() => {
+      const g = globalThis;
+      if (typeof g.__name !== "function") g.__name = (fn) => fn;
+      if (typeof g.__publicField !== "function") g.__publicField = (obj, key, value) => {
+        obj[key] = value;
+        return value;
+      };
+    });
+  }
+  async navigateToSERP(query, uule, gl, hl) {
+    const params = new URLSearchParams({ q: query, gl, hl });
+    if (uule) params.set("uule", uule);
+    const url = "https://www.google.com/search?" + params.toString();
+    try {
+      await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+    } catch (err) {
+      const diag = await this.captureDiagnostics(url);
+      throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
+    }
+    const captchaCount = await this.page.locator(PAASelectors.captchaMarker).count();
+    if (captchaCount > 0) {
+      if (this.kernelClient) {
+        try {
+          await this.page.waitForSelector(PAASelectors.container, { timeout: 45e3 });
+          return { hasPaa: true };
+        } catch {
+          throw new CaptchaError(this.captchaMessage());
+        }
+      }
+      throw new CaptchaError(this.captchaMessage());
+    }
+    const fastFound = await this.page.waitForSelector(PAASelectors.item, { timeout: 4e3 }).catch(() => null);
+    if (fastFound) return { hasPaa: true };
+    const captchaAfter = await this.page.locator(PAASelectors.captchaMarker).count();
+    if (captchaAfter > 0) throw new CaptchaError(this.captchaMessage());
+    for (let i = 1; i <= 6; i++) {
+      await this.page.evaluate((f) => {
+        window.scrollTo(0, document.body.scrollHeight * f);
+      }, i / 6);
+      await this.page.waitForTimeout(600);
+      const count = await this.page.locator(PAASelectors.item).count();
+      if (count > 0) return { hasPaa: true };
+    }
+    return { hasPaa: false };
+  }
+  async captureDiagnostics(intendedUrl) {
+    try {
+      const finalUrl = this.page.url();
+      const title = await this.page.title().catch(() => "");
+      const bodySnippet = await this.page.evaluate(() => {
+        const t = (document.body?.innerText ?? "").replace(/\s+/g, " ").trim();
+        return t.slice(0, 400);
+      }).catch(() => "");
+      const consent = /consent\.google\./.test(finalUrl) || /before you continue/i.test(bodySnippet);
+      const recaptcha = /recaptcha|unusual traffic|are you a robot/i.test(bodySnippet);
+      const flags = [
+        consent ? "CONSENT_WALL" : "",
+        recaptcha ? "BOT_CHALLENGE" : "",
+        finalUrl !== intendedUrl ? "REDIRECTED" : ""
+      ].filter(Boolean).join(",");
+      return `intended=${intendedUrl} | final=${finalUrl} | title="${title}" | flags=[${flags}] | body="${bodySnippet}"`;
+    } catch (e) {
+      return `diagnostics-failed: ${e.message}`;
+    }
+  }
+  captchaMessage() {
+    return this.kernelClient ? "Google returned a CAPTCHA on this Kernel.sh session \u2014 retrying with a fresh session." : RECAPTCHA_INSTRUCTIONS;
+  }
+  async navigateTo(url) {
+    try {
+      await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+    } catch (err) {
+      const diag = await this.captureDiagnostics(url);
+      throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
+    }
+  }
+  async navigateToChannel(channelHandle) {
+    const url = buildYouTubeChannelVideosUrl(channelHandle);
+    try {
+      await this.page.goto(url, { waitUntil: "networkidle", timeout: 3e4 });
+    } catch (err) {
+      const diag = await this.captureDiagnostics(url);
+      throw new ExtractionError(`navigateToChannel failed: ${err.message} | ${diag}`);
+    }
+  }
+  async evaluate(fn, arg) {
+    return this.page.evaluate(fn, arg);
+  }
+  getPage() {
+    return this.page;
+  }
+  async close() {
+    if (this.browser) {
+      const b = this.browser;
+      const sessionId = this.kernelSessionId;
+      const client = this.kernelClient;
+      this.browser = null;
+      this.context = null;
+      this.page = null;
+      this.kernelSessionId = null;
+      this.kernelClient = null;
+      try {
+        await b.close();
+      } finally {
+        if (client && sessionId) {
+          await client.browsers.deleteByID(sessionId).catch(
+            (err) => console.warn("Kernel session cleanup failed:", err)
+          );
+        }
+      }
+    } else if (this.context) {
+      const ctx = this.context;
+      this.context = null;
+      this.page = null;
+      await ctx.close();
+    }
+  }
+};
+
+// src/locations.ts
+var LOCATIONS = {
+  "austin": "Austin,Texas,United States",
+  "new york": "New York,New York,United States",
+  "new york city": "New York,New York,United States",
+  "nyc": "New York,New York,United States",
+  "los angeles": "Los Angeles,California,United States",
+  "la": "Los Angeles,California,United States",
+  "chicago": "Chicago,Illinois,United States",
+  "houston": "Houston,Texas,United States",
+  "phoenix": "Phoenix,Arizona,United States",
+  "philadelphia": "Philadelphia,Pennsylvania,United States",
+  "philly": "Philadelphia,Pennsylvania,United States",
+  "san antonio": "San Antonio,Texas,United States",
+  "dallas": "Dallas,Texas,United States",
+  "miami": "Miami,Florida,United States",
+  "seattle": "Seattle,Washington,United States",
+  "denver": "Denver,Colorado,United States",
+  "loveland": "Loveland,Colorado,United States",
+  "loveland co": "Loveland,Colorado,United States",
+  "fort collins": "Fort Collins,Colorado,United States",
+  "boulder": "Boulder,Colorado,United States",
+  "colorado springs": "Colorado Springs,Colorado,United States",
+  "boston": "Boston,Massachusetts,United States",
+  "atlanta": "Atlanta,Georgia,United States",
+  "san francisco": "San Francisco,California,United States",
+  "sf": "San Francisco,California,United States",
+  "portland": "Portland,Oregon,United States",
+  "las vegas": "Las Vegas,Nevada,United States",
+  "minneapolis": "Minneapolis,Minnesota,United States",
+  "detroit": "Detroit,Michigan,United States",
+  "nashville": "Nashville,Tennessee,United States",
+  "charlotte": "Charlotte,North Carolina,United States",
+  "orlando": "Orlando,Florida,United States",
+  "san diego": "San Diego,California,United States",
+  "baltimore": "Baltimore,Maryland,United States",
+  "sacramento": "Sacramento,California,United States",
+  "columbus": "Columbus,Ohio,United States",
+  "indianapolis": "Indianapolis,Indiana,United States",
+  "san jose": "San Jose,California,United States",
+  "fort worth": "Fort Worth,Texas,United States",
+  "jacksonville": "Jacksonville,Florida,United States",
+  "memphis": "Memphis,Tennessee,United States",
+  "louisville": "Louisville,Kentucky,United States",
+  "raleigh": "Raleigh,North Carolina,United States",
+  "richmond": "Richmond,Virginia,United States",
+  "salt lake city": "Salt Lake City,Utah,United States",
+  "toronto": "Toronto,Ontario,Canada",
+  "vancouver": "Vancouver,British Columbia,Canada",
+  "montreal": "Montreal,Quebec,Canada",
+  "calgary": "Calgary,Alberta,Canada",
+  "ottawa": "Ottawa,Ontario,Canada",
+  "london": "London,England,United Kingdom",
+  "manchester": "Manchester,England,United Kingdom",
+  "birmingham": "Birmingham,England,United Kingdom",
+  "edinburgh": "Edinburgh,Scotland,United Kingdom",
+  "glasgow": "Glasgow,Scotland,United Kingdom",
+  "leeds": "Leeds,England,United Kingdom",
+  "sydney": "Sydney,New South Wales,Australia",
+  "melbourne": "Melbourne,Victoria,Australia",
+  "brisbane": "Brisbane,Queensland,Australia",
+  "perth": "Perth,Western Australia,Australia",
+  "adelaide": "Adelaide,South Australia,Australia",
+  "dublin": "Dublin,Leinster,Ireland"
+};
+
+// src/uule.ts
+function encodeUule(name) {
+  const encoded = Buffer.from(String.fromCharCode(name.length) + name).toString("base64");
+  return `w+CAIQICI${encoded}`;
+}
+function normalizeLocation(input) {
+  const key = input.toLowerCase().trim();
+  return LOCATIONS[key] ?? input;
+}
+
+// src/lib/paa-answer-cleanup.ts
+var MAX_ANSWER_LENGTH = 1200;
+var BOILERPLATE_PATTERNS = [
+  /An AI Overview is not available for this search/gi,
+  /Can't generate an AI overview right now\.?\s*Try again later\.?/gi,
+  /\bAI Overview\b/gi,
+  /\bView all\b/gi
+];
+var CUT_MARKERS = [
+  /\bRelated Links\b/i,
+  /\bAsk anything in\s*AI Mode\b/i,
+  /\bAI can make mistakes\b/i,
+  /\bThis is for informational purposes only\b/i,
+  /\bShow more\b/i,
+  /\b\d+\s+sites\b/i,
+  /\b\d{1,2}\s*[msh]\s*[A-Z][A-Za-z]/,
+  /\b(?:YouTube|Reddit|Facebook|Instagram|TikTok)·/
+];
+function normalizeWhitespace(text) {
+  return text.replace(/\u00a0/g, " ").replace(/([.!?])([A-Z])/g, "$1 $2").replace(/([:;])([A-Z])/g, "$1 $2").replace(/([a-z])([A-Z][a-z])/g, "$1 $2").replace(/(\d)([A-Z][a-z])/g, "$1 $2").replace(/([a-z])(\d)/g, "$1 $2").replace(/\s+/g, " ").trim();
+}
+function cutAtFirstMarker(text) {
+  let cutAt = -1;
+  for (const marker of CUT_MARKERS) {
+    const match = marker.exec(text);
+    marker.lastIndex = 0;
+    if (match && (cutAt === -1 || match.index < cutAt)) cutAt = match.index;
+  }
+  return cutAt === -1 ? text : text.slice(0, cutAt);
+}
+function cutAtSourceTitle(text, sourceTitle) {
+  const title = sourceTitle?.trim();
+  if (!title || title.length < 8) return text;
+  const idx = text.toLowerCase().indexOf(title.toLowerCase());
+  return idx > 40 ? text.slice(0, idx) : text;
+}
+function findAttributionCut(beforeUrl) {
+  const dateMatch = beforeUrl.match(/[•·]\s*(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},\s+\d{4}/i);
+  if (dateMatch?.index && dateMatch.index > 40) return dateMatch.index;
+  const start = Math.max(0, beforeUrl.length - 260);
+  const tail = beforeUrl.slice(start);
+  const sentenceBreaks = [...tail.matchAll(/[.!?]\s*(?=[A-Z][A-Za-z0-9"'$])/g)];
+  for (const match of sentenceBreaks) {
+    const remainder = tail.slice(match.index + 1).trim();
+    const lead = remainder.slice(0, 160);
+    const looksLikeTitle = /^(?:Best|Top|What|How|Why|When|Where|Which|Can|Should|Is|Are|Do|Does)\b/i.test(remainder);
+    if (remainder.length > 20 && looksLikeTitle && /(?:\s[-|]\s|Heating|Cooling|Company|Services|Blog|Guide|Review)/i.test(lead)) {
+      return start + match.index + 1;
+    }
+  }
+  const last = sentenceBreaks.at(-1);
+  if (last?.index !== void 0) return start + last.index + 1;
+  return beforeUrl.length;
+}
+function cutAtUrlAttribution(text) {
+  const urlMatch = text.match(/https?:\/\/\S+/i);
+  if (!urlMatch?.index) return text;
+  const beforeUrl = text.slice(0, urlMatch.index);
+  return beforeUrl.slice(0, findAttributionCut(beforeUrl));
+}
+function trimToSentenceLimit(text) {
+  if (text.length <= MAX_ANSWER_LENGTH) return text;
+  const slice = text.slice(0, MAX_ANSWER_LENGTH);
+  const lastSentence = Math.max(slice.lastIndexOf("."), slice.lastIndexOf("!"), slice.lastIndexOf("?"));
+  return (lastSentence > 240 ? slice.slice(0, lastSentence + 1) : slice).trim();
+}
+function cleanPAAAnswerText(answer, question, sourceTitle) {
+  if (!answer) return void 0;
+  let text = normalizeWhitespace(answer);
+  const normalizedQuestion = question ? normalizeWhitespace(question) : "";
+  if (normalizedQuestion && text.toLowerCase().startsWith(normalizedQuestion.toLowerCase())) {
+    text = text.slice(normalizedQuestion.length).trim();
+  }
+  if (/^An error has occurred\.?\s*Please try again later\.?/i.test(text)) {
+    return void 0;
+  }
+  for (const pattern of BOILERPLATE_PATTERNS) {
+    text = text.replace(pattern, " ");
+  }
+  text = text.replace(/\b[A-Z][A-Za-z&'\u2019 -]{2,60}\+\d+\b/g, " ").replace(/\b(?:[a-z0-9-]+\.)+[a-z]{2,}\+\d+\b/gi, " ");
+  text = normalizeWhitespace(text);
+  text = cutAtFirstMarker(text);
+  text = cutAtSourceTitle(text, sourceTitle);
+  text = cutAtUrlAttribution(text);
+  text = normalizeWhitespace(text);
+  text = text.replace(/\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},\s+\d{4}$/i, "").trim();
+  text = trimToSentenceLimit(text);
+  if (!text) return void 0;
+  if (/^An error has occurred\.?\s*Please try again later\.?$/i.test(text)) return void 0;
+  return text;
+}
+
+// src/extractor/PAAExtractor.ts
+var PAAExtractor = class {
+  constructor(driver, reporter) {
+    this.driver = driver;
+    this.reporter = reporter;
+  }
+  driver;
+  reporter;
+  normalizeQuestion(q) {
+    return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
+  }
+  async extractVisibleItems(page) {
+    const sels = PAASelectors;
+    const raw = await page.evaluate((selectors) => {
+      function cleanText(el) {
+        if (!el) return "";
+        const parts = [];
+        for (const n of el.childNodes) {
+          if (n.nodeType === Node.TEXT_NODE) {
+            const text = n.textContent?.trim();
+            if (text) parts.push(text);
+          } else if (n.tagName === "STYLE" || n.tagName === "SCRIPT") {
+            continue;
+          } else {
+            const text = cleanText(n);
+            if (text) parts.push(text);
+          }
+        }
+        return parts.join(" ").replace(/\s+/g, " ").trim();
+      }
+      return Array.from(document.querySelectorAll(selectors.item)).map((pair) => ({
+        question: pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || "",
+        answer: cleanText(pair.querySelector(selectors.answerContainer)) || void 0,
+        sourceTitle: pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0,
+        sourceSite: pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0,
+        sourceCite: pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0
+      }));
+    }, sels);
+    return raw.flatMap((item) => {
+      const cleaned = {
+        ...item,
+        answer: cleanPAAAnswerText(item.answer, item.question, item.sourceTitle)
+      };
+      const result = RawPAAItemSchema.safeParse(cleaned);
+      if (!result.success) {
+        console.warn("[PAAExtractor] item parse failed:", item.question, result.error.issues[0]?.message);
+        return [];
+      }
+      return [result.data];
+    });
+  }
+  async clickItem(page, questionText) {
+    try {
+      const pairLocator = page.locator(
+        `${PAASelectors.item}[data-q="${questionText}"], ${PAASelectors.item}[data-initq="${questionText}"]`
+      ).first();
+      await pairLocator.click();
+    } catch {
+    }
+  }
+  toFlatRow(item, depth, parentQuestion, seed) {
+    return {
+      seed_query: seed,
+      question: item.question,
+      answer: item.answer ?? "",
+      source_title: item.sourceTitle ?? "",
+      source_site: item.sourceSite ?? "",
+      source_cite: item.sourceCite ?? "",
+      depth,
+      parent_question: parentQuestion ?? "",
+      extracted_at: (/* @__PURE__ */ new Date()).toISOString()
+    };
+  }
+  async runBFS(page, options) {
+    const seenKeys = /* @__PURE__ */ new Set();
+    const seenQs = /* @__PURE__ */ new Set();
+    const depthMap = /* @__PURE__ */ new Map();
+    const results = [];
+    const readAllQs = () => page.evaluate(
+      ({ sel, dataQ, dataInitQ, questionEl }) => Array.from(document.querySelectorAll(sel)).map(
+        (el) => el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || ""
+      ).filter(Boolean),
+      { sel: PAASelectors.item, dataQ: PAASelectors.itemDataQ, dataInitQ: PAASelectors.itemDataInitQ, questionEl: PAASelectors.itemQuestionEl }
+    );
+    const dupRates = [];
+    const orderedQs = [];
+    for (let round = 0; round < options.depth; round++) {
+      this.reporter.onDepth(round + 1);
+      if (seenQs.size >= options.maxQuestions) break;
+      const beforeQs = await readAllQs();
+      if (beforeQs.length >= options.maxQuestions) break;
+      const unexpandedItems = await page.$$(
+        `${PAASelectors.item}:not(.${PAASelectors.expandedClass})`
+      );
+      if (unexpandedItems.length === 0) break;
+      for (const item of unexpandedItems) {
+        try {
+          await item.scrollIntoViewIfNeeded();
+          await item.click({ force: true });
+          await page.waitForTimeout(500);
+        } catch {
+        }
+      }
+      await page.waitForTimeout(1500);
+      const afterQs = await readAllQs();
+      const newQs = afterQs.slice(beforeQs.length);
+      const newDups = newQs.filter((q) => seenQs.has(q)).length;
+      const dupRate = newQs.length > 0 ? newDups / newQs.length : 0;
+      dupRates.push(dupRate);
+      if (dupRates.length > 2) dupRates.shift();
+      const rollingDupRate = dupRates.reduce((a, b) => a + b, 0) / dupRates.length;
+      for (const q of afterQs) {
+        if (!seenQs.has(q)) {
+          seenQs.add(q);
+          orderedQs.push(q);
+        }
+        if (!depthMap.has(q)) depthMap.set(q, round + 1);
+      }
+      if (afterQs.length === beforeQs.length) break;
+      if (rollingDupRate >= 0.6) break;
+    }
+    const itemMap = new Map((await this.extractVisibleItems(page)).map((i) => [i.question, i]));
+    for (const q of orderedQs) {
+      if (results.length >= options.maxQuestions) break;
+      const key = this.normalizeQuestion(q);
+      if (seenKeys.has(key)) continue;
+      seenKeys.add(key);
+      const d = depthMap.get(q) ?? 1;
+      const item = itemMap.get(q);
+      if (item) {
+        results.push(this.toFlatRow(item, d, null, options.query));
+        this.reporter.onQuestion({ question: item.question, answer: item.answer ?? null, sourceTitle: item.sourceTitle ?? null, sourceSite: item.sourceSite ?? null, sourceCite: item.sourceCite ?? null, depth: d, parentQuestion: null, children: [] });
+      } else {
+        results.push(this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, d, null, options.query));
+      }
+    }
+    return results;
+  }
+  async extractVideos(page) {
+    const vsels = VideoSelectors;
+    return page.evaluate((sels) => {
+      const results = [];
+      const containers = Array.from(document.querySelectorAll(sels.container));
+      for (const container of containers) {
+        const headingEl = container.querySelector(sels.sectionHeading);
+        const headingText = headingEl?.textContent?.trim() ?? "";
+        const type = headingText.toLowerCase().includes("short") ? "short_video" : "video";
+        const items = Array.from(container.querySelectorAll(sels.item));
+        for (const a of items) {
+          const href = a.href;
+          if (!href || !href.includes("youtube") && !href.includes("youtu.be")) continue;
+          const raw = a.textContent?.trim() ?? "";
+          const ytIdx = raw.indexOf("YouTube");
+          if (ytIdx === -1) continue;
+          const title = raw.slice(0, ytIdx).trim();
+          const remainder = raw.slice(ytIdx + 7).replace(/^[·\s·]+/, "");
+          const channelMatch = remainder.match(/^([^·\n]+)/);
+          const channel = channelMatch ? channelMatch[1].trim() : "";
+          if (title) results.push({ type, title, channel, platform: "YouTube", duration: "", url: href });
+        }
+      }
+      return results;
+    }, vsels);
+  }
+  async extractForums(page) {
+    const fsels = ForumSelectors;
+    return page.evaluate((sels) => {
+      const results = [];
+      const sections = Array.from(document.querySelectorAll(sels.section));
+      const forumSection = sections.find((s) => s.textContent?.includes("Discussions"));
+      if (!forumSection) return results;
+      const items = Array.from(forumSection.querySelectorAll(sels.item));
+      for (const a of items) {
+        const href = a.href;
+        if (!href) continue;
+        const titleEl = a.querySelector(sels.title);
+        const sourceEl = a.querySelector(sels.source);
+        const title = titleEl?.textContent?.trim() ?? "";
+        const source = sourceEl?.textContent?.trim() ?? "";
+        if (title) results.push({ title, source, url: href });
+      }
+      return results;
+    }, fsels);
+  }
+  async extractShortVideos(page, shortUrl) {
+    try {
+      await page.goto(shortUrl, { waitUntil: "domcontentloaded" });
+      await page.waitForTimeout(1500);
+    } catch {
+      return [];
+    }
+    const svSels = {
+      item: ShortVideoSelectors.item,
+      platforms: [...ShortVideoSelectors.platforms]
+    };
+    const raw = await page.evaluate((sels) => {
+      const seen = /* @__PURE__ */ new Set();
+      const results = [];
+      const items = Array.from(document.querySelectorAll(sels.item));
+      const videoHosts = ["youtube.com", "youtu.be", "tiktok.com", "instagram.com", "facebook.com", "fb.watch"];
+      const byHref = /* @__PURE__ */ new Map();
+      for (const a of items) {
+        const href = a.href;
+        if (!href) continue;
+        if (!videoHosts.some((h) => href.includes(h))) continue;
+        const text = a.textContent?.trim() ?? "";
+        if (!byHref.has(href)) byHref.set(href, []);
+        byHref.get(href).push(text);
+      }
+      for (const [href, texts] of byHref.entries()) {
+        if (seen.has(href)) continue;
+        seen.add(href);
+        const duration = texts.find((t) => /^\d+:\d+$/.test(t)) ?? "";
+        const titleText = texts.find((t) => !/^\d+:\d+$/.test(t) && t.length > 5) ?? "";
+        if (!titleText) continue;
+        let title = titleText;
+        let platform = "";
+        let channel = "";
+        for (const p of sels.platforms) {
+          let lastIdx = -1;
+          let search = 0;
+          while (true) {
+            const found = titleText.indexOf(p, search);
+            if (found === -1) break;
+            lastIdx = found;
+            search = found + 1;
+          }
+          if (lastIdx === -1) continue;
+          const after = titleText.slice(lastIdx + p.length);
+          const isSourceTag = /^[\s·]/.test(after) || after.trim() === "";
+          if (!isSourceTag) continue;
+          title = titleText.slice(0, lastIdx).trim();
+          platform = p;
+          const stripped = after.replace(/^[\s·]+/, "");
+          const dotIdx = stripped.indexOf("\xB7");
+          channel = (dotIdx === -1 ? stripped : stripped.slice(0, dotIdx)).trim();
+          break;
+        }
+        if (title) results.push({ title, channel, platform, duration, url: href });
+      }
+      return results;
+    }, svSels);
+    return raw.map((r) => ({ type: "short_video", ...r }));
+  }
+  async extractWhatPeopleSaying(page) {
+    const sels = WhatPeopleSayingSelectors;
+    return page.evaluate((s) => {
+      const section = Array.from(document.querySelectorAll(s.sectionTag)).find((el) => el.textContent?.includes(s.sectionHeadingText)) ?? document.querySelector(".yG4QQe.TBC9ub.NbhJ1c");
+      if (!section) return [];
+      return Array.from(section.querySelectorAll(s.card)).map((card) => {
+        const link = card.querySelector(s.cardLink);
+        const url = link?.href ?? "";
+        const titleH1 = card.querySelector(s.titleH1)?.textContent?.trim();
+        const titleDiv = card.querySelector(s.titleDiv)?.textContent?.trim();
+        const title = titleH1 ?? titleDiv ?? "";
+        const sourceText = card.querySelector(s.source)?.textContent?.trim() ?? "";
+        const platformEl = card.querySelector(s.platformBadge);
+        const platformText = platformEl?.textContent?.trim() ?? "";
+        const ytChannel = card.querySelector(s.ytChannel)?.textContent?.trim() ?? "";
+        const ytDate = card.querySelector(s.ytDate)?.textContent?.trim() ?? "";
+        const authorNote = card.querySelector(s.authorNote)?.textContent?.trim() ?? null;
+        const commentLabelEl = card.querySelector(s.popularCommentLabel);
+        let popularComment = null;
+        if (commentLabelEl) {
+          let next = commentLabelEl.nextSibling;
+          while (next) {
+            const t = next.textContent?.trim();
+            if (t) {
+              popularComment = t;
+              break;
+            }
+            next = next.nextSibling;
+          }
+        }
+        const allSpans = Array.from(card.querySelectorAll("span"));
+        const duration = allSpans.find((s2) => /^\d+:\d+$/.test(s2.textContent?.trim() ?? ""))?.textContent?.trim() ?? null;
+        const engagementParts = allSpans.map((s2) => s2.textContent?.trim() ?? "").filter(
+          (t) => /\d/.test(t) && (t.includes("comment") || t.includes("reaction") || t.includes("view") || t.includes("like") || t.includes("share"))
+        );
+        const engagement = engagementParts[0] ?? "";
+        const dateCandidates = allSpans.map((s2) => s2.textContent?.trim() ?? "").filter((t) => /\d+ (day|week|month|year|hour)s? ago|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec/.test(t));
+        const date = ytDate || (dateCandidates[0] ?? "");
+        const platform = platformText || (ytChannel ? "YouTube" : "");
+        const source = ytChannel || sourceText;
+        let type = "unknown";
+        const pl = platform.toLowerCase();
+        const src = source.toLowerCase();
+        const srcRaw = sourceText.toLowerCase();
+        if (pl.includes("reddit") || src.startsWith("r/")) type = "reddit";
+        else if (pl.includes("facebook") || srcRaw.includes("facebook")) type = "facebook";
+        else if (pl.includes("instagram") || srcRaw.includes("instagram")) type = "instagram";
+        else if (pl.includes("tiktok") || srcRaw.includes("tiktok")) type = "tiktok";
+        else if (pl.includes("youtube") || !!ytChannel) type = "youtube";
+        else type = "news";
+        return { type, title, url, source, platform, popularComment, engagement, date, duration, authorNote };
+      });
+    }, sels);
+  }
+  async extractOrganicResults(page) {
+    const sels = OrganicSelectors;
+    return page.evaluate((s) => {
+      const out = [];
+      let pos = 0;
+      document.querySelectorAll(s.result).forEach((card) => {
+        const titleEl = card.querySelector(s.title);
+        if (!titleEl) return;
+        const title = titleEl.textContent?.trim() ?? "";
+        const linkEl = titleEl.closest("a");
+        const url = linkEl?.href ?? "";
+        if (!title || !url) return;
+        pos++;
+        const cite = card.querySelector(s.cite)?.textContent?.trim() ?? null;
+        const snippet = card.querySelector(s.snippet)?.textContent?.trim() ?? null;
+        const isRedditStyle = !!card.querySelector(s.redditCite);
+        const ratingEl = card.querySelector(s.ratingWrap);
+        const inlineRating = ratingEl ? { value: ratingEl.querySelector(s.ratingValue)?.textContent?.trim() ?? "", count: ratingEl.querySelector(s.reviewCount)?.textContent?.trim() ?? "" } : null;
+        let domain = "";
+        try {
+          domain = new URL(url).hostname.replace(/^www\./, "");
+        } catch {
+          domain = card.querySelector(s.siteName)?.textContent?.trim() ?? "";
+        }
+        out.push({ position: pos, title, url, domain, cite, snippet, isRedditStyle, inlineRating });
+      });
+      return out;
+    }, sels);
+  }
+  async extractLocalPack(page) {
+    const sels = LocalPackSelectors;
+    return page.evaluate((s) => {
+      const out = [];
+      let container = null;
+      document.querySelectorAll('[role="heading"]').forEach((h) => {
+        if (!container && h.textContent?.includes(s.headingText)) container = h.closest("[data-hveid]");
+      });
+      if (!container) return out;
+      container.querySelectorAll(s.card).forEach((card, i) => {
+        const name = card.querySelector(s.name)?.textContent?.trim() ?? "";
+        if (!name) return;
+        const rating = card.querySelector(s.ratingValue)?.textContent?.trim() ?? null;
+        const reviewRaw = card.querySelector(s.reviewCount)?.textContent?.trim() ?? null;
+        const reviewCount = reviewRaw ? reviewRaw.replace(/[()]/g, "").trim() : null;
+        let cid = card.querySelector("a[data-cid]")?.getAttribute("data-cid") ?? null;
+        if (!cid) {
+          for (const link of Array.from(card.querySelectorAll("a[href]"))) {
+            const m1 = link.href.match(/[?&]cid=(\d+)/);
+            if (m1) {
+              cid = m1[1];
+              break;
+            }
+            const m2 = link.href.match(/!1s0x[0-9a-f]+:0x([0-9a-f]+)/i);
+            if (m2) {
+              try {
+                cid = BigInt("0x" + m2[1]).toString();
+              } catch {
+              }
+              if (cid) break;
+            }
+          }
+        }
+        const metadata = [];
+        card.querySelectorAll("div, span").forEach((el) => {
+          const text = Array.from(el.childNodes).filter((n) => n.nodeType === 3).map((n) => n.textContent?.trim() ?? "").filter((t) => t.length > 1 && t.length < 120).join(" ");
+          if (text && !metadata.includes(text)) metadata.push(text);
+        });
+        const links = Array.from(card.querySelectorAll("a[href]"));
+        const directionsUrl = links.find((a) => a.href.includes("google.com/maps"))?.href ?? null;
+        const websiteUrl = links.find((a) => !a.href.includes("google.com") && a.href.startsWith("http"))?.href ?? null;
+        out.push({ position: i + 1, name, cid, rating, reviewCount, metadata, websiteUrl, directionsUrl });
+      });
+      return out;
+    }, sels);
+  }
+  async extractEntityIds(page) {
+    return page.evaluate(() => {
+      const kgIds = /* @__PURE__ */ new Set();
+      const cids = /* @__PURE__ */ new Set();
+      const gcids = /* @__PURE__ */ new Set();
+      const recordMap = /* @__PURE__ */ new Map();
+      function nameFromWrapper(el) {
+        const sel = [".OSrXXb", ".dbg0pd", ".tzt0oe", '[role="heading"]', "h3"];
+        for (const s of sel) {
+          const found = el.querySelector(s);
+          if (found?.textContent?.trim()) return found.textContent.trim();
+        }
+        return "";
+      }
+      document.querySelectorAll('[id^="pv-/g/"]').forEach((wrapper) => {
+        const raw = wrapper.getAttribute("id");
+        if (!raw) return;
+        const kgId = raw.replace("pv-", "");
+        kgIds.add(kgId);
+        const name = nameFromWrapper(wrapper);
+        const cidEl = wrapper.querySelector("a[data-cid]");
+        const cid = cidEl?.getAttribute("data-cid") ?? null;
+        if (cid) cids.add(cid);
+        if (name) recordMap.set(kgId, { name, kgId, cid, gcid: null });
+      });
+      document.querySelectorAll("[data-mid]").forEach((el) => {
+        const mid = el.getAttribute("data-mid");
+        if (!mid?.startsWith("/g/")) return;
+        kgIds.add(mid);
+        if (!recordMap.has(mid)) {
+          const name = nameFromWrapper(el);
+          if (name) recordMap.set(mid, { name, kgId: mid, cid: null, gcid: null });
+        }
+      });
+      document.querySelectorAll(".w7Dbne").forEach((card) => {
+        const cidEl = card.querySelector("a[data-cid]");
+        const cid = cidEl?.getAttribute("data-cid") ?? null;
+        if (!cid) return;
+        cids.add(cid);
+        const name = card.querySelector(".OSrXXb")?.textContent?.trim() ?? "";
+        if (!name) return;
+        const kgIdEl = card.querySelector('[id^="pv-/g/"]');
+        const kgId = kgIdEl ? kgIdEl.getAttribute("id").replace("pv-", "") : null;
+        const key = kgId ?? `cid:${cid}`;
+        if (recordMap.has(key)) {
+          const existing = recordMap.get(key);
+          if (!existing.cid) recordMap.set(key, { ...existing, cid });
+        } else {
+          recordMap.set(key, { name, kgId, cid, gcid: null });
+        }
+      });
+      document.querySelectorAll("a[data-cid]").forEach((el) => {
+        const cid = el.getAttribute("data-cid");
+        if (!cid) return;
+        cids.add(cid);
+        const alreadyNamed = [...recordMap.values()].some((r) => r.cid === cid);
+        if (!alreadyNamed) {
+          let node = el.parentElement;
+          let name = "";
+          for (let i = 0; i < 8 && node; i++) {
+            const h = node.querySelector('.OSrXXb, .dbg0pd, [role="heading"], h3');
+            if (h?.textContent?.trim()) {
+              name = h.textContent.trim();
+              break;
+            }
+            node = node.parentElement;
+          }
+          if (name) recordMap.set(`cid:${cid}`, { name, kgId: null, cid, gcid: null });
+        }
+      });
+      const scriptContent = Array.from(document.querySelectorAll("script:not([src])")).map((s) => s.textContent ?? "").filter((t) => t.length > 1e4).join("\n");
+      for (const m of scriptContent.matchAll(/\/g\/[a-zA-Z0-9_-]{5,20}/g)) kgIds.add(m[0]);
+      for (const m of scriptContent.matchAll(/gcid:[a-zA-Z0-9_]+/g)) gcids.add(m[0]);
+      for (const m of scriptContent.matchAll(/0x[0-9a-f]+:0x([0-9a-f]+)/gi)) {
+        try {
+          cids.add(BigInt("0x" + m[1]).toString());
+        } catch {
+        }
+      }
+      return { entities: [...recordMap.values()], kgIds: [...kgIds], cids: [...cids], gcids: [...gcids] };
+    });
+  }
+  mergeLocalPackIntoEntities(entityIds, localPack) {
+    const cidSet = new Set(entityIds.cids);
+    const records = entityIds.entities.map((r) => ({ ...r }));
+    for (const biz of localPack) {
+      if (!biz.cid) continue;
+      cidSet.add(biz.cid);
+      const nameNorm = biz.name.toLowerCase().trim();
+      const byName = records.find((r) => r.name.toLowerCase().trim() === nameNorm);
+      if (byName) {
+        if (!byName.cid) byName.cid = biz.cid;
+      } else if (!records.find((r) => r.cid === biz.cid)) {
+        records.push({ name: biz.name, kgId: null, cid: biz.cid, gcid: null });
+      }
+    }
+    return { ...entityIds, entities: records, cids: [...cidSet] };
+  }
+  async extractAISurfaces(page) {
+    const aioSels = AIOverviewSelectors;
+    const aimSels = AIModeSelectors;
+    return page.evaluate(({ aio, aim }) => {
+      const sn = window.google?.sn ?? "unknown";
+      const surface = sn === "aim" ? "aim" : sn === "web" ? "web" : "unknown";
+      function findAIORoot() {
+        const primary = document.querySelector(aio.root);
+        if (primary) return primary;
+        const headings = document.querySelectorAll('h1, h2, h3, [role="heading"]');
+        for (const h of headings) {
+          if (h.textContent?.trim() === "AI Overview") {
+            let el = h.parentElement;
+            for (let i = 0; i < 6 && el; i++) {
+              if (el.querySelectorAll("a").length > 1) return el;
+              el = el.parentElement;
+            }
+            return h.parentElement;
+          }
+        }
+        return null;
+      }
+      const aioRoot = findAIORoot();
+      const aioContainer = aioRoot ? aioRoot.closest(aio.wrapper) ?? aioRoot : null;
+      let aioText = null;
+      if (aioContainer) {
+        const clone = aioContainer.cloneNode(true);
+        clone.querySelectorAll("script,style,noscript").forEach((el) => el.remove());
+        clone.querySelectorAll('h1,h2,h3,h4,[role="heading"]').forEach((el) => el.remove());
+        clone.querySelectorAll('button,[role="button"]').forEach((el) => el.remove());
+        clone.querySelectorAll("a").forEach((el) => el.remove());
+        const candidate = clone.textContent?.replace(/\s+/g, " ").trim() || null;
+        const isErrorState = !candidate || /not available|try again|can't generate/i.test(candidate);
+        aioText = isErrorState ? null : candidate;
+      }
+      const aioDetected = !!aioRoot && aioText !== null;
+      const aioCitations = Array.from(aioContainer?.querySelectorAll("a[href]") ?? []).filter((a) => a.href && !a.href.startsWith("javascript")).map((a) => ({
+        text: a.textContent?.trim() ?? "",
+        href: a.href
+      })).filter((c) => c.text && c.href);
+      const aimRoot = document.querySelector(aim.root);
+      const aimDetected = surface === "aim" && !!aimRoot;
+      const aimContainer = aimRoot?.closest(aim.wrapper) ?? null;
+      let aimText = null;
+      if (aimContainer) {
+        const clone = aimContainer.cloneNode(true);
+        clone.querySelectorAll("script,style,noscript").forEach((el) => el.remove());
+        clone.querySelectorAll('h1,h2,h3,h4,[role="heading"]').forEach((el) => el.remove());
+        clone.querySelectorAll('button,[role="button"]').forEach((el) => el.remove());
+        clone.querySelectorAll("a").forEach((el) => el.remove());
+        const candidate = clone.textContent?.replace(/\s+/g, " ").trim() || null;
+        const isErrorState = !candidate || /not available|try again|can't generate/i.test(candidate);
+        aimText = isErrorState ? null : candidate;
+      }
+      const aimCitations = aimDetected ? Array.from(aimContainer?.querySelectorAll("a[href]") ?? []).filter((a) => a.href && !a.href.startsWith("javascript")).map((a) => ({
+        text: a.textContent?.trim() ?? "",
+        href: a.href
+      })).filter((c) => c.text && c.href) : [];
+      return {
+        surface,
+        aiOverview: { detected: aioDetected, text: aioText, citations: aioCitations },
+        aiMode: { detected: aimDetected, text: aimText, citations: aimCitations }
+      };
+    }, { aio: aioSels, aim: aimSels });
+  }
+  buildTree(flat, _seed) {
+    const roots = [];
+    const nodeMap = /* @__PURE__ */ new Map();
+    for (const row of flat) {
+      const node = {
+        question: row.question,
+        answer: row.answer || null,
+        sourceTitle: row.source_title || null,
+        sourceSite: row.source_site || null,
+        sourceCite: row.source_cite || null,
+        depth: row.depth,
+        parentQuestion: row.parent_question || null,
+        children: []
+      };
+      nodeMap.set(row.question, node);
+    }
+    for (const node of nodeMap.values()) {
+      if (node.parentQuestion && nodeMap.has(node.parentQuestion)) {
+        nodeMap.get(node.parentQuestion).children.push(node);
+      } else {
+        roots.push(node);
+      }
+    }
+    return roots;
+  }
+  async extract(options) {
+    const startMs = Date.now();
+    const config = {
+      headless: options.headless,
+      profileDir: options.profileDir,
+      proxy: options.proxy,
+      kernelApiKey: options.kernelApiKey,
+      kernelProxyId: options.kernelProxyId,
+      viewport: { width: 1280, height: 800 },
+      locale: `${options.hl}-${options.gl.toUpperCase()}`
+    };
+    let errorCount = 0;
+    try {
+      await this.driver.launch(config);
+      const uule = options.location ? encodeUule(normalizeLocation(options.location)) : null;
+      const { hasPaa } = await this.driver.navigateToSERP(options.query, uule, options.gl, options.hl);
+      const page = this.driver.getPage();
+      if (options.serpOnly) {
+        const [organicResults2, localPack2, rawEntityIds2] = await Promise.all([
+          this.extractOrganicResults(page),
+          this.extractLocalPack(page),
+          this.extractEntityIds(page)
+        ]);
+        const entityIds2 = this.mergeLocalPackIntoEntities(rawEntityIds2, localPack2);
+        let allOrganic2 = organicResults2;
+        if ((options.pages ?? 1) >= 2) {
+          const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, start: "10" });
+          if (uule) p2params.set("uule", uule);
+          await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
+          const p2organic = await this.extractOrganicResults(page);
+          allOrganic2 = [...organicResults2, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
+        }
+        const stats2 = {
+          seed: options.query,
+          totalQuestions: 0,
+          maxDepthReached: 0,
+          durationMs: Date.now() - startMs,
+          errorCount
+        };
+        this.reporter.onComplete(stats2);
+        return {
+          seed: options.query,
+          location: options.location ?? null,
+          extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          totalQuestions: 0,
+          surface: "web",
+          aiOverview: { detected: false, text: null, citations: [] },
+          aiMode: { detected: false, text: null, citations: [] },
+          whatPeopleSaying: [],
+          tree: [],
+          flat: [],
+          videos: [],
+          forums: [],
+          organicResults: allOrganic2,
+          localPack: localPack2,
+          entityIds: entityIds2,
+          stats: stats2
+        };
+      }
+      const [videos, forums, whatPeopleSaying, rawEntityIds, organicResults, localPack] = await Promise.all([
+        this.extractVideos(page),
+        this.extractForums(page),
+        this.extractWhatPeopleSaying(page),
+        this.extractEntityIds(page),
+        this.extractOrganicResults(page),
+        this.extractLocalPack(page)
+      ]);
+      const entityIds = this.mergeLocalPackIntoEntities(rawEntityIds, localPack);
+      this.reporter.onVideos(videos);
+      this.reporter.onForums(forums);
+      if (!hasPaa) {
+        let noPaaOrganic = organicResults;
+        if ((options.pages ?? 1) >= 2) {
+          const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, start: "10" });
+          if (uule) p2params.set("uule", uule);
+          await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
+          const p2organic = await this.extractOrganicResults(page);
+          noPaaOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
+        }
+        const aiSurfaces2 = await this.extractAISurfaces(page);
+        const stats2 = {
+          seed: options.query,
+          totalQuestions: 0,
+          maxDepthReached: 0,
+          durationMs: Date.now() - startMs,
+          errorCount
+        };
+        this.reporter.onComplete(stats2);
+        return {
+          seed: options.query,
+          location: options.location ?? null,
+          extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          totalQuestions: 0,
+          surface: aiSurfaces2.surface,
+          aiOverview: aiSurfaces2.aiOverview,
+          aiMode: aiSurfaces2.aiMode,
+          whatPeopleSaying,
+          tree: [],
+          flat: [],
+          videos,
+          forums,
+          organicResults: noPaaOrganic,
+          localPack,
+          entityIds,
+          stats: stats2
+        };
+      }
+      const flat = await this.runBFS(page, options);
+      const aiSurfaces = await this.extractAISurfaces(page);
+      const shortVidsParams = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, udm: ShortVideoSelectors.udm });
+      if (uule) shortVidsParams.set("uule", uule);
+      const shortVideos = await this.extractShortVideos(page, "https://www.google.com/search?" + shortVidsParams.toString());
+      this.reporter.onVideos(shortVideos);
+      let allOrganic = organicResults;
+      if ((options.pages ?? 1) >= 2) {
+        const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, start: "10" });
+        if (uule) p2params.set("uule", uule);
+        await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
+        const p2organic = await this.extractOrganicResults(page);
+        allOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
+      }
+      const allVideos = [...videos, ...shortVideos];
+      const tree = this.buildTree(flat, options.query);
+      const stats = {
+        seed: options.query,
+        totalQuestions: flat.length,
+        maxDepthReached: flat.reduce((m, r) => Math.max(m, r.depth), 0),
+        durationMs: Date.now() - startMs,
+        errorCount
+      };
+      this.reporter.onComplete(stats);
+      return {
+        seed: options.query,
+        location: options.location ?? null,
+        extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        totalQuestions: flat.length,
+        surface: aiSurfaces.surface,
+        aiOverview: aiSurfaces.aiOverview,
+        aiMode: aiSurfaces.aiMode,
+        whatPeopleSaying,
+        tree,
+        flat,
+        videos: allVideos,
+        forums,
+        organicResults: allOrganic,
+        localPack,
+        entityIds,
+        stats
+      };
+    } catch (err) {
+      errorCount++;
+      this.reporter.onError(err instanceof Error ? err : new Error(String(err)));
+      throw err;
+    } finally {
+      await this.driver.close();
+    }
+  }
+};
+
+// src/output/OutputSerializer.ts
+var import_node_fs = require("fs");
+var import_node_path = __toESM(require("path"), 1);
+var import_papaparse = __toESM(require("papaparse"), 1);
+var OutputSerializer = class {
+  async writeJSON(result, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = result.seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const filename = `${slug}-${Date.now()}.json`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, JSON.stringify(result, null, 2), "utf8");
+    return fullPath;
+  }
+  async writeCSV(rows, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const seedRaw = rows[0]?.seed_query ?? "paa";
+    const slug = seedRaw.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const csv = import_papaparse.default.unparse(rows, { header: true });
+    const filename = `${slug}-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+  async writeVideoCSV(videos, seed, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const csv = import_papaparse.default.unparse(videos, { header: true });
+    const filename = `${slug}-videos-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+  async writeForumCSV(forums, seed, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const csv = import_papaparse.default.unparse(forums, { header: true });
+    const filename = `${slug}-forums-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+  async writeAIOverviewCSV(citations, text, seed, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const rows = citations.map((c, i) => ({
+      seed_query: seed,
+      response_text: i === 0 ? text ?? "" : "",
+      citation_text: c.text,
+      citation_href: c.href
+    }));
+    const csv = import_papaparse.default.unparse(rows, { header: true });
+    const filename = `${slug}-ai-overview-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+  async writeAIModeCSV(citations, text, seed, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const rows = citations.map((c, i) => ({
+      seed_query: seed,
+      response_text: i === 0 ? text ?? "" : "",
+      citation_text: c.text,
+      citation_href: c.href
+    }));
+    const csv = import_papaparse.default.unparse(rows, { header: true });
+    const filename = `${slug}-ai-mode-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+  async writeWhatPeopleSayingCSV(cards, seed, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const rows = cards.map((c) => ({ seed_query: seed, ...c }));
+    const csv = import_papaparse.default.unparse(rows, { header: true });
+    const filename = `${slug}-what-people-saying-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+};
+
+// src/output/ProgressReporter.ts
+var ProgressReporter = class {
+  onQuestion(node) {
+    process.stdout.write(JSON.stringify({ event: "question", depth: node.depth, question: node.question }) + "\n");
+  }
+  onDepth(depth) {
+    process.stdout.write(JSON.stringify({ event: "depth", depth }) + "\n");
+  }
+  onVideos(videos) {
+    for (const v of videos) {
+      process.stdout.write(JSON.stringify({ event: "video", type: v.type, platform: v.platform, duration: v.duration, title: v.title, channel: v.channel, url: v.url }) + "\n");
+    }
+  }
+  onForums(forums) {
+    for (const f of forums) {
+      process.stdout.write(JSON.stringify({ event: "forum", title: f.title, source: f.source, url: f.url }) + "\n");
+    }
+  }
+  onComplete(stats) {
+    process.stdout.write(JSON.stringify({ event: "complete", ...stats }) + "\n");
+  }
+  onError(err) {
+    process.stderr.write(JSON.stringify({ event: "error", type: err.constructor.name, message: err.message }) + "\n");
+  }
+};
+
+// src/harvest.ts
+var MAX_ATTEMPTS = 3;
+async function extractOnce(options) {
+  const driver = new BrowserDriver();
+  const reporter = new ProgressReporter();
+  const extractor = new PAAExtractor(driver, reporter);
+  try {
+    return await extractor.extract(options);
+  } finally {
+    await driver.close();
+  }
+}
+async function harvest(rawOptions) {
+  const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};
+  const merged = {
+    kernelApiKey: process.env.KERNEL_API_KEY?.trim(),
+    kernelProxyId: process.env.KERNEL_PROXY_ID?.trim(),
+    ...raw
+  };
+  const options = HarvestOptionsSchema.parse(merged);
+  const serializer = new OutputSerializer();
+  for (let i = 0; i < MAX_ATTEMPTS; i++) {
+    try {
+      const result = await extractOnce(options);
+      if (options.format === "json" || options.format === "both") {
+        await serializer.writeJSON(result, options.outputDir);
+      }
+      if (options.format === "csv" || options.format === "both") {
+        await Promise.all([
+          serializer.writeCSV(result.flat, options.outputDir),
+          result.videos.length > 0 ? serializer.writeVideoCSV(result.videos, result.seed, options.outputDir) : Promise.resolve(""),
+          result.forums.length > 0 ? serializer.writeForumCSV(result.forums, result.seed, options.outputDir) : Promise.resolve(""),
+          result.aiOverview.detected ? serializer.writeAIOverviewCSV(result.aiOverview.citations, result.aiOverview.text, result.seed, options.outputDir) : Promise.resolve(""),
+          result.aiMode.detected ? serializer.writeAIModeCSV(result.aiMode.citations, result.aiMode.text, result.seed, options.outputDir) : Promise.resolve(""),
+          result.whatPeopleSaying.length > 0 ? serializer.writeWhatPeopleSayingCSV(result.whatPeopleSaying, result.seed, options.outputDir) : Promise.resolve("")
+        ]);
+      }
+      return result;
+    } catch (err) {
+      if (err instanceof CaptchaError && i < MAX_ATTEMPTS - 1) {
+        continue;
+      }
+      throw err;
+    }
+  }
+  const sessionDesc = options.kernelApiKey ? `${MAX_ATTEMPTS} fresh Kernel.sh sessions` : `${MAX_ATTEMPTS} attempts`;
+  throw new CaptchaError(`CAPTCHA on all ${sessionDesc}. Try again in a few minutes.`);
+}
+
+// src/cli.ts
+var program = new import_commander.Command();
+program.name("paa-harvest").description("Recursively extract Google People Also Ask questions").requiredOption("-q, --query <query>", "Seed query").option("-l, --location <location>", 'Location name (e.g. "austin" or "Austin,Texas,United States")').option("--gl <gl>", "Google country code", "us").option("--hl <hl>", "Google language code", "en").option("-d, --depth <depth>", "BFS depth (1-30)", "3").option("-m, --max-questions <n>", "Max questions to harvest", "100").option("-o, --output <dir>", "Output directory", "./paa-output").option("-f, --format <format>", "Output format: json, csv, or both", "both").option("--headless", "Run browser in headless mode", false).option("--profile <dir>", "Persistent browser profile directory").option("--proxy <url>", "Proxy server URL").option("--kernel-api-key <key>", "Kernel.sh API key (or set KERNEL_API_KEY env var)").action(async (opts) => {
+  try {
+    const result = await harvest({
+      query: opts.query,
+      location: opts.location,
+      gl: opts.gl,
+      hl: opts.hl,
+      depth: parseInt(opts.depth, 10),
+      maxQuestions: parseInt(opts.maxQuestions, 10),
+      outputDir: opts.output,
+      format: opts.format,
+      headless: opts.headless,
+      profileDir: opts.profile,
+      proxy: opts.proxy,
+      kernelApiKey: opts.kernelApiKey ?? process.env.KERNEL_API_KEY
+    });
+    console.log(JSON.stringify({ totalQuestions: result.totalQuestions, outputDir: result.stats.seed }));
+  } catch (err) {
+    console.error(err instanceof Error ? err.message : String(err));
+    process.exit(1);
+  }
+});
+async function runCli() {
+  await program.parseAsync();
+}
+
+// bin/paa-harvest.ts
+runCli();
+//# sourceMappingURL=paa-harvest.cjs.map