mcp-scraper 0.1.0

package/dist/index.cjs

@@ -0,0 +1,1689 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var src_exports = {};
+__export(src_exports, {
+  VideoGenerator: () => VideoGenerator,
+  buildClipPrompts: () => buildClipPrompts,
+  harvest: () => harvest
+});
+module.exports = __toCommonJS(src_exports);
+
+// src/schemas.ts
+var import_zod = require("zod");
+var HarvestOptionsSchema = import_zod.z.object({
+  query: import_zod.z.string().min(1),
+  location: import_zod.z.string().optional(),
+  gl: import_zod.z.string().length(2).default("us"),
+  hl: import_zod.z.string().length(2).default("en"),
+  depth: import_zod.z.number().int().min(1).max(30).default(3),
+  maxQuestions: import_zod.z.number().int().min(1).max(1e3).default(100),
+  headless: import_zod.z.boolean().default(false),
+  profileDir: import_zod.z.string().optional(),
+  proxy: import_zod.z.string().url().optional(),
+  kernelApiKey: import_zod.z.string().optional(),
+  kernelProxyId: import_zod.z.string().optional(),
+  outputDir: import_zod.z.string().default("./paa-output"),
+  format: import_zod.z.enum(["json", "csv", "both"]).default("both"),
+  serpOnly: import_zod.z.boolean().default(false),
+  pages: import_zod.z.number().int().min(1).max(2).default(1)
+});
+var MapsPlaceOptionsSchema = import_zod.z.object({
+  businessName: import_zod.z.string().min(1),
+  location: import_zod.z.string().min(1),
+  gl: import_zod.z.string().length(2).default("us"),
+  hl: import_zod.z.string().length(2).default("en"),
+  includeReviews: import_zod.z.boolean().default(false),
+  maxReviews: import_zod.z.number().int().min(1).max(500).default(50),
+  kernelApiKey: import_zod.z.string().optional(),
+  kernelProxyId: import_zod.z.string().optional(),
+  headless: import_zod.z.boolean().default(true)
+});
+var RawPAAItemSchema = import_zod.z.object({
+  question: import_zod.z.string().min(1),
+  answer: import_zod.z.string().optional(),
+  sourceTitle: import_zod.z.string().optional(),
+  sourceSite: import_zod.z.string().optional(),
+  sourceCite: import_zod.z.string().optional()
+});
+
+// src/driver/BrowserDriver.ts
+var import_playwright_extra = require("playwright-extra");
+var import_puppeteer_extra_plugin_stealth = __toESM(require("puppeteer-extra-plugin-stealth"), 1);
+var import_playwright = require("playwright");
+var import_sdk = __toESM(require("@onkernel/sdk"), 1);
+
+// src/selectors.ts
+var PAASelectors = {
+  container: ".eJH8qe.adDDi",
+  dataInitq: "[data-initq]",
+  item: ".related-question-pair",
+  itemDataQ: "data-q",
+  itemDataInitQ: "data-initq",
+  itemQuestionEl: ".JlqpRe",
+  answerContainer: ".bCOlv",
+  sourceTitle: "h3",
+  sourceSite: ".VuuXrf",
+  sourceCite: "cite",
+  clickTarget: ".JlqpRe",
+  expandedClass: "aoRk1c",
+  captchaMarker: '#captcha-form, #recaptcha, form[action*="/sorry/"], .g-recaptcha, [data-sitekey]'
+};
+var VideoSelectors = {
+  container: 'div[jscontroller="HWk0Gf"]',
+  sectionHeading: '.mgAbYb[role="heading"]',
+  item: "a.rIRoqf"
+};
+var ShortVideoSelectors = {
+  udm: "39",
+  item: "a.rIRoqf",
+  durationPattern: /^\d+:\d+$/,
+  platforms: ["YouTube", "TikTok", "Instagram", "Facebook", "X"]
+};
+var ForumSelectors = {
+  section: ".ULSxyf",
+  item: "a.KYg7td.INpicf",
+  title: ".hyYc0c",
+  source: ".K4ETW"
+};
+var WhatPeopleSayingSelectors = {
+  sectionTag: "g-section-with-header",
+  sectionHeadingText: "What people are saying",
+  card: '.dRzkFf[role="listitem"]',
+  cardLink: 'a.WlydOe[jsname="YKoRaf"]',
+  titleH1: "h1.WQWxe",
+  titleDiv: ".eAaXgc",
+  popularCommentLabel: ".qgdis",
+  source: ".sTl1Td",
+  platformBadge: ".appd0, .KrMNbf",
+  ytChannel: ".sjVJQd",
+  ytDate: ".PLq9Je",
+  authorNote: ".nDgy9d"
+};
+var AIOverviewSelectors = {
+  root: '[data-hveid="CBMQAA"]',
+  wrapper: ".Fgyi2e",
+  citations: '.Fgyi2e [data-hveid] a[jsname="pxBnId"]'
+};
+var AIModeSelectors = {
+  root: '[data-hveid="CAUQAA"]',
+  wrapper: ".Fgyi2e",
+  citations: '.Fgyi2e [data-hveid] a[jsname="pxBnId"]'
+};
+var OrganicSelectors = {
+  result: ".wHYlTd.tF2Cxc",
+  title: "h3.LC20lb",
+  siteName: ".VuuXrf",
+  cite: "cite.tjvcx",
+  snippet: ".VwiC3b",
+  redditCite: "cite.qLRx3b",
+  ratingWrap: ".Y0A0hc",
+  ratingValue: ".yi40Hd",
+  reviewCount: ".RDApEe"
+};
+var LocalPackSelectors = {
+  headingText: "Businesses",
+  card: ".w7Dbne",
+  name: ".OSrXXb",
+  ratingValue: ".yi40Hd",
+  reviewCount: ".RDApEe"
+};
+
+// src/errors.ts
+var RECAPTCHA_INSTRUCTIONS = "Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.";
+var CaptchaError = class extends Error {
+  constructor(instructions) {
+    super(`CAPTCHA detected. ${instructions}`);
+    this.instructions = instructions;
+  }
+  instructions;
+  name = "CaptchaError";
+};
+var ExtractionError = class extends Error {
+  constructor(message, cause) {
+    super(message);
+    this.cause = cause;
+  }
+  cause;
+  name = "ExtractionError";
+};
+
+// src/driver/BrowserDriver.ts
+import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
+var DESKTOP_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
+function buildYouTubeChannelVideosUrl(channelInput) {
+  const raw = channelInput.trim();
+  if (!raw) throw new Error("channelHandle is required");
+  const urlLike = /^https?:\/\//i.test(raw) || /^(www\.|m\.)?youtube\.com\//i.test(raw);
+  if (urlLike) {
+    const parsed = new URL(/^https?:\/\//i.test(raw) ? raw : `https://${raw}`);
+    const host = parsed.hostname.replace(/^www\./, "").replace(/^m\./, "").toLowerCase();
+    if (host !== "youtube.com") throw new Error("channel URL must be on youtube.com");
+    const segments = parsed.pathname.split("/").filter(Boolean);
+    const first = segments[0] ?? "";
+    const second = segments[1] ?? "";
+    if (first.startsWith("@")) return `https://www.youtube.com/${first}/videos`;
+    if (first === "channel" && second) return `https://www.youtube.com/channel/${second}/videos`;
+    if ((first === "c" || first === "user") && second) return `https://www.youtube.com/${first}/${second}/videos`;
+    throw new Error("channel URL must be a YouTube handle, /channel/UC..., /c/..., or /user/... URL");
+  }
+  const stripped = raw.replace(/^\/+/, "").replace(/\/+$/, "");
+  const withoutVideos = stripped.replace(/\/videos$/i, "");
+  if (/^UC[\w-]{20,}$/.test(withoutVideos)) {
+    return `https://www.youtube.com/channel/${withoutVideos}/videos`;
+  }
+  const handle = withoutVideos.startsWith("@") ? withoutVideos : `@${withoutVideos}`;
+  if (!/^@[\w.-]+$/.test(handle)) {
+    throw new Error("channelHandle must be an @handle, UC channel ID, or YouTube channel URL");
+  }
+  return `https://www.youtube.com/${handle}/videos`;
+}
+var BrowserDriver = class {
+  browser = null;
+  context = null;
+  page = null;
+  kernelClient = null;
+  kernelSessionId = null;
+  async launch(config) {
+    if (config.kernelApiKey) {
+      this.kernelClient = new import_sdk.default({ apiKey: config.kernelApiKey });
+      const kernelBrowser = await this.kernelClient.browsers.create({
+        stealth: true,
+        timeout_seconds: 600,
+        ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {}
+      });
+      this.kernelSessionId = kernelBrowser.session_id;
+      this.browser = await import_playwright.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
+      this.context = this.browser.contexts()[0] ?? await this.browser.newContext();
+      await this.installEsbuildHelperShims(this.context);
+      this.page = this.context.pages()[0] ?? await this.context.newPage();
+      return;
+    }
+    const launchOpts = {
+      headless: config.headless,
+      proxy: config.proxy ? { server: config.proxy } : void 0
+    };
+    const ctxOpts = {
+      viewport: config.viewport,
+      locale: config.locale,
+      userAgent: DESKTOP_USER_AGENT
+    };
+    if (config.profileDir) {
+      this.context = await import_playwright_extra.chromium.launchPersistentContext(config.profileDir, {
+        ...launchOpts,
+        ...ctxOpts
+      });
+      await this.installEsbuildHelperShims(this.context);
+      this.page = await this.context.newPage();
+    } else {
+      this.browser = await import_playwright_extra.chromium.launch(launchOpts);
+      this.context = await this.browser.newContext(ctxOpts);
+      await this.installEsbuildHelperShims(this.context);
+      this.page = await this.context.newPage();
+    }
+  }
+  async installEsbuildHelperShims(context) {
+    await context.addInitScript(() => {
+      const g = globalThis;
+      if (typeof g.__name !== "function") g.__name = (fn) => fn;
+      if (typeof g.__publicField !== "function") g.__publicField = (obj, key, value) => {
+        obj[key] = value;
+        return value;
+      };
+    });
+  }
+  async navigateToSERP(query, uule, gl, hl) {
+    const params = new URLSearchParams({ q: query, gl, hl });
+    if (uule) params.set("uule", uule);
+    const url = "https://www.google.com/search?" + params.toString();
+    try {
+      await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+    } catch (err) {
+      const diag = await this.captureDiagnostics(url);
+      throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
+    }
+    const captchaCount = await this.page.locator(PAASelectors.captchaMarker).count();
+    if (captchaCount > 0) {
+      if (this.kernelClient) {
+        try {
+          await this.page.waitForSelector(PAASelectors.container, { timeout: 45e3 });
+          return { hasPaa: true };
+        } catch {
+          throw new CaptchaError(this.captchaMessage());
+        }
+      }
+      throw new CaptchaError(this.captchaMessage());
+    }
+    const fastFound = await this.page.waitForSelector(PAASelectors.item, { timeout: 4e3 }).catch(() => null);
+    if (fastFound) return { hasPaa: true };
+    const captchaAfter = await this.page.locator(PAASelectors.captchaMarker).count();
+    if (captchaAfter > 0) throw new CaptchaError(this.captchaMessage());
+    for (let i = 1; i <= 6; i++) {
+      await this.page.evaluate((f) => {
+        window.scrollTo(0, document.body.scrollHeight * f);
+      }, i / 6);
+      await this.page.waitForTimeout(600);
+      const count = await this.page.locator(PAASelectors.item).count();
+      if (count > 0) return { hasPaa: true };
+    }
+    return { hasPaa: false };
+  }
+  async captureDiagnostics(intendedUrl) {
+    try {
+      const finalUrl = this.page.url();
+      const title = await this.page.title().catch(() => "");
+      const bodySnippet = await this.page.evaluate(() => {
+        const t = (document.body?.innerText ?? "").replace(/\s+/g, " ").trim();
+        return t.slice(0, 400);
+      }).catch(() => "");
+      const consent = /consent\.google\./.test(finalUrl) || /before you continue/i.test(bodySnippet);
+      const recaptcha = /recaptcha|unusual traffic|are you a robot/i.test(bodySnippet);
+      const flags = [
+        consent ? "CONSENT_WALL" : "",
+        recaptcha ? "BOT_CHALLENGE" : "",
+        finalUrl !== intendedUrl ? "REDIRECTED" : ""
+      ].filter(Boolean).join(",");
+      return `intended=${intendedUrl} | final=${finalUrl} | title="${title}" | flags=[${flags}] | body="${bodySnippet}"`;
+    } catch (e) {
+      return `diagnostics-failed: ${e.message}`;
+    }
+  }
+  captchaMessage() {
+    return this.kernelClient ? "Google returned a CAPTCHA on this Kernel.sh session \u2014 retrying with a fresh session." : RECAPTCHA_INSTRUCTIONS;
+  }
+  async navigateTo(url) {
+    try {
+      await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+    } catch (err) {
+      const diag = await this.captureDiagnostics(url);
+      throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
+    }
+  }
+  async navigateToChannel(channelHandle) {
+    const url = buildYouTubeChannelVideosUrl(channelHandle);
+    try {
+      await this.page.goto(url, { waitUntil: "networkidle", timeout: 3e4 });
+    } catch (err) {
+      const diag = await this.captureDiagnostics(url);
+      throw new ExtractionError(`navigateToChannel failed: ${err.message} | ${diag}`);
+    }
+  }
+  async evaluate(fn, arg) {
+    return this.page.evaluate(fn, arg);
+  }
+  getPage() {
+    return this.page;
+  }
+  async close() {
+    if (this.browser) {
+      const b = this.browser;
+      const sessionId = this.kernelSessionId;
+      const client = this.kernelClient;
+      this.browser = null;
+      this.context = null;
+      this.page = null;
+      this.kernelSessionId = null;
+      this.kernelClient = null;
+      try {
+        await b.close();
+      } finally {
+        if (client && sessionId) {
+          await client.browsers.deleteByID(sessionId).catch(
+            (err) => console.warn("Kernel session cleanup failed:", err)
+          );
+        }
+      }
+    } else if (this.context) {
+      const ctx = this.context;
+      this.context = null;
+      this.page = null;
+      await ctx.close();
+    }
+  }
+};
+
+// src/locations.ts
+var LOCATIONS = {
+  "austin": "Austin,Texas,United States",
+  "new york": "New York,New York,United States",
+  "new york city": "New York,New York,United States",
+  "nyc": "New York,New York,United States",
+  "los angeles": "Los Angeles,California,United States",
+  "la": "Los Angeles,California,United States",
+  "chicago": "Chicago,Illinois,United States",
+  "houston": "Houston,Texas,United States",
+  "phoenix": "Phoenix,Arizona,United States",
+  "philadelphia": "Philadelphia,Pennsylvania,United States",
+  "philly": "Philadelphia,Pennsylvania,United States",
+  "san antonio": "San Antonio,Texas,United States",
+  "dallas": "Dallas,Texas,United States",
+  "miami": "Miami,Florida,United States",
+  "seattle": "Seattle,Washington,United States",
+  "denver": "Denver,Colorado,United States",
+  "loveland": "Loveland,Colorado,United States",
+  "loveland co": "Loveland,Colorado,United States",
+  "fort collins": "Fort Collins,Colorado,United States",
+  "boulder": "Boulder,Colorado,United States",
+  "colorado springs": "Colorado Springs,Colorado,United States",
+  "boston": "Boston,Massachusetts,United States",
+  "atlanta": "Atlanta,Georgia,United States",
+  "san francisco": "San Francisco,California,United States",
+  "sf": "San Francisco,California,United States",
+  "portland": "Portland,Oregon,United States",
+  "las vegas": "Las Vegas,Nevada,United States",
+  "minneapolis": "Minneapolis,Minnesota,United States",
+  "detroit": "Detroit,Michigan,United States",
+  "nashville": "Nashville,Tennessee,United States",
+  "charlotte": "Charlotte,North Carolina,United States",
+  "orlando": "Orlando,Florida,United States",
+  "san diego": "San Diego,California,United States",
+  "baltimore": "Baltimore,Maryland,United States",
+  "sacramento": "Sacramento,California,United States",
+  "columbus": "Columbus,Ohio,United States",
+  "indianapolis": "Indianapolis,Indiana,United States",
+  "san jose": "San Jose,California,United States",
+  "fort worth": "Fort Worth,Texas,United States",
+  "jacksonville": "Jacksonville,Florida,United States",
+  "memphis": "Memphis,Tennessee,United States",
+  "louisville": "Louisville,Kentucky,United States",
+  "raleigh": "Raleigh,North Carolina,United States",
+  "richmond": "Richmond,Virginia,United States",
+  "salt lake city": "Salt Lake City,Utah,United States",
+  "toronto": "Toronto,Ontario,Canada",
+  "vancouver": "Vancouver,British Columbia,Canada",
+  "montreal": "Montreal,Quebec,Canada",
+  "calgary": "Calgary,Alberta,Canada",
+  "ottawa": "Ottawa,Ontario,Canada",
+  "london": "London,England,United Kingdom",
+  "manchester": "Manchester,England,United Kingdom",
+  "birmingham": "Birmingham,England,United Kingdom",
+  "edinburgh": "Edinburgh,Scotland,United Kingdom",
+  "glasgow": "Glasgow,Scotland,United Kingdom",
+  "leeds": "Leeds,England,United Kingdom",
+  "sydney": "Sydney,New South Wales,Australia",
+  "melbourne": "Melbourne,Victoria,Australia",
+  "brisbane": "Brisbane,Queensland,Australia",
+  "perth": "Perth,Western Australia,Australia",
+  "adelaide": "Adelaide,South Australia,Australia",
+  "dublin": "Dublin,Leinster,Ireland"
+};
+
+// src/uule.ts
+function encodeUule(name) {
+  const encoded = Buffer.from(String.fromCharCode(name.length) + name).toString("base64");
+  return `w+CAIQICI${encoded}`;
+}
+function normalizeLocation(input) {
+  const key = input.toLowerCase().trim();
+  return LOCATIONS[key] ?? input;
+}
+
+// src/lib/paa-answer-cleanup.ts
+var MAX_ANSWER_LENGTH = 1200;
+var BOILERPLATE_PATTERNS = [
+  /An AI Overview is not available for this search/gi,
+  /Can't generate an AI overview right now\.?\s*Try again later\.?/gi,
+  /\bAI Overview\b/gi,
+  /\bView all\b/gi
+];
+var CUT_MARKERS = [
+  /\bRelated Links\b/i,
+  /\bAsk anything in\s*AI Mode\b/i,
+  /\bAI can make mistakes\b/i,
+  /\bThis is for informational purposes only\b/i,
+  /\bShow more\b/i,
+  /\b\d+\s+sites\b/i,
+  /\b\d{1,2}\s*[msh]\s*[A-Z][A-Za-z]/,
+  /\b(?:YouTube|Reddit|Facebook|Instagram|TikTok)·/
+];
+function normalizeWhitespace(text) {
+  return text.replace(/\u00a0/g, " ").replace(/([.!?])([A-Z])/g, "$1 $2").replace(/([:;])([A-Z])/g, "$1 $2").replace(/([a-z])([A-Z][a-z])/g, "$1 $2").replace(/(\d)([A-Z][a-z])/g, "$1 $2").replace(/([a-z])(\d)/g, "$1 $2").replace(/\s+/g, " ").trim();
+}
+function cutAtFirstMarker(text) {
+  let cutAt = -1;
+  for (const marker of CUT_MARKERS) {
+    const match = marker.exec(text);
+    marker.lastIndex = 0;
+    if (match && (cutAt === -1 || match.index < cutAt)) cutAt = match.index;
+  }
+  return cutAt === -1 ? text : text.slice(0, cutAt);
+}
+function cutAtSourceTitle(text, sourceTitle) {
+  const title = sourceTitle?.trim();
+  if (!title || title.length < 8) return text;
+  const idx = text.toLowerCase().indexOf(title.toLowerCase());
+  return idx > 40 ? text.slice(0, idx) : text;
+}
+function findAttributionCut(beforeUrl) {
+  const dateMatch = beforeUrl.match(/[•·]\s*(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},\s+\d{4}/i);
+  if (dateMatch?.index && dateMatch.index > 40) return dateMatch.index;
+  const start = Math.max(0, beforeUrl.length - 260);
+  const tail = beforeUrl.slice(start);
+  const sentenceBreaks = [...tail.matchAll(/[.!?]\s*(?=[A-Z][A-Za-z0-9"'$])/g)];
+  for (const match of sentenceBreaks) {
+    const remainder = tail.slice(match.index + 1).trim();
+    const lead = remainder.slice(0, 160);
+    const looksLikeTitle = /^(?:Best|Top|What|How|Why|When|Where|Which|Can|Should|Is|Are|Do|Does)\b/i.test(remainder);
+    if (remainder.length > 20 && looksLikeTitle && /(?:\s[-|]\s|Heating|Cooling|Company|Services|Blog|Guide|Review)/i.test(lead)) {
+      return start + match.index + 1;
+    }
+  }
+  const last = sentenceBreaks.at(-1);
+  if (last?.index !== void 0) return start + last.index + 1;
+  return beforeUrl.length;
+}
+function cutAtUrlAttribution(text) {
+  const urlMatch = text.match(/https?:\/\/\S+/i);
+  if (!urlMatch?.index) return text;
+  const beforeUrl = text.slice(0, urlMatch.index);
+  return beforeUrl.slice(0, findAttributionCut(beforeUrl));
+}
+function trimToSentenceLimit(text) {
+  if (text.length <= MAX_ANSWER_LENGTH) return text;
+  const slice = text.slice(0, MAX_ANSWER_LENGTH);
+  const lastSentence = Math.max(slice.lastIndexOf("."), slice.lastIndexOf("!"), slice.lastIndexOf("?"));
+  return (lastSentence > 240 ? slice.slice(0, lastSentence + 1) : slice).trim();
+}
+function cleanPAAAnswerText(answer, question, sourceTitle) {
+  if (!answer) return void 0;
+  let text = normalizeWhitespace(answer);
+  const normalizedQuestion = question ? normalizeWhitespace(question) : "";
+  if (normalizedQuestion && text.toLowerCase().startsWith(normalizedQuestion.toLowerCase())) {
+    text = text.slice(normalizedQuestion.length).trim();
+  }
+  if (/^An error has occurred\.?\s*Please try again later\.?/i.test(text)) {
+    return void 0;
+  }
+  for (const pattern of BOILERPLATE_PATTERNS) {
+    text = text.replace(pattern, " ");
+  }
+  text = text.replace(/\b[A-Z][A-Za-z&'\u2019 -]{2,60}\+\d+\b/g, " ").replace(/\b(?:[a-z0-9-]+\.)+[a-z]{2,}\+\d+\b/gi, " ");
+  text = normalizeWhitespace(text);
+  text = cutAtFirstMarker(text);
+  text = cutAtSourceTitle(text, sourceTitle);
+  text = cutAtUrlAttribution(text);
+  text = normalizeWhitespace(text);
+  text = text.replace(/\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},\s+\d{4}$/i, "").trim();
+  text = trimToSentenceLimit(text);
+  if (!text) return void 0;
+  if (/^An error has occurred\.?\s*Please try again later\.?$/i.test(text)) return void 0;
+  return text;
+}
+
+// src/extractor/PAAExtractor.ts
+var PAAExtractor = class {
+  constructor(driver, reporter) {
+    this.driver = driver;
+    this.reporter = reporter;
+  }
+  driver;
+  reporter;
+  normalizeQuestion(q) {
+    return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
+  }
+  async extractVisibleItems(page) {
+    const sels = PAASelectors;
+    const raw = await page.evaluate((selectors) => {
+      function cleanText(el) {
+        if (!el) return "";
+        const parts = [];
+        for (const n of el.childNodes) {
+          if (n.nodeType === Node.TEXT_NODE) {
+            const text = n.textContent?.trim();
+            if (text) parts.push(text);
+          } else if (n.tagName === "STYLE" || n.tagName === "SCRIPT") {
+            continue;
+          } else {
+            const text = cleanText(n);
+            if (text) parts.push(text);
+          }
+        }
+        return parts.join(" ").replace(/\s+/g, " ").trim();
+      }
+      return Array.from(document.querySelectorAll(selectors.item)).map((pair) => ({
+        question: pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || "",
+        answer: cleanText(pair.querySelector(selectors.answerContainer)) || void 0,
+        sourceTitle: pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0,
+        sourceSite: pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0,
+        sourceCite: pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0
+      }));
+    }, sels);
+    return raw.flatMap((item) => {
+      const cleaned = {
+        ...item,
+        answer: cleanPAAAnswerText(item.answer, item.question, item.sourceTitle)
+      };
+      const result = RawPAAItemSchema.safeParse(cleaned);
+      if (!result.success) {
+        console.warn("[PAAExtractor] item parse failed:", item.question, result.error.issues[0]?.message);
+        return [];
+      }
+      return [result.data];
+    });
+  }
+  async clickItem(page, questionText) {
+    try {
+      const pairLocator = page.locator(
+        `${PAASelectors.item}[data-q="${questionText}"], ${PAASelectors.item}[data-initq="${questionText}"]`
+      ).first();
+      await pairLocator.click();
+    } catch {
+    }
+  }
+  toFlatRow(item, depth, parentQuestion, seed) {
+    return {
+      seed_query: seed,
+      question: item.question,
+      answer: item.answer ?? "",
+      source_title: item.sourceTitle ?? "",
+      source_site: item.sourceSite ?? "",
+      source_cite: item.sourceCite ?? "",
+      depth,
+      parent_question: parentQuestion ?? "",
+      extracted_at: (/* @__PURE__ */ new Date()).toISOString()
+    };
+  }
+  async runBFS(page, options) {
+    const seenKeys = /* @__PURE__ */ new Set();
+    const seenQs = /* @__PURE__ */ new Set();
+    const depthMap = /* @__PURE__ */ new Map();
+    const results = [];
+    const readAllQs = () => page.evaluate(
+      ({ sel, dataQ, dataInitQ, questionEl }) => Array.from(document.querySelectorAll(sel)).map(
+        (el) => el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || ""
+      ).filter(Boolean),
+      { sel: PAASelectors.item, dataQ: PAASelectors.itemDataQ, dataInitQ: PAASelectors.itemDataInitQ, questionEl: PAASelectors.itemQuestionEl }
+    );
+    const dupRates = [];
+    const orderedQs = [];
+    for (let round = 0; round < options.depth; round++) {
+      this.reporter.onDepth(round + 1);
+      if (seenQs.size >= options.maxQuestions) break;
+      const beforeQs = await readAllQs();
+      if (beforeQs.length >= options.maxQuestions) break;
+      const unexpandedItems = await page.$$(
+        `${PAASelectors.item}:not(.${PAASelectors.expandedClass})`
+      );
+      if (unexpandedItems.length === 0) break;
+      for (const item of unexpandedItems) {
+        try {
+          await item.scrollIntoViewIfNeeded();
+          await item.click({ force: true });
+          await page.waitForTimeout(500);
+        } catch {
+        }
+      }
+      await page.waitForTimeout(1500);
+      const afterQs = await readAllQs();
+      const newQs = afterQs.slice(beforeQs.length);
+      const newDups = newQs.filter((q) => seenQs.has(q)).length;
+      const dupRate = newQs.length > 0 ? newDups / newQs.length : 0;
+      dupRates.push(dupRate);
+      if (dupRates.length > 2) dupRates.shift();
+      const rollingDupRate = dupRates.reduce((a, b) => a + b, 0) / dupRates.length;
+      for (const q of afterQs) {
+        if (!seenQs.has(q)) {
+          seenQs.add(q);
+          orderedQs.push(q);
+        }
+        if (!depthMap.has(q)) depthMap.set(q, round + 1);
+      }
+      if (afterQs.length === beforeQs.length) break;
+      if (rollingDupRate >= 0.6) break;
+    }
+    const itemMap = new Map((await this.extractVisibleItems(page)).map((i) => [i.question, i]));
+    for (const q of orderedQs) {
+      if (results.length >= options.maxQuestions) break;
+      const key = this.normalizeQuestion(q);
+      if (seenKeys.has(key)) continue;
+      seenKeys.add(key);
+      const d = depthMap.get(q) ?? 1;
+      const item = itemMap.get(q);
+      if (item) {
+        results.push(this.toFlatRow(item, d, null, options.query));
+        this.reporter.onQuestion({ question: item.question, answer: item.answer ?? null, sourceTitle: item.sourceTitle ?? null, sourceSite: item.sourceSite ?? null, sourceCite: item.sourceCite ?? null, depth: d, parentQuestion: null, children: [] });
+      } else {
+        results.push(this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, d, null, options.query));
+      }
+    }
+    return results;
+  }
+  async extractVideos(page) {
+    const vsels = VideoSelectors;
+    return page.evaluate((sels) => {
+      const results = [];
+      const containers = Array.from(document.querySelectorAll(sels.container));
+      for (const container of containers) {
+        const headingEl = container.querySelector(sels.sectionHeading);
+        const headingText = headingEl?.textContent?.trim() ?? "";
+        const type = headingText.toLowerCase().includes("short") ? "short_video" : "video";
+        const items = Array.from(container.querySelectorAll(sels.item));
+        for (const a of items) {
+          const href = a.href;
+          if (!href || !href.includes("youtube") && !href.includes("youtu.be")) continue;
+          const raw = a.textContent?.trim() ?? "";
+          const ytIdx = raw.indexOf("YouTube");
+          if (ytIdx === -1) continue;
+          const title = raw.slice(0, ytIdx).trim();
+          const remainder = raw.slice(ytIdx + 7).replace(/^[·\s·]+/, "");
+          const channelMatch = remainder.match(/^([^·\n]+)/);
+          const channel = channelMatch ? channelMatch[1].trim() : "";
+          if (title) results.push({ type, title, channel, platform: "YouTube", duration: "", url: href });
+        }
+      }
+      return results;
+    }, vsels);
+  }
+  async extractForums(page) {
+    const fsels = ForumSelectors;
+    return page.evaluate((sels) => {
+      const results = [];
+      const sections = Array.from(document.querySelectorAll(sels.section));
+      const forumSection = sections.find((s) => s.textContent?.includes("Discussions"));
+      if (!forumSection) return results;
+      const items = Array.from(forumSection.querySelectorAll(sels.item));
+      for (const a of items) {
+        const href = a.href;
+        if (!href) continue;
+        const titleEl = a.querySelector(sels.title);
+        const sourceEl = a.querySelector(sels.source);
+        const title = titleEl?.textContent?.trim() ?? "";
+        const source = sourceEl?.textContent?.trim() ?? "";
+        if (title) results.push({ title, source, url: href });
+      }
+      return results;
+    }, fsels);
+  }
+  async extractShortVideos(page, shortUrl) {
+    try {
+      await page.goto(shortUrl, { waitUntil: "domcontentloaded" });
+      await page.waitForTimeout(1500);
+    } catch {
+      return [];
+    }
+    const svSels = {
+      item: ShortVideoSelectors.item,
+      platforms: [...ShortVideoSelectors.platforms]
+    };
+    const raw = await page.evaluate((sels) => {
+      const seen = /* @__PURE__ */ new Set();
+      const results = [];
+      const items = Array.from(document.querySelectorAll(sels.item));
+      const videoHosts = ["youtube.com", "youtu.be", "tiktok.com", "instagram.com", "facebook.com", "fb.watch"];
+      const byHref = /* @__PURE__ */ new Map();
+      for (const a of items) {
+        const href = a.href;
+        if (!href) continue;
+        if (!videoHosts.some((h) => href.includes(h))) continue;
+        const text = a.textContent?.trim() ?? "";
+        if (!byHref.has(href)) byHref.set(href, []);
+        byHref.get(href).push(text);
+      }
+      for (const [href, texts] of byHref.entries()) {
+        if (seen.has(href)) continue;
+        seen.add(href);
+        const duration = texts.find((t) => /^\d+:\d+$/.test(t)) ?? "";
+        const titleText = texts.find((t) => !/^\d+:\d+$/.test(t) && t.length > 5) ?? "";
+        if (!titleText) continue;
+        let title = titleText;
+        let platform = "";
+        let channel = "";
+        for (const p of sels.platforms) {
+          let lastIdx = -1;
+          let search = 0;
+          while (true) {
+            const found = titleText.indexOf(p, search);
+            if (found === -1) break;
+            lastIdx = found;
+            search = found + 1;
+          }
+          if (lastIdx === -1) continue;
+          const after = titleText.slice(lastIdx + p.length);
+          const isSourceTag = /^[\s·]/.test(after) || after.trim() === "";
+          if (!isSourceTag) continue;
+          title = titleText.slice(0, lastIdx).trim();
+          platform = p;
+          const stripped = after.replace(/^[\s·]+/, "");
+          const dotIdx = stripped.indexOf("\xB7");
+          channel = (dotIdx === -1 ? stripped : stripped.slice(0, dotIdx)).trim();
+          break;
+        }
+        if (title) results.push({ title, channel, platform, duration, url: href });
+      }
+      return results;
+    }, svSels);
+    return raw.map((r) => ({ type: "short_video", ...r }));
+  }
+  async extractWhatPeopleSaying(page) {
+    const sels = WhatPeopleSayingSelectors;
+    return page.evaluate((s) => {
+      const section = Array.from(document.querySelectorAll(s.sectionTag)).find((el) => el.textContent?.includes(s.sectionHeadingText)) ?? document.querySelector(".yG4QQe.TBC9ub.NbhJ1c");
+      if (!section) return [];
+      return Array.from(section.querySelectorAll(s.card)).map((card) => {
+        const link = card.querySelector(s.cardLink);
+        const url = link?.href ?? "";
+        const titleH1 = card.querySelector(s.titleH1)?.textContent?.trim();
+        const titleDiv = card.querySelector(s.titleDiv)?.textContent?.trim();
+        const title = titleH1 ?? titleDiv ?? "";
+        const sourceText = card.querySelector(s.source)?.textContent?.trim() ?? "";
+        const platformEl = card.querySelector(s.platformBadge);
+        const platformText = platformEl?.textContent?.trim() ?? "";
+        const ytChannel = card.querySelector(s.ytChannel)?.textContent?.trim() ?? "";
+        const ytDate = card.querySelector(s.ytDate)?.textContent?.trim() ?? "";
+        const authorNote = card.querySelector(s.authorNote)?.textContent?.trim() ?? null;
+        const commentLabelEl = card.querySelector(s.popularCommentLabel);
+        let popularComment = null;
+        if (commentLabelEl) {
+          let next = commentLabelEl.nextSibling;
+          while (next) {
+            const t = next.textContent?.trim();
+            if (t) {
+              popularComment = t;
+              break;
+            }
+            next = next.nextSibling;
+          }
+        }
+        const allSpans = Array.from(card.querySelectorAll("span"));
+        const duration = allSpans.find((s2) => /^\d+:\d+$/.test(s2.textContent?.trim() ?? ""))?.textContent?.trim() ?? null;
+        const engagementParts = allSpans.map((s2) => s2.textContent?.trim() ?? "").filter(
+          (t) => /\d/.test(t) && (t.includes("comment") || t.includes("reaction") || t.includes("view") || t.includes("like") || t.includes("share"))
+        );
+        const engagement = engagementParts[0] ?? "";
+        const dateCandidates = allSpans.map((s2) => s2.textContent?.trim() ?? "").filter((t) => /\d+ (day|week|month|year|hour)s? ago|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec/.test(t));
+        const date = ytDate || (dateCandidates[0] ?? "");
+        const platform = platformText || (ytChannel ? "YouTube" : "");
+        const source = ytChannel || sourceText;
+        let type = "unknown";
+        const pl = platform.toLowerCase();
+        const src = source.toLowerCase();
+        const srcRaw = sourceText.toLowerCase();
+        if (pl.includes("reddit") || src.startsWith("r/")) type = "reddit";
+        else if (pl.includes("facebook") || srcRaw.includes("facebook")) type = "facebook";
+        else if (pl.includes("instagram") || srcRaw.includes("instagram")) type = "instagram";
+        else if (pl.includes("tiktok") || srcRaw.includes("tiktok")) type = "tiktok";
+        else if (pl.includes("youtube") || !!ytChannel) type = "youtube";
+        else type = "news";
+        return { type, title, url, source, platform, popularComment, engagement, date, duration, authorNote };
+      });
+    }, sels);
+  }
+  async extractOrganicResults(page) {
+    const sels = OrganicSelectors;
+    return page.evaluate((s) => {
+      const out = [];
+      let pos = 0;
+      document.querySelectorAll(s.result).forEach((card) => {
+        const titleEl = card.querySelector(s.title);
+        if (!titleEl) return;
+        const title = titleEl.textContent?.trim() ?? "";
+        const linkEl = titleEl.closest("a");
+        const url = linkEl?.href ?? "";
+        if (!title || !url) return;
+        pos++;
+        const cite = card.querySelector(s.cite)?.textContent?.trim() ?? null;
+        const snippet = card.querySelector(s.snippet)?.textContent?.trim() ?? null;
+        const isRedditStyle = !!card.querySelector(s.redditCite);
+        const ratingEl = card.querySelector(s.ratingWrap);
+        const inlineRating = ratingEl ? { value: ratingEl.querySelector(s.ratingValue)?.textContent?.trim() ?? "", count: ratingEl.querySelector(s.reviewCount)?.textContent?.trim() ?? "" } : null;
+        let domain = "";
+        try {
+          domain = new URL(url).hostname.replace(/^www\./, "");
+        } catch {
+          domain = card.querySelector(s.siteName)?.textContent?.trim() ?? "";
+        }
+        out.push({ position: pos, title, url, domain, cite, snippet, isRedditStyle, inlineRating });
+      });
+      return out;
+    }, sels);
+  }
+  async extractLocalPack(page) {
+    const sels = LocalPackSelectors;
+    return page.evaluate((s) => {
+      const out = [];
+      let container = null;
+      document.querySelectorAll('[role="heading"]').forEach((h) => {
+        if (!container && h.textContent?.includes(s.headingText)) container = h.closest("[data-hveid]");
+      });
+      if (!container) return out;
+      container.querySelectorAll(s.card).forEach((card, i) => {
+        const name = card.querySelector(s.name)?.textContent?.trim() ?? "";
+        if (!name) return;
+        const rating = card.querySelector(s.ratingValue)?.textContent?.trim() ?? null;
+        const reviewRaw = card.querySelector(s.reviewCount)?.textContent?.trim() ?? null;
+        const reviewCount = reviewRaw ? reviewRaw.replace(/[()]/g, "").trim() : null;
+        let cid = card.querySelector("a[data-cid]")?.getAttribute("data-cid") ?? null;
+        if (!cid) {
+          for (const link of Array.from(card.querySelectorAll("a[href]"))) {
+            const m1 = link.href.match(/[?&]cid=(\d+)/);
+            if (m1) {
+              cid = m1[1];
+              break;
+            }
+            const m2 = link.href.match(/!1s0x[0-9a-f]+:0x([0-9a-f]+)/i);
+            if (m2) {
+              try {
+                cid = BigInt("0x" + m2[1]).toString();
+              } catch {
+              }
+              if (cid) break;
+            }
+          }
+        }
+        const metadata = [];
+        card.querySelectorAll("div, span").forEach((el) => {
+          const text = Array.from(el.childNodes).filter((n) => n.nodeType === 3).map((n) => n.textContent?.trim() ?? "").filter((t) => t.length > 1 && t.length < 120).join(" ");
+          if (text && !metadata.includes(text)) metadata.push(text);
+        });
+        const links = Array.from(card.querySelectorAll("a[href]"));
+        const directionsUrl = links.find((a) => a.href.includes("google.com/maps"))?.href ?? null;
+        const websiteUrl = links.find((a) => !a.href.includes("google.com") && a.href.startsWith("http"))?.href ?? null;
+        out.push({ position: i + 1, name, cid, rating, reviewCount, metadata, websiteUrl, directionsUrl });
+      });
+      return out;
+    }, sels);
+  }
+  async extractEntityIds(page) {
+    return page.evaluate(() => {
+      const kgIds = /* @__PURE__ */ new Set();
+      const cids = /* @__PURE__ */ new Set();
+      const gcids = /* @__PURE__ */ new Set();
+      const recordMap = /* @__PURE__ */ new Map();
+      function nameFromWrapper(el) {
+        const sel = [".OSrXXb", ".dbg0pd", ".tzt0oe", '[role="heading"]', "h3"];
+        for (const s of sel) {
+          const found = el.querySelector(s);
+          if (found?.textContent?.trim()) return found.textContent.trim();
+        }
+        return "";
+      }
+      document.querySelectorAll('[id^="pv-/g/"]').forEach((wrapper) => {
+        const raw = wrapper.getAttribute("id");
+        if (!raw) return;
+        const kgId = raw.replace("pv-", "");
+        kgIds.add(kgId);
+        const name = nameFromWrapper(wrapper);
+        const cidEl = wrapper.querySelector("a[data-cid]");
+        const cid = cidEl?.getAttribute("data-cid") ?? null;
+        if (cid) cids.add(cid);
+        if (name) recordMap.set(kgId, { name, kgId, cid, gcid: null });
+      });
+      document.querySelectorAll("[data-mid]").forEach((el) => {
+        const mid = el.getAttribute("data-mid");
+        if (!mid?.startsWith("/g/")) return;
+        kgIds.add(mid);
+        if (!recordMap.has(mid)) {
+          const name = nameFromWrapper(el);
+          if (name) recordMap.set(mid, { name, kgId: mid, cid: null, gcid: null });
+        }
+      });
+      document.querySelectorAll(".w7Dbne").forEach((card) => {
+        const cidEl = card.querySelector("a[data-cid]");
+        const cid = cidEl?.getAttribute("data-cid") ?? null;
+        if (!cid) return;
+        cids.add(cid);
+        const name = card.querySelector(".OSrXXb")?.textContent?.trim() ?? "";
+        if (!name) return;
+        const kgIdEl = card.querySelector('[id^="pv-/g/"]');
+        const kgId = kgIdEl ? kgIdEl.getAttribute("id").replace("pv-", "") : null;
+        const key = kgId ?? `cid:${cid}`;
+        if (recordMap.has(key)) {
+          const existing = recordMap.get(key);
+          if (!existing.cid) recordMap.set(key, { ...existing, cid });
+        } else {
+          recordMap.set(key, { name, kgId, cid, gcid: null });
+        }
+      });
+      document.querySelectorAll("a[data-cid]").forEach((el) => {
+        const cid = el.getAttribute("data-cid");
+        if (!cid) return;
+        cids.add(cid);
+        const alreadyNamed = [...recordMap.values()].some((r) => r.cid === cid);
+        if (!alreadyNamed) {
+          let node = el.parentElement;
+          let name = "";
+          for (let i = 0; i < 8 && node; i++) {
+            const h = node.querySelector('.OSrXXb, .dbg0pd, [role="heading"], h3');
+            if (h?.textContent?.trim()) {
+              name = h.textContent.trim();
+              break;
+            }
+            node = node.parentElement;
+          }
+          if (name) recordMap.set(`cid:${cid}`, { name, kgId: null, cid, gcid: null });
+        }
+      });
+      const scriptContent = Array.from(document.querySelectorAll("script:not([src])")).map((s) => s.textContent ?? "").filter((t) => t.length > 1e4).join("\n");
+      for (const m of scriptContent.matchAll(/\/g\/[a-zA-Z0-9_-]{5,20}/g)) kgIds.add(m[0]);
+      for (const m of scriptContent.matchAll(/gcid:[a-zA-Z0-9_]+/g)) gcids.add(m[0]);
+      for (const m of scriptContent.matchAll(/0x[0-9a-f]+:0x([0-9a-f]+)/gi)) {
+        try {
+          cids.add(BigInt("0x" + m[1]).toString());
+        } catch {
+        }
+      }
+      return { entities: [...recordMap.values()], kgIds: [...kgIds], cids: [...cids], gcids: [...gcids] };
+    });
+  }
+  mergeLocalPackIntoEntities(entityIds, localPack) {
+    const cidSet = new Set(entityIds.cids);
+    const records = entityIds.entities.map((r) => ({ ...r }));
+    for (const biz of localPack) {
+      if (!biz.cid) continue;
+      cidSet.add(biz.cid);
+      const nameNorm = biz.name.toLowerCase().trim();
+      const byName = records.find((r) => r.name.toLowerCase().trim() === nameNorm);
+      if (byName) {
+        if (!byName.cid) byName.cid = biz.cid;
+      } else if (!records.find((r) => r.cid === biz.cid)) {
+        records.push({ name: biz.name, kgId: null, cid: biz.cid, gcid: null });
+      }
+    }
+    return { ...entityIds, entities: records, cids: [...cidSet] };
+  }
+  async extractAISurfaces(page) {
+    const aioSels = AIOverviewSelectors;
+    const aimSels = AIModeSelectors;
+    return page.evaluate(({ aio, aim }) => {
+      const sn = window.google?.sn ?? "unknown";
+      const surface = sn === "aim" ? "aim" : sn === "web" ? "web" : "unknown";
+      function findAIORoot() {
+        const primary = document.querySelector(aio.root);
+        if (primary) return primary;
+        const headings = document.querySelectorAll('h1, h2, h3, [role="heading"]');
+        for (const h of headings) {
+          if (h.textContent?.trim() === "AI Overview") {
+            let el = h.parentElement;
+            for (let i = 0; i < 6 && el; i++) {
+              if (el.querySelectorAll("a").length > 1) return el;
+              el = el.parentElement;
+            }
+            return h.parentElement;
+          }
+        }
+        return null;
+      }
+      const aioRoot = findAIORoot();
+      const aioContainer = aioRoot ? aioRoot.closest(aio.wrapper) ?? aioRoot : null;
+      let aioText = null;
+      if (aioContainer) {
+        const clone = aioContainer.cloneNode(true);
+        clone.querySelectorAll("script,style,noscript").forEach((el) => el.remove());
+        clone.querySelectorAll('h1,h2,h3,h4,[role="heading"]').forEach((el) => el.remove());
+        clone.querySelectorAll('button,[role="button"]').forEach((el) => el.remove());
+        clone.querySelectorAll("a").forEach((el) => el.remove());
+        const candidate = clone.textContent?.replace(/\s+/g, " ").trim() || null;
+        const isErrorState = !candidate || /not available|try again|can't generate/i.test(candidate);
+        aioText = isErrorState ? null : candidate;
+      }
+      const aioDetected = !!aioRoot && aioText !== null;
+      const aioCitations = Array.from(aioContainer?.querySelectorAll("a[href]") ?? []).filter((a) => a.href && !a.href.startsWith("javascript")).map((a) => ({
+        text: a.textContent?.trim() ?? "",
+        href: a.href
+      })).filter((c) => c.text && c.href);
+      const aimRoot = document.querySelector(aim.root);
+      const aimDetected = surface === "aim" && !!aimRoot;
+      const aimContainer = aimRoot?.closest(aim.wrapper) ?? null;
+      let aimText = null;
+      if (aimContainer) {
+        const clone = aimContainer.cloneNode(true);
+        clone.querySelectorAll("script,style,noscript").forEach((el) => el.remove());
+        clone.querySelectorAll('h1,h2,h3,h4,[role="heading"]').forEach((el) => el.remove());
+        clone.querySelectorAll('button,[role="button"]').forEach((el) => el.remove());
+        clone.querySelectorAll("a").forEach((el) => el.remove());
+        const candidate = clone.textContent?.replace(/\s+/g, " ").trim() || null;
+        const isErrorState = !candidate || /not available|try again|can't generate/i.test(candidate);
+        aimText = isErrorState ? null : candidate;
+      }
+      const aimCitations = aimDetected ? Array.from(aimContainer?.querySelectorAll("a[href]") ?? []).filter((a) => a.href && !a.href.startsWith("javascript")).map((a) => ({
+        text: a.textContent?.trim() ?? "",
+        href: a.href
+      })).filter((c) => c.text && c.href) : [];
+      return {
+        surface,
+        aiOverview: { detected: aioDetected, text: aioText, citations: aioCitations },
+        aiMode: { detected: aimDetected, text: aimText, citations: aimCitations }
+      };
+    }, { aio: aioSels, aim: aimSels });
+  }
+  buildTree(flat, _seed) {
+    const roots = [];
+    const nodeMap = /* @__PURE__ */ new Map();
+    for (const row of flat) {
+      const node = {
+        question: row.question,
+        answer: row.answer || null,
+        sourceTitle: row.source_title || null,
+        sourceSite: row.source_site || null,
+        sourceCite: row.source_cite || null,
+        depth: row.depth,
+        parentQuestion: row.parent_question || null,
+        children: []
+      };
+      nodeMap.set(row.question, node);
+    }
+    for (const node of nodeMap.values()) {
+      if (node.parentQuestion && nodeMap.has(node.parentQuestion)) {
+        nodeMap.get(node.parentQuestion).children.push(node);
+      } else {
+        roots.push(node);
+      }
+    }
+    return roots;
+  }
+  async extract(options) {
+    const startMs = Date.now();
+    const config = {
+      headless: options.headless,
+      profileDir: options.profileDir,
+      proxy: options.proxy,
+      kernelApiKey: options.kernelApiKey,
+      kernelProxyId: options.kernelProxyId,
+      viewport: { width: 1280, height: 800 },
+      locale: `${options.hl}-${options.gl.toUpperCase()}`
+    };
+    let errorCount = 0;
+    try {
+      await this.driver.launch(config);
+      const uule = options.location ? encodeUule(normalizeLocation(options.location)) : null;
+      const { hasPaa } = await this.driver.navigateToSERP(options.query, uule, options.gl, options.hl);
+      const page = this.driver.getPage();
+      if (options.serpOnly) {
+        const [organicResults2, localPack2, rawEntityIds2] = await Promise.all([
+          this.extractOrganicResults(page),
+          this.extractLocalPack(page),
+          this.extractEntityIds(page)
+        ]);
+        const entityIds2 = this.mergeLocalPackIntoEntities(rawEntityIds2, localPack2);
+        let allOrganic2 = organicResults2;
+        if ((options.pages ?? 1) >= 2) {
+          const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, start: "10" });
+          if (uule) p2params.set("uule", uule);
+          await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
+          const p2organic = await this.extractOrganicResults(page);
+          allOrganic2 = [...organicResults2, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
+        }
+        const stats2 = {
+          seed: options.query,
+          totalQuestions: 0,
+          maxDepthReached: 0,
+          durationMs: Date.now() - startMs,
+          errorCount
+        };
+        this.reporter.onComplete(stats2);
+        return {
+          seed: options.query,
+          location: options.location ?? null,
+          extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          totalQuestions: 0,
+          surface: "web",
+          aiOverview: { detected: false, text: null, citations: [] },
+          aiMode: { detected: false, text: null, citations: [] },
+          whatPeopleSaying: [],
+          tree: [],
+          flat: [],
+          videos: [],
+          forums: [],
+          organicResults: allOrganic2,
+          localPack: localPack2,
+          entityIds: entityIds2,
+          stats: stats2
+        };
+      }
+      const [videos, forums, whatPeopleSaying, rawEntityIds, organicResults, localPack] = await Promise.all([
+        this.extractVideos(page),
+        this.extractForums(page),
+        this.extractWhatPeopleSaying(page),
+        this.extractEntityIds(page),
+        this.extractOrganicResults(page),
+        this.extractLocalPack(page)
+      ]);
+      const entityIds = this.mergeLocalPackIntoEntities(rawEntityIds, localPack);
+      this.reporter.onVideos(videos);
+      this.reporter.onForums(forums);
+      if (!hasPaa) {
+        let noPaaOrganic = organicResults;
+        if ((options.pages ?? 1) >= 2) {
+          const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, start: "10" });
+          if (uule) p2params.set("uule", uule);
+          await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
+          const p2organic = await this.extractOrganicResults(page);
+          noPaaOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
+        }
+        const aiSurfaces2 = await this.extractAISurfaces(page);
+        const stats2 = {
+          seed: options.query,
+          totalQuestions: 0,
+          maxDepthReached: 0,
+          durationMs: Date.now() - startMs,
+          errorCount
+        };
+        this.reporter.onComplete(stats2);
+        return {
+          seed: options.query,
+          location: options.location ?? null,
+          extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          totalQuestions: 0,
+          surface: aiSurfaces2.surface,
+          aiOverview: aiSurfaces2.aiOverview,
+          aiMode: aiSurfaces2.aiMode,
+          whatPeopleSaying,
+          tree: [],
+          flat: [],
+          videos,
+          forums,
+          organicResults: noPaaOrganic,
+          localPack,
+          entityIds,
+          stats: stats2
+        };
+      }
+      const flat = await this.runBFS(page, options);
+      const aiSurfaces = await this.extractAISurfaces(page);
+      const shortVidsParams = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, udm: ShortVideoSelectors.udm });
+      if (uule) shortVidsParams.set("uule", uule);
+      const shortVideos = await this.extractShortVideos(page, "https://www.google.com/search?" + shortVidsParams.toString());
+      this.reporter.onVideos(shortVideos);
+      let allOrganic = organicResults;
+      if ((options.pages ?? 1) >= 2) {
+        const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, start: "10" });
+        if (uule) p2params.set("uule", uule);
+        await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
+        const p2organic = await this.extractOrganicResults(page);
+        allOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
+      }
+      const allVideos = [...videos, ...shortVideos];
+      const tree = this.buildTree(flat, options.query);
+      const stats = {
+        seed: options.query,
+        totalQuestions: flat.length,
+        maxDepthReached: flat.reduce((m, r) => Math.max(m, r.depth), 0),
+        durationMs: Date.now() - startMs,
+        errorCount
+      };
+      this.reporter.onComplete(stats);
+      return {
+        seed: options.query,
+        location: options.location ?? null,
+        extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        totalQuestions: flat.length,
+        surface: aiSurfaces.surface,
+        aiOverview: aiSurfaces.aiOverview,
+        aiMode: aiSurfaces.aiMode,
+        whatPeopleSaying,
+        tree,
+        flat,
+        videos: allVideos,
+        forums,
+        organicResults: allOrganic,
+        localPack,
+        entityIds,
+        stats
+      };
+    } catch (err) {
+      errorCount++;
+      this.reporter.onError(err instanceof Error ? err : new Error(String(err)));
+      throw err;
+    } finally {
+      await this.driver.close();
+    }
+  }
+};
+
+// src/output/OutputSerializer.ts
+var import_node_fs = require("fs");
+var import_node_path = __toESM(require("path"), 1);
+var import_papaparse = __toESM(require("papaparse"), 1);
+var OutputSerializer = class {
+  async writeJSON(result, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = result.seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const filename = `${slug}-${Date.now()}.json`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, JSON.stringify(result, null, 2), "utf8");
+    return fullPath;
+  }
+  async writeCSV(rows, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const seedRaw = rows[0]?.seed_query ?? "paa";
+    const slug = seedRaw.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const csv = import_papaparse.default.unparse(rows, { header: true });
+    const filename = `${slug}-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+  async writeVideoCSV(videos, seed, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const csv = import_papaparse.default.unparse(videos, { header: true });
+    const filename = `${slug}-videos-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+  async writeForumCSV(forums, seed, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const csv = import_papaparse.default.unparse(forums, { header: true });
+    const filename = `${slug}-forums-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+  async writeAIOverviewCSV(citations, text, seed, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const rows = citations.map((c, i) => ({
+      seed_query: seed,
+      response_text: i === 0 ? text ?? "" : "",
+      citation_text: c.text,
+      citation_href: c.href
+    }));
+    const csv = import_papaparse.default.unparse(rows, { header: true });
+    const filename = `${slug}-ai-overview-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+  async writeAIModeCSV(citations, text, seed, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const rows = citations.map((c, i) => ({
+      seed_query: seed,
+      response_text: i === 0 ? text ?? "" : "",
+      citation_text: c.text,
+      citation_href: c.href
+    }));
+    const csv = import_papaparse.default.unparse(rows, { header: true });
+    const filename = `${slug}-ai-mode-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+  async writeWhatPeopleSayingCSV(cards, seed, outputDir) {
+    await import_node_fs.promises.mkdir(outputDir, { recursive: true });
+    const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
+    const rows = cards.map((c) => ({ seed_query: seed, ...c }));
+    const csv = import_papaparse.default.unparse(rows, { header: true });
+    const filename = `${slug}-what-people-saying-${Date.now()}.csv`;
+    const fullPath = import_node_path.default.join(outputDir, filename);
+    await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
+    return fullPath;
+  }
+};
+
+// src/output/ProgressReporter.ts
+var ProgressReporter = class {
+  onQuestion(node) {
+    process.stdout.write(JSON.stringify({ event: "question", depth: node.depth, question: node.question }) + "\n");
+  }
+  onDepth(depth) {
+    process.stdout.write(JSON.stringify({ event: "depth", depth }) + "\n");
+  }
+  onVideos(videos) {
+    for (const v of videos) {
+      process.stdout.write(JSON.stringify({ event: "video", type: v.type, platform: v.platform, duration: v.duration, title: v.title, channel: v.channel, url: v.url }) + "\n");
+    }
+  }
+  onForums(forums) {
+    for (const f of forums) {
+      process.stdout.write(JSON.stringify({ event: "forum", title: f.title, source: f.source, url: f.url }) + "\n");
+    }
+  }
+  onComplete(stats) {
+    process.stdout.write(JSON.stringify({ event: "complete", ...stats }) + "\n");
+  }
+  onError(err) {
+    process.stderr.write(JSON.stringify({ event: "error", type: err.constructor.name, message: err.message }) + "\n");
+  }
+};
+
+// src/harvest.ts
+var MAX_ATTEMPTS = 3;
+async function extractOnce(options) {
+  const driver = new BrowserDriver();
+  const reporter = new ProgressReporter();
+  const extractor = new PAAExtractor(driver, reporter);
+  try {
+    return await extractor.extract(options);
+  } finally {
+    await driver.close();
+  }
+}
+async function harvest(rawOptions) {
+  const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};
+  const merged = {
+    kernelApiKey: process.env.KERNEL_API_KEY?.trim(),
+    kernelProxyId: process.env.KERNEL_PROXY_ID?.trim(),
+    ...raw
+  };
+  const options = HarvestOptionsSchema.parse(merged);
+  const serializer = new OutputSerializer();
+  for (let i = 0; i < MAX_ATTEMPTS; i++) {
+    try {
+      const result = await extractOnce(options);
+      if (options.format === "json" || options.format === "both") {
+        await serializer.writeJSON(result, options.outputDir);
+      }
+      if (options.format === "csv" || options.format === "both") {
+        await Promise.all([
+          serializer.writeCSV(result.flat, options.outputDir),
+          result.videos.length > 0 ? serializer.writeVideoCSV(result.videos, result.seed, options.outputDir) : Promise.resolve(""),
+          result.forums.length > 0 ? serializer.writeForumCSV(result.forums, result.seed, options.outputDir) : Promise.resolve(""),
+          result.aiOverview.detected ? serializer.writeAIOverviewCSV(result.aiOverview.citations, result.aiOverview.text, result.seed, options.outputDir) : Promise.resolve(""),
+          result.aiMode.detected ? serializer.writeAIModeCSV(result.aiMode.citations, result.aiMode.text, result.seed, options.outputDir) : Promise.resolve(""),
+          result.whatPeopleSaying.length > 0 ? serializer.writeWhatPeopleSayingCSV(result.whatPeopleSaying, result.seed, options.outputDir) : Promise.resolve("")
+        ]);
+      }
+      return result;
+    } catch (err) {
+      if (err instanceof CaptchaError && i < MAX_ATTEMPTS - 1) {
+        continue;
+      }
+      throw err;
+    }
+  }
+  const sessionDesc = options.kernelApiKey ? `${MAX_ATTEMPTS} fresh Kernel.sh sessions` : `${MAX_ATTEMPTS} attempts`;
+  throw new CaptchaError(`CAPTCHA on all ${sessionDesc}. Try again in a few minutes.`);
+}
+
+// src/video/VideoGenerator.ts
+var import_node_child_process2 = require("child_process");
+var import_node_fs3 = require("fs");
+var import_node_os = require("os");
+var import_node_path3 = require("path");
+var import_client2 = require("@fal-ai/client");
+
+// src/video/promptBuilder.ts
+var DEEPINFRA_URL = "https://api.deepinfra.com/v1/openai/chat/completions";
+var OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions";
+var QWEN_MODEL = "Qwen/Qwen3.6-35B-A3B";
+var SYSTEM_PROMPT = `You are a video prompt engineer for Seedance 2.0 text-to-video AI.
+
+IMPORTANT TECHNICAL CONTEXT: Clip 2 will be generated using image-to-video, meaning it will start from the exact last frame of Clip 1. The two clips will be visually seamless \u2014 same location, same characters, continuous motion. You must write prompts that make this feel like one uninterrupted 16-second video.
+
+Your job: turn a PAA question and its answer into a complete short-form video with visuals, narration, and a background audio mood.
+
+Produce four things:
+
+1. clip1 (~8s): Show a real person experiencing the situation the question describes. End on a specific frozen moment \u2014 a held expression, a paused action \u2014 that clip 2 continues from.
+
+2. clip2 (~8s): Continue from that exact frozen frame. Deliver the informational payoff from the answer. Show specific facts playing out visually. Describe motion continuing from clip 1's ending frame, not a new scene.
+
+3. voiceover: A spoken narration that fits in 10 seconds of TTS audio. HARD LIMIT: 22 words maximum. Count every word before finalizing \u2014 if it exceeds 22 words, cut it. Starts with the problem, ends with the answer. No filler, no "in conclusion". Informational and direct.
+
+4. audioMood: 6-10 words describing INSTRUMENTAL background music only \u2014 no vocals, no voice, no lyrics. Describe instruments and mood (e.g. "warm acoustic guitar, uplifting, professional home service, no vocals"). Will be passed to an AI audio model.
+
+Rules for visuals:
+- No text, captions, graphics, or overlays
+- Photorealistic, natural lighting, specific details
+- Describe exactly what the camera sees
+
+Respond with JSON only:
+{"clip1": "...", "clip2": "...", "voiceover": "...", "audioMood": "..."}`;
+async function callLLM(apiKey, baseUrl, question, answer) {
+  const res = await fetch(baseUrl, {
+    method: "POST",
+    headers: { "Authorization": `Bearer ${apiKey}`, "Content-Type": "application/json" },
+    body: JSON.stringify({
+      model: QWEN_MODEL,
+      temperature: 0.7,
+      messages: [
+        { role: "system", content: SYSTEM_PROMPT },
+        { role: "user", content: `Question: ${question}
+
+Answer: ${answer.slice(0, 500)}` }
+      ]
+    })
+  });
+  if (!res.ok) throw new Error(`LLM call failed (${res.status}): ${await res.text()}`);
+  const data = await res.json();
+  const raw = data.choices[0]?.message?.content?.trim() ?? "";
+  const match = raw.match(/\{[\s\S]*\}/);
+  if (!match) throw new Error(`No JSON in QWEN response: ${raw.slice(0, 200)}`);
+  const parsed = JSON.parse(match[0]);
+  if (!parsed.clip1 || !parsed.clip2 || !parsed.voiceover || !parsed.audioMood) {
+    throw new Error(`QWEN response missing fields: ${raw.slice(0, 200)}`);
+  }
+  return parsed;
+}
+async function buildClipPrompts(question, answer) {
+  const deepinfraKey = process.env["DEEPINFRA_API_KEY"];
+  const openrouterKey = process.env["OPENROUTER_API_KEY"];
+  if (deepinfraKey) {
+    try {
+      return await callLLM(deepinfraKey, DEEPINFRA_URL, question, answer);
+    } catch (err) {
+      console.warn("[promptBuilder] DeepInfra failed, trying OpenRouter:", err.message);
+    }
+  }
+  if (openrouterKey) {
+    return await callLLM(openrouterKey, OPENROUTER_URL, question, answer);
+  }
+  throw new Error("No LLM key \u2014 set DEEPINFRA_API_KEY or OPENROUTER_API_KEY");
+}
+
+// src/video/AudioGenerator.ts
+var TTS_MODEL = "fal-ai/inworld-tts";
+var MMAUDIO_MODEL = "fal-ai/mmaudio-v2";
+var QUEUE_BASE = "https://queue.fal.run";
+async function rawQueueRun(model, input, apiKey) {
+  const headers = { "Authorization": `Key ${apiKey}`, "Content-Type": "application/json" };
+  const submitRes = await fetch(`${QUEUE_BASE}/${model}`, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(input)
+  });
+  if (!submitRes.ok) throw new Error(`${model} submit failed (${submitRes.status}): ${await submitRes.text()}`);
+  const { request_id } = await submitRes.json();
+  console.log(`[fal] submitted ${model} \u2192 ${request_id}`);
+  while (true) {
+    await new Promise((r) => setTimeout(r, 5e3));
+    const statusRes = await fetch(`${QUEUE_BASE}/${model}/requests/${request_id}/status`, { headers });
+    if (!statusRes.ok) continue;
+    const { status } = await statusRes.json();
+    console.log(`[fal] ${request_id} \u2192 ${status}`);
+    if (status === "FAILED") throw new Error(`${model} request ${request_id} failed`);
+    if (status !== "COMPLETED") continue;
+    const resultRes = await fetch(`${QUEUE_BASE}/${model}/requests/${request_id}`, { headers });
+    if (!resultRes.ok) throw new Error(`Result fetch failed (${resultRes.status})`);
+    return await resultRes.json();
+  }
+}
+function getKey() {
+  const key = process.env["FAL_KEY"];
+  if (!key) throw new Error("FAL_KEY required");
+  return key;
+}
+async function generateVoiceover(text, voice = "Serena (en)") {
+  console.log("[AudioGenerator] Generating voiceover...");
+  const out = await rawQueueRun(TTS_MODEL, { text, voice, sample_rate_hertz: 48e3 }, getKey());
+  return out.audio.url;
+}
+async function addBackgroundAudio(videoUrl, mood, durationSeconds) {
+  console.log("[AudioGenerator] Adding background audio via MMAudio V2...");
+  const out = await rawQueueRun(MMAUDIO_MODEL, {
+    video_url: videoUrl,
+    prompt: mood,
+    negative_prompt: "speech, voice, talking, dialogue, narration, vocals, singing, human voice, conversation, words, lyrics, announcer, commentary",
+    duration: durationSeconds,
+    cfg_strength: 4.5
+  }, getKey());
+  return out.video.url;
+}
+
+// src/video/VideoMixer.ts
+var import_node_child_process = require("child_process");
+var import_node_fs2 = require("fs");
+var import_node_path2 = require("path");
+var import_client = require("@fal-ai/client");
+async function download(url, destPath) {
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`Download failed (${res.status}): ${url}`);
+  (0, import_node_fs2.writeFileSync)(destPath, Buffer.from(await res.arrayBuffer()));
+}
+async function concatenateClips(clip1Url, clip2Url, outDir) {
+  (0, import_node_fs2.mkdirSync)(outDir, { recursive: true });
+  const ts = Date.now();
+  const p1 = (0, import_node_path2.join)(outDir, `clip1-${ts}.mp4`);
+  const p2 = (0, import_node_path2.join)(outDir, `clip2-${ts}.mp4`);
+  const out = (0, import_node_path2.join)(outDir, `combined-${ts}.mp4`);
+  console.log("[VideoMixer] Downloading clips...");
+  await Promise.all([download(clip1Url, p1), download(clip2Url, p2)]);
+  console.log("[VideoMixer] Concatenating...");
+  (0, import_node_child_process.execSync)(
+    `ffmpeg -i "${p1}" -i "${p2}" -filter_complex "[0:v][1:v]concat=n=2:v=1:a=0[v]" -map "[v]" -y "${out}" -loglevel error`
+  );
+  return out;
+}
+async function uploadToFal(localPath) {
+  const { readFileSync: readFileSync2 } = await import("fs");
+  const blob = new Blob([readFileSync2(localPath)], { type: "video/mp4" });
+  const url = await import_client.fal.storage.upload(blob);
+  console.log("[VideoMixer] Uploaded to fal:", url);
+  return url;
+}
+async function overlayVoiceover(videoPath, voiceoverUrl, outDir) {
+  const ts = Date.now();
+  const wav = (0, import_node_path2.join)(outDir, `voiceover-${ts}.wav`);
+  const out = (0, import_node_path2.join)(outDir, `final-${ts}.mp4`);
+  console.log("[VideoMixer] Downloading voiceover...");
+  await download(voiceoverUrl, wav);
+  console.log("[VideoMixer] Mixing voiceover over background audio...");
+  (0, import_node_child_process.execSync)(
+    `ffmpeg -i "${videoPath}" -i "${wav}" -filter_complex "[0:a]volume=0.2[bg];[1:a]volume=1.0[vo];[bg][vo]amix=inputs=2:duration=first[a]" -map 0:v -map "[a]" -c:v copy -y "${out}" -loglevel error`
+  );
+  return out;
+}
+
+// src/video/VideoGenerator.ts
+var T2V = "bytedance/seedance-2.0/text-to-video";
+var I2V = "bytedance/seedance-2.0/image-to-video";
+function buildInput(prompt, opts, seed, imageUrl) {
+  return {
+    prompt,
+    resolution: opts.resolution ?? "720p",
+    duration: opts.clipDurationSeconds ?? 8,
+    aspect_ratio: opts.aspectRatio ?? "16:9",
+    generate_audio: false,
+    ...seed !== void 0 ? { seed } : {},
+    ...imageUrl !== void 0 ? { image_url: imageUrl } : {}
+  };
+}
+async function generate(model, input) {
+  const { request_id } = await import_client2.fal.queue.submit(model, { input });
+  console.log(`[fal] submitted ${model} \u2192 ${request_id}`);
+  while (true) {
+    await new Promise((r) => setTimeout(r, 5e3));
+    const s = await import_client2.fal.queue.status(model, { requestId: request_id, logs: false });
+    console.log(`[fal] ${request_id} \u2192 ${s.status}`);
+    if (s.status === "FAILED") throw new Error(`Request ${request_id} failed`);
+    if (s.status !== "COMPLETED") continue;
+    const result = await import_client2.fal.queue.result(model, { requestId: request_id });
+    return result.data;
+  }
+}
+async function extractLastFrame(videoUrl, outDir) {
+  const ts = Date.now();
+  const mp4Path = (0, import_node_path3.join)(outDir, `clip1-raw-${ts}.mp4`);
+  const jpgPath = (0, import_node_path3.join)(outDir, `last-frame-${ts}.jpg`);
+  const res = await fetch(videoUrl);
+  if (!res.ok) throw new Error(`Failed to download clip 1 (${res.status})`);
+  (0, import_node_fs3.writeFileSync)(mp4Path, Buffer.from(await res.arrayBuffer()));
+  try {
+    (0, import_node_child_process2.execSync)(`ffmpeg -sseof -0.1 -i "${mp4Path}" -vframes 1 -y "${jpgPath}" -loglevel error`);
+  } finally {
+    try {
+      (0, import_node_fs3.unlinkSync)(mp4Path);
+    } catch {
+    }
+  }
+  return jpgPath;
+}
+var VideoGenerator = class {
+  constructor(apiKey) {
+    const key = apiKey ?? process.env["FAL_KEY"];
+    if (!key) throw new Error("FAL_KEY is required");
+    import_client2.fal.config({ credentials: key });
+  }
+  async generateClipPair(question, answer, opts = {}) {
+    const outDir = opts.outputDir ?? (0, import_node_path3.join)((0, import_node_os.tmpdir)(), `paa-video-${Date.now()}`);
+    (0, import_node_fs3.mkdirSync)(outDir, { recursive: true });
+    console.log("\n[1/7] Generating prompts via QWEN 3.6...");
+    const prompts = await buildClipPrompts(question, answer);
+    console.log("  Voiceover:", prompts.voiceover);
+    console.log("  Audio mood:", prompts.audioMood);
+    console.log("\n[2/7] Generating clip 1 (text-to-video)...");
+    const result1 = await generate(T2V, buildInput(prompts.clip1, opts, opts.seed));
+    console.log("\n[3/7] Extracting last frame \u2192 clip 2 start...");
+    const jpgPath = await extractLastFrame(result1.video.url, outDir);
+    const imageBlob = new Blob([(0, import_node_fs3.readFileSync)(jpgPath)], { type: "image/jpeg" });
+    const frameUrl = await import_client2.fal.storage.upload(imageBlob);
+    try {
+      (0, import_node_fs3.unlinkSync)(jpgPath);
+    } catch {
+    }
+    console.log("\n[4/7] Generating clip 2 (image-to-video from last frame)...");
+    const seed2 = opts.seed !== void 0 ? opts.seed + 1 : void 0;
+    const result2 = await generate(I2V, buildInput(prompts.clip2, opts, seed2, frameUrl));
+    console.log("\n[5/7] Concatenating clips + generating voiceover (parallel)...");
+    const [combinedPath, voiceoverUrl] = await Promise.all([
+      concatenateClips(result1.video.url, result2.video.url, outDir),
+      generateVoiceover(prompts.voiceover, opts.ttsVoice)
+    ]);
+    console.log("\n[6/7] Adding background audio via MMAudio V2...");
+    const falVideoUrl = await uploadToFal(combinedPath);
+    const totalDuration = (opts.clipDurationSeconds ?? 8) * 2;
+    const videoWithAudioUrl = await addBackgroundAudio(falVideoUrl, prompts.audioMood, totalDuration);
+    console.log("\n[7/7] Overlaying voiceover on final video...");
+    const videoWithAudioPath = (0, import_node_path3.join)(outDir, `with-bg-audio-${Date.now()}.mp4`);
+    const bgRes = await fetch(videoWithAudioUrl);
+    (0, import_node_fs3.writeFileSync)(videoWithAudioPath, Buffer.from(await bgRes.arrayBuffer()));
+    const finalVideoPath = await overlayVoiceover(videoWithAudioPath, voiceoverUrl, outDir);
+    return {
+      clip1Url: result1.video.url,
+      clip2Url: result2.video.url,
+      finalVideoPath,
+      seed: result1.seed,
+      promptClip1: prompts.clip1,
+      promptClip2: prompts.clip2,
+      voiceover: prompts.voiceover,
+      audioMood: prompts.audioMood
+    };
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  VideoGenerator,
+  buildClipPrompts,
+  harvest
+});
+//# sourceMappingURL=index.cjs.map