mcp-scraper 0.1.0 → 0.1.1

package/dist/bin/paa-harvest.cjs

@@ -33,6 +33,10 @@ var HarvestOptionsSchema = import_zod.z.object({
   location: import_zod.z.string().optional(),
   gl: import_zod.z.string().length(2).default("us"),
   hl: import_zod.z.string().length(2).default("en"),
+  device: import_zod.z.enum(["desktop", "mobile"]).default("desktop"),
+  proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location"),
+  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional(),
+  debug: import_zod.z.boolean().default(false),
   depth: import_zod.z.number().int().min(1).max(30).default(3),
   maxQuestions: import_zod.z.number().int().min(1).max(1e3).default(100),
   headless: import_zod.z.boolean().default(false),
@@ -40,6 +44,7 @@ var HarvestOptionsSchema = import_zod.z.object({
   proxy: import_zod.z.string().url().optional(),
   kernelApiKey: import_zod.z.string().optional(),
   kernelProxyId: import_zod.z.string().optional(),
+  kernelProxyResolution: import_zod.z.unknown().optional(),
   outputDir: import_zod.z.string().default("./paa-output"),
   format: import_zod.z.enum(["json", "csv", "both"]).default("both"),
   serpOnly: import_zod.z.boolean().default(false),
@@ -63,6 +68,45 @@ var RawPAAItemSchema = import_zod.z.object({
   sourceSite: import_zod.z.string().optional(),
   sourceCite: import_zod.z.string().optional()
 });
+var RawMapsOverviewSchema = import_zod.z.object({
+  name: import_zod.z.string().nullable(),
+  rating: import_zod.z.string().nullable(),
+  reviewCount: import_zod.z.string().nullable(),
+  category: import_zod.z.string().nullable(),
+  address: import_zod.z.string().nullable(),
+  hoursSummary: import_zod.z.string().nullable(),
+  phone: import_zod.z.string().nullable(),
+  phoneDisplay: import_zod.z.string().nullable(),
+  website: import_zod.z.string().nullable(),
+  plusCode: import_zod.z.string().nullable(),
+  bookingUrl: import_zod.z.string().nullable()
+});
+var RawMapsHoursRowSchema = import_zod.z.object({
+  day: import_zod.z.string(),
+  hours: import_zod.z.string()
+});
+var RawMapsReviewStatsSchema = import_zod.z.object({
+  reviewHistogram: import_zod.z.array(import_zod.z.object({
+    stars: import_zod.z.number(),
+    count: import_zod.z.string()
+  })),
+  reviewTopics: import_zod.z.array(import_zod.z.object({
+    label: import_zod.z.string(),
+    count: import_zod.z.string()
+  }))
+});
+var RawMapsReviewCardSchema = import_zod.z.object({
+  reviewId: import_zod.z.string(),
+  author: import_zod.z.string().nullable(),
+  stars: import_zod.z.string().nullable(),
+  date: import_zod.z.string().nullable(),
+  text: import_zod.z.string().nullable(),
+  ownerResponse: import_zod.z.string().nullable()
+});
+var RawMapsAboutAttributeSchema = import_zod.z.object({
+  section: import_zod.z.string(),
+  attribute: import_zod.z.string()
+});
 
 // src/driver/BrowserDriver.ts
 var import_playwright_extra = require("playwright-extra");
@@ -78,7 +122,7 @@ var PAASelectors = {
   itemDataQ: "data-q",
   itemDataInitQ: "data-initq",
   itemQuestionEl: ".JlqpRe",
-  answerContainer: ".bCOlv",
+  answerContainer: ".bCOlv, .hgKElc, .wDYxhc, .LGOjhe, .fo7IQd, .fmW3u",
   sourceTitle: "h3",
   sourceSite: ".VuuXrf",
   sourceCite: "cite",
@@ -118,9 +162,16 @@ var WhatPeopleSayingSelectors = {
   authorNote: ".nDgy9d"
 };
 var AIOverviewSelectors = {
-  root: '[data-hveid="CBMQAA"]',
+  root: "[data-lhcontainer][data-streaming-container][eid]",
+  legacyRoot: '[data-hveid="CBMQAA"]',
   wrapper: ".Fgyi2e",
-  citations: '.Fgyi2e [data-hveid] a[jsname="pxBnId"]'
+  controller: '[jscontroller="AkrxPe"]',
+  contentSubtree: '[data-subtree="mfc"]',
+  header: ".heWuVc",
+  heading: ".Fzsovc.cwYVJe.RJPOee",
+  showMoreButton: '[aria-label="Show more AI Overview"]',
+  sourcesPanel: ".OZ9ddf.WAUd4",
+  disclaimer: ".DuQANe.MSJHRb"
 };
 var AIModeSelectors = {
   root: '[data-hveid="CAUQAA"]',
@@ -148,6 +199,9 @@ var LocalPackSelectors = {
 
 // src/errors.ts
 var RECAPTCHA_INSTRUCTIONS = "Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.";
+function sanitizeVendorName(message) {
+  return message.replace(/kernel\.sh\s+sessions?/gi, "sessions").replace(/kernel\.sh\s+session/gi, "this session").replace(/kernel\.sh/gi, "the service").replace(/kernel\s+sessions?/gi, "sessions").replace(/kernel\s+session/gi, "this session").replace(/\bkernel\b/gi, "the service").replace(/  +/g, " ").trim();
+}
 var CaptchaError = class extends Error {
   constructor(instructions) {
     super(`CAPTCHA detected. ${instructions}`);
@@ -164,10 +218,55 @@ var ExtractionError = class extends Error {
   cause;
   name = "ExtractionError";
 };
+var RequestAbortedError = class extends Error {
+  name = "RequestAbortedError";
+  constructor(message = "Request aborted before harvest completed") {
+    super(message);
+  }
+};
 
 // src/driver/BrowserDriver.ts
 import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
 var DESKTOP_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
+var MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
+var DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
+var KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
+var KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
+function positiveIntFromEnv(name, fallback) {
+  const raw = process.env[name];
+  if (!raw) return fallback;
+  const parsed = Number(raw);
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+function proxyIdSuffix(proxyId) {
+  return proxyId ? proxyId.slice(-6) : null;
+}
+function errorText(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+function rankCheckContextOptions(config) {
+  return {
+    viewport: config.viewport,
+    locale: config.locale,
+    userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : DESKTOP_USER_AGENT),
+    ...config.deviceScaleFactor ? { deviceScaleFactor: config.deviceScaleFactor } : {},
+    ...config.isMobile !== void 0 ? { isMobile: config.isMobile } : {},
+    ...config.hasTouch !== void 0 ? { hasTouch: config.hasTouch } : {}
+  };
+}
+async function withTimeout(promise, timeoutMs, label) {
+  let timeout;
+  try {
+    return await Promise.race([
+      promise,
+      new Promise((_, reject) => {
+        timeout = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
+      })
+    ]);
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+}
 function buildYouTubeChannelVideosUrl(channelInput) {
   const raw = channelInput.trim();
   if (!raw) throw new Error("channelHandle is required");
@@ -201,30 +300,101 @@ var BrowserDriver = class {
   page = null;
   kernelClient = null;
   kernelSessionId = null;
+  debugEnabled = false;
+  debugSnapshot = {
+    kernel: null,
+    context: null,
+    networkLocation: null,
+    serpNavigation: null
+  };
   async launch(config) {
+    this.debugEnabled = config.debug === true;
+    const proxyMode = config.proxyMode ?? (config.kernelProxyId ? "configured" : "none");
+    const device = config.isMobile ? "mobile" : "desktop";
+    this.debugSnapshot = {
+      kernel: null,
+      context: {
+        viewport: config.viewport,
+        locale: config.locale,
+        device,
+        userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : DESKTOP_USER_AGENT),
+        deviceScaleFactor: config.deviceScaleFactor ?? null,
+        isMobile: config.isMobile === true,
+        hasTouch: config.hasTouch === true
+      },
+      networkLocation: null,
+      serpNavigation: null
+    };
     if (config.kernelApiKey) {
       this.kernelClient = new import_sdk.default({ apiKey: config.kernelApiKey });
+      const timeoutSeconds = positiveIntFromEnv("KERNEL_BROWSER_TIMEOUT_SECONDS", DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS);
       const kernelBrowser = await this.kernelClient.browsers.create({
         stealth: true,
-        timeout_seconds: 600,
+        timeout_seconds: timeoutSeconds,
         ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {}
       });
       this.kernelSessionId = kernelBrowser.session_id;
+      let defaultProxyDisabled = null;
+      let defaultProxyDisableError = null;
+      if (proxyMode === "none") {
+        try {
+          await withTimeout(
+            this.kernelClient.browsers.update(this.kernelSessionId, { disable_default_proxy: true }),
+            5e3,
+            `Kernel session ${this.kernelSessionId} disable default proxy`
+          );
+          defaultProxyDisabled = true;
+        } catch (err) {
+          defaultProxyDisabled = false;
+          defaultProxyDisableError = errorText(err);
+        }
+      }
+      const kernelDebug = {
+        sessionId: this.kernelSessionId,
+        proxyMode,
+        requestedProxyIdPresent: Boolean(config.kernelProxyId),
+        requestedProxyIdSuffix: proxyIdSuffix(config.kernelProxyId),
+        createdProxyIdPresent: typeof kernelBrowser.proxy_id === "string" ? Boolean(kernelBrowser.proxy_id) : null,
+        createdProxyIdSuffix: proxyIdSuffix(kernelBrowser.proxy_id),
+        retrievedProxyIdPresent: null,
+        retrievedProxyIdSuffix: null,
+        retrievedProxyIdMatchesRequested: null,
+        defaultProxyDisabled,
+        defaultProxyDisableError,
+        proxyResolution: config.kernelProxyResolution ?? null,
+        timeoutSeconds,
+        stealth: typeof kernelBrowser.stealth === "boolean" ? kernelBrowser.stealth : null,
+        profilePresent: null,
+        poolPresent: null,
+        retrieveError: null
+      };
+      this.debugSnapshot.kernel = kernelDebug;
+      console.info(JSON.stringify({
+        event: "kernel_browser_created",
+        kernel_session_id: this.kernelSessionId,
+        timeout_seconds: timeoutSeconds,
+        proxy_mode: proxyMode,
+        proxy_id_present: Boolean(config.kernelProxyId),
+        proxy_resolution_source: config.kernelProxyResolution?.source
+      }));
+      if (this.debugEnabled) {
+        await this.populateKernelRetrieveDebug(kernelDebug, config.kernelProxyId);
+      }
       this.browser = await import_playwright.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
-      this.context = this.browser.contexts()[0] ?? await this.browser.newContext();
+      this.context = await this.browser.newContext(rankCheckContextOptions(config));
       await this.installEsbuildHelperShims(this.context);
-      this.page = this.context.pages()[0] ?? await this.context.newPage();
+      this.page = await this.context.newPage();
+      await this.page.setViewportSize(config.viewport);
+      if (this.debugEnabled) {
+        this.debugSnapshot.networkLocation = await this.captureBrowserNetworkLocation();
+      }
       return;
     }
     const launchOpts = {
       headless: config.headless,
       proxy: config.proxy ? { server: config.proxy } : void 0
     };
-    const ctxOpts = {
-      viewport: config.viewport,
-      locale: config.locale,
-      userAgent: DESKTOP_USER_AGENT
-    };
+    const ctxOpts = rankCheckContextOptions(config);
     if (config.profileDir) {
       this.context = await import_playwright_extra.chromium.launchPersistentContext(config.profileDir, {
         ...launchOpts,
@@ -238,6 +408,107 @@ var BrowserDriver = class {
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
     }
+    if (this.debugEnabled) {
+      this.debugSnapshot.networkLocation = await this.captureBrowserNetworkLocation();
+    }
+  }
+  async populateKernelRetrieveDebug(kernelDebug, requestedProxyId) {
+    if (!this.kernelClient || !this.kernelSessionId) return;
+    try {
+      const retrieved = await withTimeout(
+        this.kernelClient.browsers.retrieve(this.kernelSessionId),
+        5e3,
+        `Kernel session ${this.kernelSessionId} retrieve`
+      );
+      kernelDebug.retrievedProxyIdPresent = typeof retrieved.proxy_id === "string" ? Boolean(retrieved.proxy_id) : false;
+      kernelDebug.retrievedProxyIdSuffix = proxyIdSuffix(retrieved.proxy_id);
+      kernelDebug.retrievedProxyIdMatchesRequested = requestedProxyId ? retrieved.proxy_id === requestedProxyId : !retrieved.proxy_id;
+      kernelDebug.timeoutSeconds = typeof retrieved.timeout_seconds === "number" ? retrieved.timeout_seconds : kernelDebug.timeoutSeconds;
+      kernelDebug.stealth = typeof retrieved.stealth === "boolean" ? retrieved.stealth : kernelDebug.stealth;
+      kernelDebug.profilePresent = Boolean(retrieved.profile);
+      kernelDebug.poolPresent = Boolean(retrieved.pool);
+    } catch (err) {
+      kernelDebug.retrieveError = errorText(err);
+    }
+  }
+  async captureBrowserNetworkLocation() {
+    const fallback = (message, source = "ipapi.co") => ({
+      source,
+      ip: null,
+      city: null,
+      region: null,
+      country: null,
+      org: null,
+      timezone: null,
+      error: message
+    });
+    if (!this.context) return fallback("browser context is not available");
+    let debugPage = null;
+    try {
+      debugPage = await this.context.newPage();
+      const ipwho = await this.loadJsonInDebugPage(debugPage, "https://ipwho.is/");
+      if (ipwho) {
+        const connection = typeof ipwho.connection === "object" && ipwho.connection !== null ? ipwho.connection : {};
+        return {
+          source: "ipwho.is",
+          ip: typeof ipwho.ip === "string" ? ipwho.ip : null,
+          city: typeof ipwho.city === "string" ? ipwho.city : null,
+          region: typeof ipwho.region === "string" ? ipwho.region : null,
+          country: typeof ipwho.country === "string" ? ipwho.country : null,
+          org: typeof connection.org === "string" ? connection.org : null,
+          timezone: typeof ipwho.timezone === "object" && ipwho.timezone !== null && typeof ipwho.timezone.id === "string" ? ipwho.timezone.id : null,
+          error: null
+        };
+      }
+      const ipify = await this.loadJsonInDebugPage(debugPage, "https://api64.ipify.org?format=json");
+      if (ipify) {
+        return {
+          source: "api64.ipify.org",
+          ip: typeof ipify.ip === "string" ? ipify.ip : null,
+          city: null,
+          region: null,
+          country: null,
+          org: null,
+          timezone: null,
+          error: null
+        };
+      }
+      await withTimeout(
+        debugPage.goto("https://ipapi.co/json/", { waitUntil: "domcontentloaded", timeout: 7e3 }),
+        8e3,
+        "browser network location navigation"
+      );
+      const body = await debugPage.locator("body").innerText({ timeout: 2e3 });
+      const data = JSON.parse(body);
+      return {
+        source: "ipapi.co",
+        ip: typeof data.ip === "string" ? data.ip : null,
+        city: typeof data.city === "string" ? data.city : null,
+        region: typeof data.region === "string" ? data.region : null,
+        country: typeof data.country_name === "string" ? data.country_name : typeof data.country === "string" ? data.country : null,
+        org: typeof data.org === "string" ? data.org : null,
+        timezone: typeof data.timezone === "string" ? data.timezone : null,
+        error: null
+      };
+    } catch (err) {
+      return fallback(errorText(err));
+    } finally {
+      await debugPage?.close().catch(() => {
+      });
+    }
+  }
+  async loadJsonInDebugPage(debugPage, url) {
+    try {
+      await withTimeout(
+        debugPage.goto(url, { waitUntil: "domcontentloaded", timeout: 7e3 }),
+        8e3,
+        `browser network location navigation ${url}`
+      );
+      const body = await debugPage.locator("body").innerText({ timeout: 2e3 });
+      return JSON.parse(body);
+    } catch {
+      return null;
+    }
   }
   async installEsbuildHelperShims(context) {
     await context.addInitScript(() => {
@@ -249,42 +520,79 @@ var BrowserDriver = class {
       };
     });
   }
-  async navigateToSERP(query, uule, gl, hl) {
-    const params = new URLSearchParams({ q: query, gl, hl });
+  async navigateToSERP(query, uule, gl, hl, options) {
+    const params = new URLSearchParams({ q: query, gl, hl, pws: "0" });
+    if (options?.num) params.set("num", String(options.num));
     if (uule) params.set("uule", uule);
     const url = "https://www.google.com/search?" + params.toString();
+    const navDebug = options?.debug ? {
+      requestedUrl: url,
+      finalUrl: null,
+      title: null,
+      bodySnippet: null,
+      hasPaa: null,
+      captchaDetected: null,
+      googleSorryUrl: null,
+      redirected: null
+    } : null;
+    if (navDebug) this.debugSnapshot.serpNavigation = navDebug;
     try {
       await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
     } catch (err) {
+      await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: null, captchaDetected: null });
       const diag = await this.captureDiagnostics(url);
       throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
     }
     const captchaCount = await this.page.locator(PAASelectors.captchaMarker).count();
     if (captchaCount > 0) {
-      if (this.kernelClient) {
-        try {
-          await this.page.waitForSelector(PAASelectors.container, { timeout: 45e3 });
-          return { hasPaa: true };
-        } catch {
-          throw new CaptchaError(this.captchaMessage());
-        }
-      }
+      await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
       throw new CaptchaError(this.captchaMessage());
     }
     const fastFound = await this.page.waitForSelector(PAASelectors.item, { timeout: 4e3 }).catch(() => null);
-    if (fastFound) return { hasPaa: true };
+    if (fastFound) {
+      await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: true, captchaDetected: false });
+      return { hasPaa: true };
+    }
     const captchaAfter = await this.page.locator(PAASelectors.captchaMarker).count();
-    if (captchaAfter > 0) throw new CaptchaError(this.captchaMessage());
+    if (captchaAfter > 0) {
+      await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
+      throw new CaptchaError(this.captchaMessage());
+    }
     for (let i = 1; i <= 6; i++) {
       await this.page.evaluate((f) => {
         window.scrollTo(0, document.body.scrollHeight * f);
       }, i / 6);
       await this.page.waitForTimeout(600);
       const count = await this.page.locator(PAASelectors.item).count();
-      if (count > 0) return { hasPaa: true };
+      if (count > 0) {
+        await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: true, captchaDetected: false });
+        return { hasPaa: true };
+      }
     }
+    await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: false });
     return { hasPaa: false };
   }
+  async updateSerpNavigationDebug(navDebug, requestedUrl, state) {
+    if (!navDebug || !this.page) return;
+    try {
+      const finalUrl = this.page.url();
+      const title = await this.page.title().catch(() => "");
+      const bodySnippet = await this.page.evaluate(() => {
+        const text = (document.body?.innerText ?? "").replace(/\s+/g, " ").trim();
+        return text.slice(0, 500);
+      }).catch(() => "");
+      const textCaptcha = /recaptcha|unusual traffic|are you a robot/i.test(bodySnippet);
+      navDebug.finalUrl = finalUrl;
+      navDebug.title = title;
+      navDebug.bodySnippet = bodySnippet;
+      navDebug.hasPaa = state.hasPaa;
+      navDebug.captchaDetected = state.captchaDetected ?? textCaptcha;
+      navDebug.googleSorryUrl = /google\.[^/]+\/sorry\//i.test(finalUrl);
+      navDebug.redirected = finalUrl !== requestedUrl;
+    } catch (err) {
+      navDebug.bodySnippet = `debug capture failed: ${errorText(err)}`;
+    }
+  }
   async captureDiagnostics(intendedUrl) {
     try {
       const finalUrl = this.page.url();
@@ -306,7 +614,7 @@ var BrowserDriver = class {
     }
   }
   captchaMessage() {
-    return this.kernelClient ? "Google returned a CAPTCHA on this Kernel.sh session \u2014 retrying with a fresh session." : RECAPTCHA_INSTRUCTIONS;
+    return this.kernelClient ? "Google returned a CAPTCHA on this session \u2014 retrying with a fresh session." : RECAPTCHA_INSTRUCTIONS;
   }
   async navigateTo(url) {
     try {
@@ -331,6 +639,12 @@ var BrowserDriver = class {
   getPage() {
     return this.page;
   }
+  getKernelSessionId() {
+    return this.kernelSessionId;
+  }
+  getDebugSnapshot() {
+    return this.debugSnapshot;
+  }
   async close() {
     if (this.browser) {
       const b = this.browser;
@@ -341,21 +655,84 @@ var BrowserDriver = class {
       this.page = null;
       this.kernelSessionId = null;
       this.kernelClient = null;
-      try {
-        await b.close();
-      } finally {
-        if (client && sessionId) {
-          await client.browsers.deleteByID(sessionId).catch(
-            (err) => console.warn("Kernel session cleanup failed:", err)
-          );
+      if (client && sessionId) {
+        console.info(JSON.stringify({
+          event: "kernel_browser_delete_started",
+          kernel_session_id: sessionId
+        }));
+        const deleteSession = withTimeout(
+          client.browsers.deleteByID(sessionId),
+          KERNEL_SESSION_DELETE_TIMEOUT_MS,
+          `Kernel session ${sessionId} delete`
+        );
+        const closeBrowser = withTimeout(
+          b.close(),
+          KERNEL_BROWSER_CLOSE_TIMEOUT_MS,
+          `Kernel browser ${sessionId} close`
+        );
+        const [deleteResult, closeResult] = await Promise.allSettled([deleteSession, closeBrowser]);
+        const result = {
+          kernelSessionId: sessionId,
+          kernelDeleteStarted: true,
+          kernelDeleteSucceeded: deleteResult.status === "fulfilled",
+          kernelDeleteError: deleteResult.status === "rejected" ? deleteResult.reason instanceof Error ? deleteResult.reason.message : String(deleteResult.reason) : null,
+          browserCloseSucceeded: closeResult.status === "fulfilled",
+          browserCloseError: closeResult.status === "rejected" ? closeResult.reason instanceof Error ? closeResult.reason.message : String(closeResult.reason) : null
+        };
+        if (deleteResult.status === "rejected") {
+          console.warn(JSON.stringify({
+            event: "kernel_browser_delete_failed",
+            kernel_session_id: sessionId,
+            message: result.kernelDeleteError
+          }));
+          console.warn(`Kernel session cleanup failed for ${sessionId}:`, deleteResult.reason);
+        } else {
+          console.info(JSON.stringify({
+            event: "kernel_browser_delete_succeeded",
+            kernel_session_id: sessionId
+          }));
         }
+        if (closeResult.status === "rejected") {
+          console.warn(JSON.stringify({
+            event: "kernel_browser_close_failed",
+            kernel_session_id: sessionId,
+            message: result.browserCloseError
+          }));
+          console.warn(`Kernel browser close failed for ${sessionId}:`, closeResult.reason);
+        }
+        return result;
       }
+      await b.close();
+      return {
+        kernelSessionId: null,
+        kernelDeleteStarted: false,
+        kernelDeleteSucceeded: null,
+        kernelDeleteError: null,
+        browserCloseSucceeded: true,
+        browserCloseError: null
+      };
     } else if (this.context) {
       const ctx = this.context;
       this.context = null;
       this.page = null;
       await ctx.close();
+      return {
+        kernelSessionId: null,
+        kernelDeleteStarted: false,
+        kernelDeleteSucceeded: null,
+        kernelDeleteError: null,
+        browserCloseSucceeded: true,
+        browserCloseError: null
+      };
     }
+    return {
+      kernelSessionId: null,
+      kernelDeleteStarted: false,
+      kernelDeleteSucceeded: null,
+      kernelDeleteError: null,
+      browserCloseSucceeded: null,
+      browserCloseError: null
+    };
   }
 };
 
@@ -426,13 +803,157 @@ var LOCATIONS = {
 };
 
 // src/uule.ts
+function encodeVarint(value) {
+  const bytes = [];
+  let remaining = value;
+  do {
+    let byte = remaining & 127;
+    remaining >>>= 7;
+    if (remaining > 0) byte |= 128;
+    bytes.push(byte);
+  } while (remaining > 0);
+  return bytes;
+}
 function encodeUule(name) {
-  const encoded = Buffer.from(String.fromCharCode(name.length) + name).toString("base64");
-  return `w+CAIQICI${encoded}`;
+  const locationBytes = Buffer.from(name, "utf8");
+  const payload = Buffer.concat([
+    Buffer.from([8, 2, 16, 32, 34]),
+    Buffer.from(encodeVarint(locationBytes.length)),
+    locationBytes
+  ]);
+  return `w+${payload.toString("base64")}`;
 }
 function normalizeLocation(input) {
-  const key = input.toLowerCase().trim();
-  return LOCATIONS[key] ?? input;
+  const raw = input.toLowerCase().trim();
+  if (LOCATIONS[raw]) return LOCATIONS[raw];
+  const beforeComma = raw.split(",")[0].trim();
+  if (beforeComma !== raw && LOCATIONS[beforeComma]) return LOCATIONS[beforeComma];
+  const withoutState = raw.replace(/\s+[a-z]{2}$/, "").trim();
+  if (withoutState !== raw && LOCATIONS[withoutState]) return LOCATIONS[withoutState];
+  return input;
+}
+
+// src/serp-location-debug.ts
+var STATE_TO_CODE = {
+  alabama: "AL",
+  alaska: "AK",
+  arizona: "AZ",
+  arkansas: "AR",
+  california: "CA",
+  colorado: "CO",
+  connecticut: "CT",
+  delaware: "DE",
+  florida: "FL",
+  georgia: "GA",
+  hawaii: "HI",
+  idaho: "ID",
+  illinois: "IL",
+  indiana: "IN",
+  iowa: "IA",
+  kansas: "KS",
+  kentucky: "KY",
+  louisiana: "LA",
+  maine: "ME",
+  maryland: "MD",
+  massachusetts: "MA",
+  michigan: "MI",
+  minnesota: "MN",
+  mississippi: "MS",
+  missouri: "MO",
+  montana: "MT",
+  nebraska: "NE",
+  nevada: "NV",
+  "new hampshire": "NH",
+  "new jersey": "NJ",
+  "new mexico": "NM",
+  "new york": "NY",
+  "north carolina": "NC",
+  "north dakota": "ND",
+  ohio: "OH",
+  oklahoma: "OK",
+  oregon: "OR",
+  pennsylvania: "PA",
+  "rhode island": "RI",
+  "south carolina": "SC",
+  "south dakota": "SD",
+  tennessee: "TN",
+  texas: "TX",
+  utah: "UT",
+  vermont: "VT",
+  virginia: "VA",
+  washington: "WA",
+  "west virginia": "WV",
+  wisconsin: "WI",
+  wyoming: "WY",
+  "district of columbia": "DC"
+};
+var STATE_PATTERN = [
+  ...Object.keys(STATE_TO_CODE).map((s) => s.replace(/\s+/g, "\\s+")),
+  ...Object.values(STATE_TO_CODE)
+].join("|");
+var CITY_STATE_RE = new RegExp(`\\b([A-Z][A-Za-z]+(?:[\\s.-][A-Z][A-Za-z]+){0,4}),?\\s+(${STATE_PATTERN})\\b`, "gi");
+function normalizeRegionCode(input) {
+  if (!input) return null;
+  const trimmed = input.trim();
+  if (/^[A-Z]{2}$/i.test(trimmed)) return trimmed.toUpperCase();
+  return STATE_TO_CODE[trimmed.toLowerCase()] ?? null;
+}
+function normalizeCity(input) {
+  const cleaned = input.replace(/\s+/g, " ").trim().replace(/^.*\b(?:in|near|around|serving)\s+/i, "");
+  return cleaned.toLowerCase().replace(/\b[a-z]/g, (char) => char.toUpperCase());
+}
+function parseExpected(canonicalLocation) {
+  if (!canonicalLocation) return null;
+  const [city = "", region = ""] = canonicalLocation.split(",").map((part) => part.trim());
+  return {
+    city: normalizeCity(city),
+    regionCode: normalizeRegionCode(region),
+    canonicalLocation
+  };
+}
+function addCandidate(candidates, city, region, example) {
+  const normalizedCity = normalizeCity(city);
+  const regionCode = normalizeRegionCode(region);
+  if (!normalizedCity || !regionCode) return;
+  const key = `${normalizedCity.toLowerCase()}|${regionCode}`;
+  const existing = candidates.get(key);
+  if (existing) {
+    existing.count++;
+    if (existing.examples.length < 3 && !existing.examples.includes(example)) existing.examples.push(example);
+    return;
+  }
+  candidates.set(key, { city: normalizedCity, regionCode, count: 1, examples: [example] });
+}
+function scanText(candidates, text) {
+  const normalized = decodeURIComponent(text).replace(/[+/|_-]+/g, " ");
+  for (const match of normalized.matchAll(CITY_STATE_RE)) {
+    addCandidate(candidates, match[1] ?? "", match[2] ?? "", normalized.slice(0, 180));
+  }
+}
+function inferSerpLocationEvidence(canonicalLocation, organicResults, localPack) {
+  const expected = parseExpected(canonicalLocation);
+  const candidates = /* @__PURE__ */ new Map();
+  for (const result of organicResults) {
+    scanText(candidates, [result.title, result.snippet ?? "", result.cite ?? "", result.url].join(" "));
+  }
+  for (const business of localPack) {
+    scanText(candidates, [business.name, ...business.metadata, business.websiteUrl ?? "", business.directionsUrl ?? ""].join(" "));
+  }
+  const rankedCandidates = Array.from(candidates.values()).sort((a, b) => b.count - a.count || a.city.localeCompare(b.city)).slice(0, 8);
+  if (!expected) {
+    return { status: "not_requested", expected: null, candidates: rankedCandidates };
+  }
+  if (rankedCandidates.length === 0) {
+    return { status: "unknown", expected, candidates: [] };
+  }
+  const matched = rankedCandidates.some(
+    (candidate) => candidate.city.toLowerCase() === expected.city.toLowerCase() && (expected.regionCode == null || candidate.regionCode === expected.regionCode)
+  );
+  return {
+    status: matched ? "matched" : "mismatch",
+    expected,
+    candidates: rankedCandidates
+  };
 }
 
 // src/lib/paa-answer-cleanup.ts
@@ -527,7 +1048,220 @@ function cleanPAAAnswerText(answer, question, sourceTitle) {
   return text;
 }
 
+// src/extractor/ai-surfaces.ts
+async function extractAISurfacesFromDocument(config) {
+  const selectors = config ?? {
+    aio: {
+      root: "[data-lhcontainer][data-streaming-container][eid]",
+      legacyRoot: '[data-hveid="CBMQAA"]',
+      wrapper: ".Fgyi2e",
+      controller: '[jscontroller="AkrxPe"]',
+      contentSubtree: '[data-subtree="mfc"]',
+      heading: ".Fzsovc.cwYVJe.RJPOee",
+      header: ".heWuVc",
+      showMoreButton: '[aria-label="Show more AI Overview"]',
+      sourcesPanel: ".OZ9ddf.WAUd4",
+      disclaimer: ".DuQANe.MSJHRb"
+    },
+    aim: {
+      root: '[data-hveid="CAUQAA"]',
+      wrapper: ".Fgyi2e"
+    },
+    expandWaitMs: 1500
+  };
+  const sn = window.google?.sn ?? "unknown";
+  const surface = sn === "aim" ? "aim" : sn === "web" ? "web" : "unknown";
+  function textOf(el) {
+    if (!el) return "";
+    return (el.innerText ?? el.textContent ?? "").trim();
+  }
+  function hasAIOverviewLabel(el) {
+    const heading = el.querySelector(selectors.aio.heading);
+    if (textOf(heading) === "AI Overview") return true;
+    const header = el.querySelector(selectors.aio.header);
+    if (textOf(header).split(/\n|\s{2,}/).some((part) => part.trim() === "AI Overview")) return true;
+    return textOf(el).includes("AI Overview");
+  }
+  function findAIORoot() {
+    const primaryRoots = Array.from(document.querySelectorAll(selectors.aio.root));
+    const labeledPrimary = primaryRoots.find(hasAIOverviewLabel);
+    if (labeledPrimary) return labeledPrimary;
+    if (primaryRoots.length > 0) return primaryRoots[0];
+    if (selectors.aio.legacyRoot) {
+      const legacy = document.querySelector(selectors.aio.legacyRoot);
+      if (legacy) return legacy;
+    }
+    const headings = document.querySelectorAll(`${selectors.aio.heading}, h1, h2, h3, [role="heading"]`);
+    for (const h of headings) {
+      if (textOf(h) !== "AI Overview") continue;
+      let el = h.parentElement;
+      for (let i = 0; i < 8 && el; i++) {
+        if (el.matches(selectors.aio.root) || el.querySelector(selectors.aio.controller) || el.querySelector(selectors.aio.contentSubtree)) {
+          return el;
+        }
+        el = el.parentElement;
+      }
+      return h.parentElement;
+    }
+    return null;
+  }
+  function cleanText(target) {
+    if (!target) return null;
+    const clone = target.cloneNode(true);
+    clone.querySelectorAll([
+      "script",
+      "style",
+      "noscript",
+      "img",
+      "picture",
+      "video",
+      selectors.aio.header,
+      selectors.aio.showMoreButton,
+      selectors.aio.sourcesPanel,
+      selectors.aio.disclaimer,
+      '[data-subtree="dfa"]',
+      "[data-src-id]",
+      '[role="dialog"]',
+      ".HWMcu",
+      ".bTFeG",
+      ".CyMdWb",
+      ".MFrAxb",
+      ".F0OfWd.hfWAgb",
+      ".x2qcTc.fZavHb",
+      ".SvjEff",
+      ".sR2MY",
+      ".lKuDef",
+      ".GSPQcc",
+      "a[href]",
+      "button",
+      '[role="button"]'
+    ].join(",")).forEach((el) => el.remove());
+    const holder = document.createElement("div");
+    holder.style.position = "fixed";
+    holder.style.left = "-10000px";
+    holder.style.top = "0";
+    holder.style.width = `${Math.max(320, Math.round(target.getBoundingClientRect?.().width || 960))}px`;
+    holder.style.opacity = "0";
+    holder.style.pointerEvents = "none";
+    holder.append(clone);
+    document.body.append(holder);
+    const rendered = clone.innerText || clone.textContent || "";
+    holder.remove();
+    const lines = rendered.replace(/\r/g, "").replace(/[ \t]+\n/g, "\n").replace(/\n[ \t]+/g, "\n").replace(/\n{3,}/g, "\n\n").replace(/[ \t]{2,}/g, " ").trim().split("\n").map((line) => line.replace(/\u00a0/g, " ").trim()).filter(Boolean);
+    const filteredLines = [];
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      const next = lines[i + 1] ?? "";
+      if (line === "AI Overview") continue;
+      if (line === "Show more") continue;
+      if (/^AI can make mistakes/i.test(line)) continue;
+      if (/^Thank you\b/i.test(line)) continue;
+      if (/^Your feedback helps Google improve/i.test(line)) continue;
+      if (/^\+?\d+$/.test(line)) continue;
+      if (/^\+\d+$/.test(next) && line.length <= 80) {
+        i++;
+        continue;
+      }
+      filteredLines.push(line);
+    }
+    const raw = filteredLines.join("\n").replace(/\n{3,}/g, "\n\n").trim();
+    if (!raw || /not available|try again|can't generate/i.test(raw)) return null;
+    return raw;
+  }
+  function normalizeHref(rawHref) {
+    if (!rawHref || rawHref.startsWith("javascript:")) return null;
+    let href = rawHref;
+    try {
+      const absolute = new URL(rawHref, window.location.href);
+      const q = absolute.searchParams.get("q") ?? absolute.searchParams.get("url");
+      if (/(\.|^)google\./i.test(absolute.hostname) && q?.startsWith("http")) {
+        href = q;
+      } else {
+        href = absolute.href;
+      }
+    } catch {
+      return null;
+    }
+    if (!/^https?:\/\//i.test(href)) return null;
+    try {
+      const url = new URL(href);
+      const isGoogleInternal = /(\.|^)google\./i.test(url.hostname);
+      if (isGoogleInternal) return null;
+      return url.href;
+    } catch {
+      return null;
+    }
+  }
+  function extractCitations(root) {
+    if (!root) return [];
+    const seen = /* @__PURE__ */ new Set();
+    const citations = [];
+    for (const a of Array.from(root.querySelectorAll("a[href]"))) {
+      const href = normalizeHref(a.getAttribute("href") ?? "");
+      if (!href || seen.has(href)) continue;
+      seen.add(href);
+      let fallbackHost = "";
+      try {
+        fallbackHost = new URL(href).hostname.replace(/^www\./, "");
+      } catch {
+      }
+      citations.push({
+        text: textOf(a) || fallbackHost || href,
+        href
+      });
+    }
+    return citations;
+  }
+  async function maybeExpand(root) {
+    const button = root.querySelector(selectors.aio.showMoreButton);
+    if (!button || button.getAttribute("aria-expanded") !== "false") return false;
+    button.click();
+    const waitMs = selectors.expandWaitMs ?? 1500;
+    if (waitMs > 0) await new Promise((resolve) => setTimeout(resolve, waitMs));
+    return true;
+  }
+  const aioRoot = findAIORoot();
+  let aioText = null;
+  let aioCitations = [];
+  let aioExpanded = false;
+  let aioFullyExpanded = false;
+  let aioSections = [];
+  if (aioRoot) {
+    aioExpanded = await maybeExpand(aioRoot);
+    const controller = aioRoot.querySelector(selectors.aio.controller);
+    const contentSubtree = aioRoot.querySelector(selectors.aio.contentSubtree);
+    const showMore = aioRoot.querySelector(selectors.aio.showMoreButton);
+    aioFullyExpanded = controller?.getAttribute("data-trnct") === "false" || showMore?.getAttribute("aria-expanded") === "true" || !showMore;
+    aioText = cleanText(contentSubtree ?? controller ?? aioRoot);
+    aioSections = (aioText ?? "").split("\n").map((line) => line.trim()).filter((line) => /^\d+\.\s+.+/.test(line));
+    aioCitations = extractCitations(aioRoot);
+  }
+  const aimRoot = document.querySelector(selectors.aim.root);
+  const aimDetected = surface === "aim" && !!aimRoot;
+  const aimContainer = aimRoot?.closest(selectors.aim.wrapper) ?? aimRoot;
+  const aimText = cleanText(aimContainer);
+  const aimCitations = aimDetected ? extractCitations(aimContainer) : [];
+  return {
+    surface,
+    aiOverview: {
+      detected: !!aioRoot && aioText !== null,
+      text: aioText,
+      citations: aioCitations,
+      expanded: aioExpanded,
+      fullyExpanded: aioFullyExpanded,
+      sections: aioSections
+    },
+    aiMode: {
+      detected: aimDetected && aimText !== null,
+      text: aimText,
+      citations: aimCitations
+    }
+  };
+}
+
 // src/extractor/PAAExtractor.ts
+var DESKTOP_USER_AGENT2 = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
+var MOBILE_USER_AGENT2 = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
 var PAAExtractor = class {
   constructor(driver, reporter) {
     this.driver = driver;
@@ -538,6 +1272,17 @@ var PAAExtractor = class {
   normalizeQuestion(q) {
     return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
   }
+  throwIfAborted(signal) {
+    if (!signal?.aborted) return;
+    if (signal.reason instanceof DOMException && signal.reason.name === "TimeoutError") throw signal.reason;
+    throw new RequestAbortedError();
+  }
+  async throwIfCaptcha(page, context) {
+    const captchaCount = await page.locator(PAASelectors.captchaMarker).count().catch(() => 0);
+    if (captchaCount > 0) {
+      throw new CaptchaError(`${context} returned a CAPTCHA \u2014 retrying with a fresh session.`);
+    }
+  }
   async extractVisibleItems(page) {
     const sels = PAASelectors;
     const raw = await page.evaluate((selectors) => {
@@ -600,10 +1345,10 @@ var PAAExtractor = class {
       extracted_at: (/* @__PURE__ */ new Date()).toISOString()
     };
   }
-  async runBFS(page, options) {
+  async runBFS(page, options, signal) {
     const seenKeys = /* @__PURE__ */ new Set();
     const seenQs = /* @__PURE__ */ new Set();
-    const depthMap = /* @__PURE__ */ new Map();
+    const orderedQs = [];
     const results = [];
     const readAllQs = () => page.evaluate(
       ({ sel, dataQ, dataInitQ, questionEl }) => Array.from(document.querySelectorAll(sel)).map(
@@ -611,42 +1356,43 @@ var PAAExtractor = class {
       ).filter(Boolean),
       { sel: PAASelectors.item, dataQ: PAASelectors.itemDataQ, dataInitQ: PAASelectors.itemDataInitQ, questionEl: PAASelectors.itemQuestionEl }
     );
-    const dupRates = [];
-    const orderedQs = [];
-    for (let round = 0; round < options.depth; round++) {
-      this.reporter.onDepth(round + 1);
-      if (seenQs.size >= options.maxQuestions) break;
+    let round = 0;
+    while (seenQs.size < options.maxQuestions) {
+      this.throwIfAborted(signal);
+      await this.throwIfCaptcha(page, "Google PAA expansion");
       const beforeQs = await readAllQs();
       if (beforeQs.length >= options.maxQuestions) break;
-      const unexpandedItems = await page.$$(
-        `${PAASelectors.item}:not(.${PAASelectors.expandedClass})`
-      );
-      if (unexpandedItems.length === 0) break;
-      for (const item of unexpandedItems) {
+      const unexpandedSel = `${PAASelectors.item}:not(.${PAASelectors.expandedClass}) ${PAASelectors.clickTarget}`;
+      const unexpandedCount = await page.locator(unexpandedSel).count();
+      if (unexpandedCount === 0) break;
+      this.reporter.onDepth(++round);
+      for (let ci = 0; ci < unexpandedCount; ci++) {
+        this.throwIfAborted(signal);
         try {
-          await item.scrollIntoViewIfNeeded();
-          await item.click({ force: true });
+          const btn = page.locator(unexpandedSel).first();
+          await btn.scrollIntoViewIfNeeded();
+          await btn.hover({ force: true });
+          await page.waitForTimeout(100);
+          await btn.click({ force: true });
           await page.waitForTimeout(500);
         } catch {
         }
       }
-      await page.waitForTimeout(1500);
+      await page.waitForFunction(
+        ({ sel, min }) => document.querySelectorAll(sel).length > min,
+        { sel: PAASelectors.item, min: beforeQs.length },
+        { timeout: 5e3 }
+      ).catch(() => {
+      });
+      await this.throwIfCaptcha(page, "Google PAA expansion");
       const afterQs = await readAllQs();
-      const newQs = afterQs.slice(beforeQs.length);
-      const newDups = newQs.filter((q) => seenQs.has(q)).length;
-      const dupRate = newQs.length > 0 ? newDups / newQs.length : 0;
-      dupRates.push(dupRate);
-      if (dupRates.length > 2) dupRates.shift();
-      const rollingDupRate = dupRates.reduce((a, b) => a + b, 0) / dupRates.length;
+      if (afterQs.length === beforeQs.length) break;
       for (const q of afterQs) {
         if (!seenQs.has(q)) {
           seenQs.add(q);
           orderedQs.push(q);
         }
-        if (!depthMap.has(q)) depthMap.set(q, round + 1);
       }
-      if (afterQs.length === beforeQs.length) break;
-      if (rollingDupRate >= 0.6) break;
     }
     const itemMap = new Map((await this.extractVisibleItems(page)).map((i) => [i.question, i]));
     for (const q of orderedQs) {
@@ -654,13 +1400,12 @@ var PAAExtractor = class {
       const key = this.normalizeQuestion(q);
       if (seenKeys.has(key)) continue;
       seenKeys.add(key);
-      const d = depthMap.get(q) ?? 1;
       const item = itemMap.get(q);
       if (item) {
-        results.push(this.toFlatRow(item, d, null, options.query));
-        this.reporter.onQuestion({ question: item.question, answer: item.answer ?? null, sourceTitle: item.sourceTitle ?? null, sourceSite: item.sourceSite ?? null, sourceCite: item.sourceCite ?? null, depth: d, parentQuestion: null, children: [] });
+        results.push(this.toFlatRow(item, 1, null, options.query));
+        this.reporter.onQuestion({ question: item.question, answer: item.answer ?? null, sourceTitle: item.sourceTitle ?? null, sourceSite: item.sourceSite ?? null, sourceCite: item.sourceCite ?? null, depth: 1, parentQuestion: null, children: [] });
       } else {
-        results.push(this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, d, null, options.query));
+        results.push(this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, 1, null, options.query));
       }
     }
     return results;
@@ -718,6 +1463,7 @@ var PAAExtractor = class {
     } catch {
       return [];
     }
+    await this.throwIfCaptcha(page, "Google short video search");
     const svSels = {
       item: ShortVideoSelectors.item,
       platforms: [...ShortVideoSelectors.platforms]
@@ -999,69 +1745,11 @@ var PAAExtractor = class {
     return { ...entityIds, entities: records, cids: [...cidSet] };
   }
   async extractAISurfaces(page) {
-    const aioSels = AIOverviewSelectors;
-    const aimSels = AIModeSelectors;
-    return page.evaluate(({ aio, aim }) => {
-      const sn = window.google?.sn ?? "unknown";
-      const surface = sn === "aim" ? "aim" : sn === "web" ? "web" : "unknown";
-      function findAIORoot() {
-        const primary = document.querySelector(aio.root);
-        if (primary) return primary;
-        const headings = document.querySelectorAll('h1, h2, h3, [role="heading"]');
-        for (const h of headings) {
-          if (h.textContent?.trim() === "AI Overview") {
-            let el = h.parentElement;
-            for (let i = 0; i < 6 && el; i++) {
-              if (el.querySelectorAll("a").length > 1) return el;
-              el = el.parentElement;
-            }
-            return h.parentElement;
-          }
-        }
-        return null;
-      }
-      const aioRoot = findAIORoot();
-      const aioContainer = aioRoot ? aioRoot.closest(aio.wrapper) ?? aioRoot : null;
-      let aioText = null;
-      if (aioContainer) {
-        const clone = aioContainer.cloneNode(true);
-        clone.querySelectorAll("script,style,noscript").forEach((el) => el.remove());
-        clone.querySelectorAll('h1,h2,h3,h4,[role="heading"]').forEach((el) => el.remove());
-        clone.querySelectorAll('button,[role="button"]').forEach((el) => el.remove());
-        clone.querySelectorAll("a").forEach((el) => el.remove());
-        const candidate = clone.textContent?.replace(/\s+/g, " ").trim() || null;
-        const isErrorState = !candidate || /not available|try again|can't generate/i.test(candidate);
-        aioText = isErrorState ? null : candidate;
-      }
-      const aioDetected = !!aioRoot && aioText !== null;
-      const aioCitations = Array.from(aioContainer?.querySelectorAll("a[href]") ?? []).filter((a) => a.href && !a.href.startsWith("javascript")).map((a) => ({
-        text: a.textContent?.trim() ?? "",
-        href: a.href
-      })).filter((c) => c.text && c.href);
-      const aimRoot = document.querySelector(aim.root);
-      const aimDetected = surface === "aim" && !!aimRoot;
-      const aimContainer = aimRoot?.closest(aim.wrapper) ?? null;
-      let aimText = null;
-      if (aimContainer) {
-        const clone = aimContainer.cloneNode(true);
-        clone.querySelectorAll("script,style,noscript").forEach((el) => el.remove());
-        clone.querySelectorAll('h1,h2,h3,h4,[role="heading"]').forEach((el) => el.remove());
-        clone.querySelectorAll('button,[role="button"]').forEach((el) => el.remove());
-        clone.querySelectorAll("a").forEach((el) => el.remove());
-        const candidate = clone.textContent?.replace(/\s+/g, " ").trim() || null;
-        const isErrorState = !candidate || /not available|try again|can't generate/i.test(candidate);
-        aimText = isErrorState ? null : candidate;
-      }
-      const aimCitations = aimDetected ? Array.from(aimContainer?.querySelectorAll("a[href]") ?? []).filter((a) => a.href && !a.href.startsWith("javascript")).map((a) => ({
-        text: a.textContent?.trim() ?? "",
-        href: a.href
-      })).filter((c) => c.text && c.href) : [];
-      return {
-        surface,
-        aiOverview: { detected: aioDetected, text: aioText, citations: aioCitations },
-        aiMode: { detected: aimDetected, text: aimText, citations: aimCitations }
-      };
-    }, { aio: aioSels, aim: aimSels });
+    return page.evaluate(extractAISurfacesFromDocument, {
+      aio: AIOverviewSelectors,
+      aim: AIModeSelectors,
+      expandWaitMs: 1500
+    });
   }
   buildTree(flat, _seed) {
     const roots = [];
@@ -1088,23 +1776,70 @@ var PAAExtractor = class {
     }
     return roots;
   }
-  async extract(options) {
+  getBrowserDebugSnapshot() {
+    return this.driver.getDebugSnapshot();
+  }
+  buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence) {
+    if (!options.debug) return void 0;
+    return {
+      enabled: true,
+      request: {
+        query: options.query,
+        locationInput: options.location ?? null,
+        canonicalLocation,
+        uule,
+        gl: options.gl,
+        hl: options.hl,
+        device: options.device,
+        proxyMode: options.proxyMode,
+        proxyZip: options.proxyZip ?? null,
+        serpOnly: options.serpOnly,
+        pages: options.pages ?? 1
+      },
+      browser: this.getBrowserDebugSnapshot(),
+      ...locationEvidence ? { locationEvidence } : {}
+    };
+  }
+  async extract(options, signal) {
     const startMs = Date.now();
+    const isMobile = options.device === "mobile";
     const config = {
       headless: options.headless,
       profileDir: options.profileDir,
       proxy: options.proxy,
       kernelApiKey: options.kernelApiKey,
       kernelProxyId: options.kernelProxyId,
-      viewport: { width: 1280, height: 800 },
-      locale: `${options.hl}-${options.gl.toUpperCase()}`
+      kernelProxyResolution: options.kernelProxyResolution,
+      proxyMode: options.proxyMode,
+      viewport: isMobile ? { width: 390, height: 844 } : { width: 1280, height: 800 },
+      locale: `${options.hl}-${options.gl.toUpperCase()}`,
+      userAgent: isMobile ? MOBILE_USER_AGENT2 : DESKTOP_USER_AGENT2,
+      deviceScaleFactor: isMobile ? 3 : 1,
+      isMobile,
+      hasTouch: isMobile,
+      debug: options.debug
     };
     let errorCount = 0;
+    const diagnosticWarnings = [];
     try {
+      this.throwIfAborted(signal);
       await this.driver.launch(config);
-      const uule = options.location ? encodeUule(normalizeLocation(options.location)) : null;
-      const { hasPaa } = await this.driver.navigateToSERP(options.query, uule, options.gl, options.hl);
+      this.throwIfAborted(signal);
+      const canonicalLocation = options.location ? normalizeLocation(options.location) : null;
+      const uule = canonicalLocation ? encodeUule(canonicalLocation) : null;
+      const { hasPaa } = await this.driver.navigateToSERP(
+        options.query,
+        uule,
+        options.gl,
+        options.hl,
+        {
+          ...options.serpOnly ? { num: 100 } : {},
+          debug: options.debug
+        }
+      );
+      this.throwIfAborted(signal);
       const page = this.driver.getPage();
+      await this.throwIfCaptcha(page, "Google SERP");
       if (options.serpOnly) {
         const [organicResults2, localPack2, rawEntityIds2] = await Promise.all([
           this.extractOrganicResults(page),
@@ -1112,13 +1847,19 @@ var PAAExtractor = class {
           this.extractEntityIds(page)
         ]);
         const entityIds2 = this.mergeLocalPackIntoEntities(rawEntityIds2, localPack2);
+        const aiSurfaces2 = await this.extractAISurfaces(page);
+        let locationEvidence2 = options.debug ? inferSerpLocationEvidence(canonicalLocation, organicResults2, localPack2) : void 0;
         let allOrganic2 = organicResults2;
         if ((options.pages ?? 1) >= 2) {
-          const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, start: "10" });
+          const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
           if (uule) p2params.set("uule", uule);
           await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
+          await this.throwIfCaptcha(page, "Google SERP page 2");
           const p2organic = await this.extractOrganicResults(page);
           allOrganic2 = [...organicResults2, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
+          if (options.debug) {
+            locationEvidence2 = inferSerpLocationEvidence(canonicalLocation, allOrganic2, localPack2);
+          }
         }
         const stats2 = {
           seed: options.query,
@@ -1132,10 +1873,15 @@ var PAAExtractor = class {
           seed: options.query,
           location: options.location ?? null,
           extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          diagnostics: {
+            completionStatus: "serp_only",
+            problem: null,
+            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence2) } : {}
+          },
           totalQuestions: 0,
-          surface: "web",
-          aiOverview: { detected: false, text: null, citations: [] },
-          aiMode: { detected: false, text: null, citations: [] },
+          surface: aiSurfaces2.surface,
+          aiOverview: aiSurfaces2.aiOverview,
+          aiMode: aiSurfaces2.aiMode,
           whatPeopleSaying: [],
           tree: [],
           flat: [],
@@ -1156,16 +1902,22 @@ var PAAExtractor = class {
         this.extractLocalPack(page)
       ]);
       const entityIds = this.mergeLocalPackIntoEntities(rawEntityIds, localPack);
+      const initialLocationEvidence = options.debug ? inferSerpLocationEvidence(canonicalLocation, organicResults, localPack) : void 0;
       this.reporter.onVideos(videos);
       this.reporter.onForums(forums);
       if (!hasPaa) {
         let noPaaOrganic = organicResults;
+        let locationEvidence2 = initialLocationEvidence;
         if ((options.pages ?? 1) >= 2) {
-          const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, start: "10" });
+          const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
           if (uule) p2params.set("uule", uule);
           await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
+          await this.throwIfCaptcha(page, "Google SERP page 2");
           const p2organic = await this.extractOrganicResults(page);
           noPaaOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
+          if (options.debug) {
+            locationEvidence2 = inferSerpLocationEvidence(canonicalLocation, noPaaOrganic, localPack);
+          }
         }
         const aiSurfaces2 = await this.extractAISurfaces(page);
         const stats2 = {
@@ -1180,6 +1932,11 @@ var PAAExtractor = class {
           seed: options.query,
           location: options.location ?? null,
           extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          diagnostics: {
+            completionStatus: "no_paa",
+            problem: null,
+            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence2) } : {}
+          },
           totalQuestions: 0,
           surface: aiSurfaces2.surface,
           aiOverview: aiSurfaces2.aiOverview,
@@ -1195,19 +1952,37 @@ var PAAExtractor = class {
           stats: stats2
         };
       }
-      const flat = await this.runBFS(page, options);
+      const flat = await this.runBFS(page, options, signal);
+      this.throwIfAborted(signal);
       const aiSurfaces = await this.extractAISurfaces(page);
-      const shortVidsParams = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, udm: ShortVideoSelectors.udm });
+      const shortVidsParams = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", udm: ShortVideoSelectors.udm });
       if (uule) shortVidsParams.set("uule", uule);
-      const shortVideos = await this.extractShortVideos(page, "https://www.google.com/search?" + shortVidsParams.toString());
+      let shortVideos = [];
+      try {
+        shortVideos = await this.extractShortVideos(page, "https://www.google.com/search?" + shortVidsParams.toString());
+      } catch (err) {
+        if (!(err instanceof CaptchaError)) throw err;
+        errorCount++;
+        diagnosticWarnings.push({
+          code: "short_videos_captcha_skipped",
+          surface: "short_videos",
+          message: err.message,
+          retryable: true
+        });
+      }
       this.reporter.onVideos(shortVideos);
       let allOrganic = organicResults;
+      let locationEvidence = initialLocationEvidence;
       if ((options.pages ?? 1) >= 2) {
-        const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, start: "10" });
+        const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
         if (uule) p2params.set("uule", uule);
         await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
+        await this.throwIfCaptcha(page, "Google SERP page 2");
         const p2organic = await this.extractOrganicResults(page);
         allOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
+        if (options.debug) {
+          locationEvidence = inferSerpLocationEvidence(canonicalLocation, allOrganic, localPack);
+        }
       }
       const allVideos = [...videos, ...shortVideos];
       const tree = this.buildTree(flat, options.query);
@@ -1223,6 +1998,12 @@ var PAAExtractor = class {
         seed: options.query,
         location: options.location ?? null,
         extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        diagnostics: {
+          completionStatus: "paa_found",
+          problem: null,
+          ...diagnosticWarnings.length > 0 ? { warnings: diagnosticWarnings } : {},
+          ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence) } : {}
+        },
         totalQuestions: flat.length,
         surface: aiSurfaces.surface,
         aiOverview: aiSurfaces.aiOverview,
@@ -1241,8 +2022,6 @@ var PAAExtractor = class {
       errorCount++;
       this.reporter.onError(err instanceof Error ? err : new Error(String(err)));
       throw err;
-    } finally {
-      await this.driver.close();
     }
   }
 };
@@ -1356,53 +2135,646 @@ var ProgressReporter = class {
   }
 };
 
+// src/kernel-proxy-resolver.ts
+var import_sdk2 = __toESM(require("@onkernel/sdk"), 1);
+var US_STATE_CODES = {
+  alabama: "AL",
+  alaska: "AK",
+  arizona: "AZ",
+  arkansas: "AR",
+  california: "CA",
+  colorado: "CO",
+  connecticut: "CT",
+  delaware: "DE",
+  florida: "FL",
+  georgia: "GA",
+  hawaii: "HI",
+  idaho: "ID",
+  illinois: "IL",
+  indiana: "IN",
+  iowa: "IA",
+  kansas: "KS",
+  kentucky: "KY",
+  louisiana: "LA",
+  maine: "ME",
+  maryland: "MD",
+  massachusetts: "MA",
+  michigan: "MI",
+  minnesota: "MN",
+  mississippi: "MS",
+  missouri: "MO",
+  montana: "MT",
+  nebraska: "NE",
+  nevada: "NV",
+  "new hampshire": "NH",
+  "new jersey": "NJ",
+  "new mexico": "NM",
+  "new york": "NY",
+  "north carolina": "NC",
+  "north dakota": "ND",
+  ohio: "OH",
+  oklahoma: "OK",
+  oregon: "OR",
+  pennsylvania: "PA",
+  "rhode island": "RI",
+  "south carolina": "SC",
+  "south dakota": "SD",
+  tennessee: "TN",
+  texas: "TX",
+  utah: "UT",
+  vermont: "VT",
+  virginia: "VA",
+  washington: "WA",
+  "west virginia": "WV",
+  wisconsin: "WI",
+  wyoming: "WY"
+};
+var US_CITY_CENTER_ZIPS = {
+  "atlanta|GA": "30303",
+  "austin|TX": "78701",
+  "baltimore|MD": "21201",
+  "boston|MA": "02108",
+  "boulder|CO": "80302",
+  "charlotte|NC": "28202",
+  "chicago|IL": "60601",
+  "colorado_springs|CO": "80903",
+  "columbus|OH": "43215",
+  "dallas|TX": "75201",
+  "denver|CO": "80202",
+  "detroit|MI": "48226",
+  "fort_collins|CO": "80524",
+  "fort_worth|TX": "76102",
+  "houston|TX": "77002",
+  "indianapolis|IN": "46204",
+  "jacksonville|FL": "32202",
+  "las_vegas|NV": "89101",
+  "los_angeles|CA": "90012",
+  "louisville|KY": "40202",
+  "loveland|CO": "80537",
+  "memphis|TN": "38103",
+  "miami|FL": "33131",
+  "minneapolis|MN": "55401",
+  "nashville|TN": "37203",
+  "new_york|NY": "10001",
+  "orlando|FL": "32801",
+  "philadelphia|PA": "19103",
+  "phoenix|AZ": "85004",
+  "portland|OR": "97205",
+  "raleigh|NC": "27601",
+  "richmond|VA": "23219",
+  "sacramento|CA": "95814",
+  "salt_lake_city|UT": "84101",
+  "san_antonio|TX": "78205",
+  "san_diego|CA": "92101",
+  "san_francisco|CA": "94103",
+  "san_jose|CA": "95113",
+  "seattle|WA": "98101"
+};
+function proxyIdSuffix2(proxyId) {
+  return proxyId ? proxyId.slice(-6) : null;
+}
+function resolution(source, proxyMode, proxyId, target, error) {
+  return {
+    kernelProxyId: proxyId,
+    resolution: {
+      source,
+      proxyMode,
+      proxyIdPresent: Boolean(proxyId),
+      proxyIdSuffix: proxyIdSuffix2(proxyId),
+      target,
+      error
+    }
+  };
+}
+function normalizeStateName(value) {
+  return value.trim().toLowerCase().replace(/\s+/g, " ");
+}
+function normalizeCountryName(value) {
+  return value.trim().toLowerCase().replace(/\./g, "").replace(/\s+/g, " ");
+}
+function isUnitedStates(country) {
+  if (!country) return true;
+  const normalized = normalizeCountryName(country);
+  return normalized === "united states" || normalized === "united states of america" || normalized === "usa" || normalized === "us";
+}
+function stateCodeFor(region) {
+  const trimmed = region.trim();
+  if (/^[A-Za-z]{2}$/.test(trimmed)) return trimmed.toUpperCase();
+  return US_STATE_CODES[normalizeStateName(trimmed)] ?? null;
+}
+function kernelCityIdentifierCandidates(city) {
+  const ascii = city.normalize("NFKD").replace(/[^\x00-\x7F]/g, "").toLowerCase();
+  const words = ascii.split(/[^a-z0-9]+/).filter(Boolean);
+  const underscored = words.join("_");
+  const compact = words.join("");
+  return Array.from(new Set([underscored, compact].filter(Boolean)));
+}
+function proxyName(country, state, city) {
+  return city ? `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}-${city}` : `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}`;
+}
+function zipProxyName(zip) {
+  return `mcp-serp-residential-us-zip-${zip}`;
+}
+function parseKernelLocationProxyTarget(location, gl) {
+  if (!location || gl.toLowerCase() !== "us") return null;
+  const canonicalLocation = normalizeLocation(location);
+  let parts = canonicalLocation.split(",").map((part) => part.trim()).filter(Boolean);
+  if (parts.length > 1 && isUnitedStates(parts[parts.length - 1])) {
+    parts = parts.slice(0, -1);
+  }
+  if (parts.length === 1) {
+    const stateOnly = stateCodeFor(parts[0]);
+    if (!stateOnly) return null;
+    return {
+      canonicalLocation,
+      level: "state",
+      country: "US",
+      state: stateOnly,
+      city: "",
+      cityCandidates: [],
+      proxyName: proxyName("US", stateOnly),
+      config: {
+        country: "US",
+        state: stateOnly
+      }
+    };
+  }
+  const [city = "", region = ""] = parts;
+  if (!city || !region) return null;
+  const state = stateCodeFor(region);
+  if (!state) return null;
+  const cityCandidates = kernelCityIdentifierCandidates(city);
+  const primaryCity = cityCandidates[0];
+  if (!primaryCity) return null;
+  return {
+    canonicalLocation,
+    level: "city",
+    country: "US",
+    state,
+    city: primaryCity,
+    cityCandidates,
+    proxyName: proxyName("US", state, primaryCity),
+    config: {
+      country: "US",
+      state,
+      city: primaryCity
+    }
+  };
+}
+function cityZipKey(target) {
+  return `${target.city}|${target.state}`;
+}
+function knownZipFor(target, explicitZip) {
+  if (explicitZip && /^\d{5}$/.test(explicitZip)) return explicitZip;
+  return US_CITY_CENTER_ZIPS[cityZipKey(target)] ?? null;
+}
+function zipTarget(target, zip) {
+  return {
+    ...target,
+    level: "zip",
+    zip,
+    proxyName: zipProxyName(zip),
+    config: {
+      country: target.country,
+      state: target.state,
+      zip
+    }
+  };
+}
+function configMatches(config, target, city) {
+  if (target.level === "zip") {
+    return config?.country?.toUpperCase() === target.country && config?.zip === target.zip;
+  }
+  return config?.country?.toUpperCase() === target.country && config?.state?.toUpperCase() === target.state && (city ? config?.city === city : !config?.city);
+}
+function findExistingTargetProxy(proxies, target) {
+  return proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === target.proxyName || configMatches(proxy.config, target, target.level === "city" ? target.city : void 0))) ?? null;
+}
+function findExistingProxy(proxies, target) {
+  for (const city of target.cityCandidates) {
+    const name = proxyName(target.country, target.state, city);
+    const found = proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === name || configMatches(proxy.config, target, city)));
+    if (found) return found;
+  }
+  return null;
+}
+function stateTarget(target) {
+  return {
+    ...target,
+    level: "state",
+    proxyName: proxyName(target.country, target.state),
+    config: {
+      country: target.country,
+      state: target.state
+    }
+  };
+}
+function findExistingStateProxy(proxies, target) {
+  const name = proxyName(target.country, target.state);
+  return proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === name || configMatches(proxy.config, target))) ?? null;
+}
+function escalatedTargetLevel(target, attemptIndex) {
+  return stateTarget(target);
+}
+function errorText2(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+async function resolveKernelProxyId(options) {
+  if (options.proxyMode === "none") {
+    return resolution("disabled", options.proxyMode, void 0, null, null);
+  }
+  if (options.proxyMode === "configured") {
+    return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, null, null);
+  }
+  const target = parseKernelLocationProxyTarget(options.location, options.gl);
+  if (!target || !options.kernelApiKey) {
+    return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, target ? null : "location could not be normalized to a US city/state proxy target");
+  }
+  const kernel = new import_sdk2.default({ apiKey: options.kernelApiKey });
+  try {
+    const attemptIndex = options.attemptIndex ?? 0;
+    if (attemptIndex >= 1) {
+      const escalatedTarget = escalatedTargetLevel(target, attemptIndex);
+      const createErrors2 = [];
+      try {
+        const created = await kernel.proxies.create({
+          type: "residential",
+          name: escalatedTarget.proxyName,
+          config: escalatedTarget.config
+        });
+        if (created.id) {
+          return resolution("location_created", options.proxyMode, created.id, escalatedTarget, null);
+        }
+        createErrors2.push(`${escalatedTarget.state}: Kernel did not return a proxy id`);
+      } catch (err) {
+        createErrors2.push(`${escalatedTarget.state}: ${errorText2(err)}`);
+      }
+      return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, escalatedTarget, createErrors2.join(" | "));
+    }
+    const proxies = await kernel.proxies.list();
+    const zip = knownZipFor(target, options.proxyZip);
+    const createErrors = [];
+    if (zip) {
+      const targetZip = zipTarget(target, zip);
+      const existingZip = findExistingTargetProxy(proxies, targetZip);
+      if (existingZip?.id) {
+        return resolution("location_reused", options.proxyMode, existingZip.id, targetZip, null);
+      }
+      try {
+        const created = await kernel.proxies.create({
+          type: "residential",
+          name: targetZip.proxyName,
+          config: {
+            country: targetZip.country,
+            zip
+          }
+        });
+        if (created.id) {
+          return resolution("location_created", options.proxyMode, created.id, targetZip, null);
+        }
+        createErrors.push(`${zip}: Kernel did not return a proxy id`);
+      } catch (err) {
+        createErrors.push(`${zip}: ${errorText2(err)}`);
+      }
+    }
+    const existing = findExistingProxy(proxies, target);
+    if (existing?.id) {
+      return resolution("location_reused", options.proxyMode, existing.id, target, createErrors.join(" | ") || null);
+    }
+    for (const city of target.cityCandidates) {
+      try {
+        const created = await kernel.proxies.create({
+          type: "residential",
+          name: proxyName(target.country, target.state, city),
+          config: {
+            country: target.country,
+            state: target.state,
+            city
+          }
+        });
+        if (created.id) {
+          return resolution("location_created", options.proxyMode, created.id, {
+            ...target,
+            level: "city",
+            city,
+            proxyName: proxyName(target.country, target.state, city),
+            config: {
+              country: target.country,
+              state: target.state,
+              city
+            }
+          }, null);
+        }
+        createErrors.push(`${city}: Kernel did not return a proxy id`);
+      } catch (err) {
+        createErrors.push(`${city}: ${errorText2(err)}`);
+      }
+    }
+    const fallbackTarget = stateTarget(target);
+    const existingState = findExistingStateProxy(proxies, fallbackTarget);
+    if (existingState?.id) {
+      return resolution("location_reused", options.proxyMode, existingState.id, fallbackTarget, createErrors.join(" | "));
+    }
+    try {
+      const created = await kernel.proxies.create({
+        type: "residential",
+        name: fallbackTarget.proxyName,
+        config: fallbackTarget.config
+      });
+      if (created.id) {
+        return resolution("location_created", options.proxyMode, created.id, fallbackTarget, createErrors.join(" | "));
+      }
+      createErrors.push(`${fallbackTarget.state}: Kernel did not return a proxy id`);
+    } catch (err) {
+      createErrors.push(`${fallbackTarget.state}: ${errorText2(err)}`);
+    }
+    return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
+  } catch (err) {
+    return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, errorText2(err));
+  }
+}
+
 // src/harvest.ts
 var MAX_ATTEMPTS = 3;
-async function extractOnce(options) {
+function abortReason(signal) {
+  if (signal.reason instanceof DOMException && signal.reason.name === "TimeoutError") return signal.reason;
+  return new RequestAbortedError();
+}
+function getAbortSignal(rawOptions) {
+  if (!rawOptions || typeof rawOptions !== "object") return void 0;
+  const signal = rawOptions.signal;
+  if (signal instanceof AbortSignal) return signal;
+  return void 0;
+}
+function getAttemptLogSink(rawOptions) {
+  if (!rawOptions || typeof rawOptions !== "object") return void 0;
+  const sink = rawOptions.onAttemptEvent;
+  return typeof sink === "function" ? sink : void 0;
+}
+async function emitAttemptEvent(sink, event) {
+  if (!sink) return;
+  try {
+    await sink(event);
+  } catch (err) {
+    console.warn(JSON.stringify({
+      event: "harvest_attempt_log_failed",
+      attempt_number: event.attemptNumber,
+      message: err instanceof Error ? err.message : String(err)
+    }));
+  }
+}
+function classifyAttemptError(err) {
+  if (err instanceof CaptchaError) return "captcha";
+  if (err instanceof RequestAbortedError) return "request_aborted";
+  if (err instanceof DOMException && (err.name === "TimeoutError" || err.name === "AbortError")) return "timeout";
+  const message = err instanceof Error ? err.message : String(err);
+  return /timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message) ? "timeout" : "error";
+}
+function classifyAttemptResult(result) {
+  return result.diagnostics?.completionStatus ?? (result.totalQuestions > 0 ? "paa_found" : "no_paa");
+}
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+async function extractOnce(options, signal) {
   const driver = new BrowserDriver();
   const reporter = new ProgressReporter();
   const extractor = new PAAExtractor(driver, reporter);
+  if (signal?.aborted) {
+    return {
+      result: null,
+      error: abortReason(signal),
+      cleanup: await driver.close(),
+      debug: null
+    };
+  }
+  let onAbort;
+  const abortPromise = signal ? new Promise((_, reject) => {
+    onAbort = () => reject(abortReason(signal));
+    signal.addEventListener("abort", onAbort, { once: true });
+  }) : null;
+  let result = null;
+  let error = null;
+  let cleanup;
+  let debug = null;
   try {
-    return await extractor.extract(options);
+    const extraction = extractor.extract(options, signal);
+    if (abortPromise) extraction.catch(() => {
+    });
+    result = await (abortPromise ? Promise.race([extraction, abortPromise]) : extraction);
+  } catch (err) {
+    error = err;
   } finally {
-    await driver.close();
+    if (signal && onAbort) signal.removeEventListener("abort", onAbort);
+    debug = result?.diagnostics.debug ?? (options.debug ? {
+      enabled: true,
+      request: {
+        query: options.query,
+        locationInput: options.location ?? null,
+        canonicalLocation: null,
+        uule: null,
+        gl: options.gl,
+        hl: options.hl,
+        device: options.device,
+        proxyMode: options.proxyMode,
+        proxyZip: options.proxyZip ?? null,
+        serpOnly: options.serpOnly,
+        pages: options.pages ?? 1
+      },
+      browser: driver.getDebugSnapshot()
+    } : null);
+    cleanup = await driver.close();
   }
+  return error ? { result: null, error, cleanup, debug } : { result, error: null, cleanup, debug };
 }
 async function harvest(rawOptions) {
   const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};
-  const merged = {
-    kernelApiKey: process.env.KERNEL_API_KEY?.trim(),
-    kernelProxyId: process.env.KERNEL_PROXY_ID?.trim(),
-    ...raw
+  const signal = getAbortSignal(rawOptions);
+  const onAttemptEvent = getAttemptLogSink(rawOptions);
+  const requestedProxyMode = raw.proxyMode;
+  const proxyMode = requestedProxyMode === "none" ? "none" : requestedProxyMode === "configured" ? "configured" : "location";
+  const kernelApiKey = typeof raw.kernelApiKey === "string" ? raw.kernelApiKey.trim() : process.env.KERNEL_API_KEY?.trim();
+  const configuredKernelProxyId = typeof raw.kernelProxyId === "string" ? raw.kernelProxyId.trim() : process.env.KERNEL_PROXY_ID?.trim();
+  const proxyOpts = {
+    kernelApiKey,
+    proxyMode,
+    configuredKernelProxyId,
+    location: typeof raw.location === "string" ? raw.location : void 0,
+    proxyZip: typeof raw.proxyZip === "string" ? raw.proxyZip : void 0,
+    gl: typeof raw.gl === "string" ? raw.gl : "us"
   };
-  const options = HarvestOptionsSchema.parse(merged);
   const serializer = new OutputSerializer();
   for (let i = 0; i < MAX_ATTEMPTS; i++) {
+    const attemptNumber = i + 1;
+    const startedAtMs = Date.now();
     try {
-      const result = await extractOnce(options);
-      if (options.format === "json" || options.format === "both") {
-        await serializer.writeJSON(result, options.outputDir);
+      if (signal?.aborted) throw abortReason(signal);
+      const resolution2 = await resolveKernelProxyId({ ...proxyOpts, attemptIndex: i });
+      const mergedAttempt = {
+        ...raw,
+        kernelApiKey,
+        kernelProxyId: resolution2.kernelProxyId,
+        kernelProxyResolution: resolution2.resolution,
+        proxyMode
+      };
+      if (proxyMode === "none") mergedAttempt.kernelProxyId = void 0;
+      const attemptOptions = HarvestOptionsSchema.parse(mergedAttempt);
+      await emitAttemptEvent(onAttemptEvent, {
+        type: "started",
+        attemptNumber,
+        maxAttempts: MAX_ATTEMPTS,
+        query: attemptOptions.query,
+        location: attemptOptions.location ?? null,
+        maxQuestions: attemptOptions.maxQuestions,
+        startedAt: new Date(startedAtMs).toISOString()
+      });
+      console.info(JSON.stringify({
+        event: "harvest_attempt_started",
+        attempt_number: attemptNumber,
+        max_attempts: MAX_ATTEMPTS,
+        query: attemptOptions.query,
+        location: attemptOptions.location ?? null,
+        max_questions: attemptOptions.maxQuestions
+      }));
+      const attempt = await extractOnce(attemptOptions, signal);
+      if (attempt.error) {
+        const err = attempt.error;
+        if (err instanceof CaptchaError) {
+          const willRetry = i < MAX_ATTEMPTS - 1;
+          console.warn(JSON.stringify({
+            event: "harvest_attempt_captcha",
+            attempt_number: attemptNumber,
+            max_attempts: MAX_ATTEMPTS,
+            message: err.message,
+            will_retry: willRetry
+          }));
+          await emitAttemptEvent(onAttemptEvent, {
+            type: "finished",
+            attemptNumber,
+            maxAttempts: MAX_ATTEMPTS,
+            outcome: "captcha",
+            kernelSessionId: attempt.cleanup.kernelSessionId,
+            questionCount: 0,
+            durationMs: Date.now() - startedAtMs,
+            error: err.message,
+            willRetry,
+            cleanup: attempt.cleanup,
+            debug: attempt.debug,
+            completedAt: (/* @__PURE__ */ new Date()).toISOString()
+          });
+          if (willRetry) continue;
+          break;
+        }
+        await emitAttemptEvent(onAttemptEvent, {
+          type: "finished",
+          attemptNumber,
+          maxAttempts: MAX_ATTEMPTS,
+          outcome: classifyAttemptError(err),
+          kernelSessionId: attempt.cleanup.kernelSessionId,
+          questionCount: 0,
+          durationMs: Date.now() - startedAtMs,
+          error: errorMessage(err),
+          willRetry: false,
+          cleanup: attempt.cleanup,
+          debug: attempt.debug,
+          completedAt: (/* @__PURE__ */ new Date()).toISOString()
+        });
+        throw err;
+      }
+      const result = attempt.result;
+      if (!result) throw new Error("Harvest attempt completed without a result");
+      await emitAttemptEvent(onAttemptEvent, {
+        type: "finished",
+        attemptNumber,
+        maxAttempts: MAX_ATTEMPTS,
+        outcome: classifyAttemptResult(result),
+        kernelSessionId: attempt.cleanup.kernelSessionId,
+        questionCount: result.totalQuestions,
+        durationMs: Date.now() - startedAtMs,
+        error: null,
+        willRetry: false,
+        cleanup: attempt.cleanup,
+        debug: attempt.debug,
+        completedAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
+      if (attemptOptions.format === "json" || attemptOptions.format === "both") {
+        await serializer.writeJSON(result, attemptOptions.outputDir);
       }
-      if (options.format === "csv" || options.format === "both") {
+      if (attemptOptions.format === "csv" || attemptOptions.format === "both") {
         await Promise.all([
-          serializer.writeCSV(result.flat, options.outputDir),
-          result.videos.length > 0 ? serializer.writeVideoCSV(result.videos, result.seed, options.outputDir) : Promise.resolve(""),
-          result.forums.length > 0 ? serializer.writeForumCSV(result.forums, result.seed, options.outputDir) : Promise.resolve(""),
-          result.aiOverview.detected ? serializer.writeAIOverviewCSV(result.aiOverview.citations, result.aiOverview.text, result.seed, options.outputDir) : Promise.resolve(""),
-          result.aiMode.detected ? serializer.writeAIModeCSV(result.aiMode.citations, result.aiMode.text, result.seed, options.outputDir) : Promise.resolve(""),
-          result.whatPeopleSaying.length > 0 ? serializer.writeWhatPeopleSayingCSV(result.whatPeopleSaying, result.seed, options.outputDir) : Promise.resolve("")
+          serializer.writeCSV(result.flat, attemptOptions.outputDir),
+          result.videos.length > 0 ? serializer.writeVideoCSV(result.videos, result.seed, attemptOptions.outputDir) : Promise.resolve(""),
+          result.forums.length > 0 ? serializer.writeForumCSV(result.forums, result.seed, attemptOptions.outputDir) : Promise.resolve(""),
+          result.aiOverview.detected ? serializer.writeAIOverviewCSV(result.aiOverview.citations, result.aiOverview.text, result.seed, attemptOptions.outputDir) : Promise.resolve(""),
+          result.aiMode.detected ? serializer.writeAIModeCSV(result.aiMode.citations, result.aiMode.text, result.seed, attemptOptions.outputDir) : Promise.resolve(""),
+          result.whatPeopleSaying.length > 0 ? serializer.writeWhatPeopleSayingCSV(result.whatPeopleSaying, result.seed, attemptOptions.outputDir) : Promise.resolve("")
         ]);
       }
       return result;
     } catch (err) {
-      if (err instanceof CaptchaError && i < MAX_ATTEMPTS - 1) {
-        continue;
+      if (err instanceof CaptchaError) {
+        const willRetry = i < MAX_ATTEMPTS - 1;
+        console.warn(JSON.stringify({
+          event: "harvest_attempt_captcha",
+          attempt_number: attemptNumber,
+          max_attempts: MAX_ATTEMPTS,
+          message: err.message,
+          will_retry: willRetry
+        }));
+        await emitAttemptEvent(onAttemptEvent, {
+          type: "finished",
+          attemptNumber,
+          maxAttempts: MAX_ATTEMPTS,
+          outcome: "captcha",
+          kernelSessionId: null,
+          questionCount: 0,
+          durationMs: Date.now() - startedAtMs,
+          error: err.message,
+          willRetry,
+          cleanup: {
+            kernelSessionId: null,
+            kernelDeleteStarted: false,
+            kernelDeleteSucceeded: null,
+            kernelDeleteError: null,
+            browserCloseSucceeded: null,
+            browserCloseError: null
+          },
+          debug: null,
+          completedAt: (/* @__PURE__ */ new Date()).toISOString()
+        });
+        if (willRetry) continue;
+        break;
       }
+      await emitAttemptEvent(onAttemptEvent, {
+        type: "finished",
+        attemptNumber,
+        maxAttempts: MAX_ATTEMPTS,
+        outcome: classifyAttemptError(err),
+        kernelSessionId: null,
+        questionCount: 0,
+        durationMs: Date.now() - startedAtMs,
+        error: errorMessage(err),
+        willRetry: false,
+        cleanup: {
+          kernelSessionId: null,
+          kernelDeleteStarted: false,
+          kernelDeleteSucceeded: null,
+          kernelDeleteError: null,
+          browserCloseSucceeded: null,
+          browserCloseError: null
+        },
+        debug: null,
+        completedAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
       throw err;
     }
   }
-  const sessionDesc = options.kernelApiKey ? `${MAX_ATTEMPTS} fresh Kernel.sh sessions` : `${MAX_ATTEMPTS} attempts`;
-  throw new CaptchaError(`CAPTCHA on all ${sessionDesc}. Try again in a few minutes.`);
+  console.warn(JSON.stringify({
+    event: "harvest_captcha_exhausted",
+    max_attempts: MAX_ATTEMPTS,
+    session_kind: kernelApiKey ? "kernel" : "local"
+  }));
+  throw new CaptchaError(sanitizeVendorName(`CAPTCHA on all ${MAX_ATTEMPTS} fresh sessions. Try again in a few minutes.`));
 }
 
 // src/cli.ts